diff --git a/docs/SPEC-TABLES.md b/docs/SPEC-TABLES.md index cb45b97c4..e2a1b9578 100644 --- a/docs/SPEC-TABLES.md +++ b/docs/SPEC-TABLES.md @@ -1699,6 +1699,21 @@ about a value's storage is the map's: field has (§2.1, §2.5), one member, and the node it names takes its index where the map is reached. +**AN ARRAY OF POINTERS IS ITS ARRAY FORM'S ROW OVER A REFERENCE ELEMENT** +(§2.1, §4.2). The element is a pointer, so the ELEMENT is the eight-byte +reference and the ARRAY FORM decides the rest, exactly as the same three +spellings decide it at any other field: + +- `[N]*T` stores `N` references, one member; +- `[..N]*T` stores `N` references beside its `int32` used count, two members; +- `[]*T` stores the sixteen-byte list slot over reference elements, one member. + +Each slot names a node of the walk below, so two slots may name one node and a +slot may name none, and nothing about that is the map's either. `[E]*T`, +`[N]*string` and `[]*bytes` are refused by name (§2.4, §15), and the refusal +reaches a map's value at the entry, because the value is a field of a table +nobody wrote. + **THE HANDLE FOLLOWS THE STORAGE.** `Insert`, `Find` and `Each` hand back a pointer to the `value` member where the storage is ONE member, and the ENTRY where it is two, because two members are not one addressable slot and a caller @@ -1706,9 +1721,12 @@ that cannot set the length or the count cannot fill the value. A `[N]T` value's handle points at the ARRAY rather than at its first element, so the extent survives the handoff. A `*T`, `*string` or `*bytes` value's BUILDER handle is the SLOT, which is what an `Emplace` fills, and the const `Find` answers the -RESOLVED node, one add on the self-relative delta. Where the handle is the -entry, the caller fills `value` and its companion and leaves `key` to the map, -which owns the order the key carries. +RESOLVED node, one add on the self-relative delta. THE ARRAY FORM DECIDES AN +ARRAY OF POINTERS' HANDLE, NEVER THE ELEMENT: a `[N]*T` hands back the array of +references, a `[..N]*T` the entry, a `[]*T` the list slot, and the caller +resolves each slot as it resolves any pointer. Where the handle is the entry, +the caller fills `value` and its companion and leaves `key` to the map, which +owns the order the key carries. **And a map is a BY-VALUE EDGE of the ONE declaration-order walk** (§3.1, schema#438). The numbering, the pack measure and the pack are one walk over @@ -6158,7 +6176,9 @@ without a rule of its own.** The value is an ordinary field of the generated entry and its storage is that field's own row (§2.8), so an entry's body carries a field header, a length and a payload of the value's own kind: an array's `N` and ELEMENT KIND where the value is `[N]T`, `[..N]T`, `[E]T` or an -unbounded `[]T`; a kind `17` NODE INDEX where it is `*T`, `*string` or +unbounded `[]T`, and that element kind is `17` where the element is a pointer, +so a `[N]*T`, a `[..N]*T` and a `[]*T` are the array strategies over node +indices; a kind `17` NODE INDEX where the value is `*T`, `*string` or `*bytes`, and the blob record it names is a record of the node table like any other; and the kind `12` and kind `33` payloads the text strategies above name. The strategies are enumerated over field positions, so each lands inside an diff --git a/internal/codegen/cpptable/maps.go b/internal/codegen/cpptable/maps.go index 26d5af7df..b042629c4 100644 --- a/internal/codegen/cpptable/maps.go +++ b/internal/codegen/cpptable/maps.go @@ -82,24 +82,35 @@ func mapKeyOrderType(f *ir.Field) string { return "uint64_t" } -// mapValueIsPointer reports a `map[K]*T` — the value is a pointer SLOT, so the -// const form's Find answers the resolved `const T *` (docs/SPEC-TABLES.md §2.8). -func mapValueIsPointer(f *ir.Field) bool { return ir.MapValueField(f).Type.Pointer } +// mapValueIsPointer reports a `map[K]*T`, whose value is ONE pointer SLOT, so +// the const form's Find answers the resolved `const T *` (docs/SPEC-TABLES.md +// §2.8). AN ARRAY OF POINTERS IS NOT THIS CASE. `[N]*T`, `[..N]*T` and `[]*T` +// store a `TableRef` PER ELEMENT (§2.1, §4.2), so each takes the arm its array +// form takes and carries `TableRef` as the element; a predicate that read +// `Type.Pointer` alone sent all three here and spelled `At` on an array. +// `[E]*T` is refused by name (§2.4, §15) and reaches no arm at all. +func mapValueIsPointer(f *ir.Field) bool { + value := ir.MapValueField(f) + return value.Type.Pointer && value.Array == ir.ArrayNone && value.KeyEnum == "" +} // mapValueIsPair reports a value whose storage is TWO MEMBERS: the buffer or // the element array, and the `int32` length or count beside it that says how // much of it is used (docs/SPEC-TABLES.md §2.8, §4.2, §7.2). That is -// `string(N)`, `wstring(N)`, `bytes(N)` and `[..N]T`. Each is an ordinary -// field of the generated entry, and two members are not one addressable slot, -// so the handle every map surface hands back is the ENTRY, which reaches both. +// `string(N)`, `wstring(N)`, `bytes(N)`, `[..N]T` and `[..N]*T`. Each is an +// ordinary field of the generated entry, and two members are not one +// addressable slot, so the handle every map surface hands back is the ENTRY, +// which reaches both. A COUNTED ARRAY IS A PAIR WHATEVER ITS ELEMENT IS: a +// pointer element changes the element's storage to a `TableRef` and changes +// nothing about the count beside it. func mapValueIsPair(f *ir.Field) bool { value := ir.MapValueField(f) - if value.Type.Pointer { - return false - } if value.Array == ir.ArrayCounted { return true } + if value.Type.Pointer { + return false + } switch value.Type.Kind { case ir.TString, ir.TWString, ir.TBytes: return true @@ -107,13 +118,13 @@ func mapValueIsPair(f *ir.Field) bool { return false } -// mapValueIsFixedArray reports a `[N]T` value. Its storage is ONE member, an -// array of a fixed extent (docs/SPEC-TABLES.md §4.2), and a pointer to that -// member keeps the extent. `[E]T` is not this case: it stores a +// mapValueIsFixedArray reports a `[N]T` or `[N]*T` value. Its storage is ONE +// member, an array of a fixed extent (docs/SPEC-TABLES.md §4.2), and a pointer +// to that member keeps the extent. `[E]T` is not this case: it stores a // `TableKeyed`, which is an ordinary one-member type spelling. func mapValueIsFixedArray(f *ir.Field) bool { value := ir.MapValueField(f) - return !value.Type.Pointer && value.KeyEnum == "" && value.Array == ir.ArrayFixed + return value.KeyEnum == "" && value.Array == ir.ArrayFixed } // mapValueArrayAlias names the array type a `[N]T` value's handle points at. @@ -121,6 +132,20 @@ func mapValueIsFixedArray(f *ir.Field) bool { // around the function name, so the alias is what makes the handle readable. func mapValueArrayAlias(f *ir.Field) string { return mapEntryOf(f).Name + "Value" } +// mapValueElementType is the C++ type ONE ELEMENT of an array value is stored +// as: a `TableRef` where the element is a pointer, which is the row §4.2 gives +// an array of pointers at a field, and the element's own type otherwise +// (docs/SPEC-TABLES.md §2.1, §4.2). +func (g *tableGen) mapValueElementType(f *ir.Field) string { + value := ir.MapValueField(f) + if value.Type.Pointer { + g.noteRef(value.Type.Name) + return "TableRef" + } + typ, _ := g.cppFieldType(value.Type) + return typ +} + // ---- the runtime (docs/SPEC-TABLES.md §2.8) ---- // tableMapRuntime is the map half of the variable-length runtime: the storage @@ -886,10 +911,10 @@ func (g *tableGen) emitMapEntrySurface(owner *ir.Struct, f *ir.Field) { g.pf("inline const %s * TableEntryFound( const %s * entry ) { return entry != NULL ? %sAt( entry->value ) : NULL; }\n", t, n, t) g.pf("inline TableRef * TableEntryValue( %s * entry ) { return &entry->value; } // the builder hands back the SLOT\n", n) case mapValueIsPair(f): - // a map[K]string(N), map[K]wstring(N), map[K]bytes(N) or map[K][..N]T: - // the value's storage is the buffer or the element array and the int32 - // used length or count beside it (§2.8, §4.2, §7.2), two members, so - // the ENTRY is the handle that reaches both. A caller fills + // a map[K]string(N), map[K]wstring(N), map[K]bytes(N), map[K][..N]T or + // map[K][..N]*T: the value's storage is the buffer or the element array + // and the int32 used length or count beside it (§2.8, §4.2, §7.2), two + // members, so the ENTRY is the handle that reaches both. A caller fills // `entry->value` and its companion, and leaves `entry->key` to the // map, which owns the sort the key carries. g.pf("// THIS VALUE'S STORAGE IS A PAIR (§2.8, §4.2, §7.2): `value` beside\n") @@ -898,10 +923,12 @@ func (g *tableGen) emitMapEntrySurface(owner *ir.Struct, f *ir.Field) { g.pf("inline const %s * TableEntryFound( const %s * entry ) { return entry; }\n", n, n) g.pf("inline %s * TableEntryValue( %s * entry ) { return entry; }\n", n, n) case mapValueIsFixedArray(f): - // a map[K][N]T: the value's storage is ONE member, an array of a fixed - // extent (§4.2), so the handle is a pointer to the ARRAY and keeps - // that extent. The alias is what lets a return type spell it. - elem, _ := g.cppFieldType(value.Type) + // a map[K][N]T or map[K][N]*T: the value's storage is ONE member, an + // array of a fixed extent (§4.2), so the handle is a pointer to the + // ARRAY and keeps that extent. A pointer element makes that element a + // `TableRef` (§2.1) and changes nothing else. The alias is what lets a + // return type spell it. + elem := g.mapValueElementType(f) alias := mapValueArrayAlias(f) g.pf("// A FIXED ARRAY VALUE IS ONE MEMBER (§2.8, §4.2): the handle points at the\n") g.pf("// ARRAY and not at its first element, so the extent survives the handoff.\n") diff --git a/tables/maps/Crews.schema b/tables/maps/Crews.schema new file mode 100644 index 000000000..265dacc55 --- /dev/null +++ b/tables/maps/Crews.schema @@ -0,0 +1,14 @@ +package mapdemo + +// A map whose VALUE is a COUNTED ARRAY OF POINTERS (docs/SPEC-TABLES.md §2.1, +// §2.8): "A VALUE is anything a table field can hold ... `[..N]T`", and the +// element is a pointer, so the value is an ordinary field of the generated +// entry and its storage is that field's own row, `TableRef value[N]` beside +// its `int32` used count (§4.2). Two members are not one addressable slot, so +// the handle every map surface hands back is the ENTRY, which reaches both. + +table Crews +{ + members map[uint32][..2]*Item // [..N]*T: pointer slots and the used count beside them + after int32 +} diff --git a/tables/maps/Pairs.schema b/tables/maps/Pairs.schema new file mode 100644 index 000000000..12e341bbb --- /dev/null +++ b/tables/maps/Pairs.schema @@ -0,0 +1,15 @@ +package mapdemo + +// A map whose VALUE is a FIXED ARRAY OF POINTERS (docs/SPEC-TABLES.md §2.1, +// §2.8): "A VALUE is anything a table field can hold ... `[N]T`", and the +// element is a pointer, so the value is an ordinary field of the generated +// entry and its storage is that field's own row, `TableRef value[N]` (§4.2). +// That is ONE member, so the handle points at the ARRAY and keeps the extent. +// Every slot names a node, and two slots may name one node exactly as two +// pointer fields may. + +table Pairs +{ + slots map[uint32][2]*Item // [N]*T: a fixed extent of pointer slots + after int32 +} diff --git a/tables/maps/Trails.schema b/tables/maps/Trails.schema new file mode 100644 index 000000000..c52fee00c --- /dev/null +++ b/tables/maps/Trails.schema @@ -0,0 +1,15 @@ +package mapdemo + +// A map whose VALUE is an UNBOUNDED ARRAY OF POINTERS (docs/SPEC-TABLES.md +// §2.1, §2.8, §2.9): "A VALUE is anything a table field can hold ... and an +// unbounded `[]T` (§2.9)", and the element is a pointer, so the value is an +// ordinary field of the generated entry and its storage is that field's own +// row, the sixteen-byte list slot over `TableRef` elements (§4.2). That is one +// member, so the handle is the slot itself and the list's own surface reaches +// the pointer slots, which ride in the HOLDER'S node extent as any list's do. + +table Trails +{ + steps map[uint32][]*Item // []*T: an unbounded extent of pointer slots + after int32 +} diff --git a/tables/maps/tables.baseline b/tables/maps/tables.baseline index 5a7c732e8..a74c2dece 100644 --- a/tables/maps/tables.baseline +++ b/tables/maps/tables.baseline @@ -29,6 +29,10 @@ table 0a53e00afba279af.7b024c46e98d3404 field key id=0x3dc94a19365b10ec kind=8 field value id=0x7ce4fd9430e80cea kind=17 type=ShipConfig +table 1404200dab337086.e68c2e6bb1ee5646 + field key id=0x3dc94a19365b10ec kind=8 + field value id=0x7ce4fd9430e80cea kind=14 elem=17 type=Item array=fixed bound=2 + table 1f781dc01a2b5152.a3a7061ff10a8138 field key id=0x3dc94a19365b10ec kind=12 size=8 field value id=0x7ce4fd9430e80cea kind=14 elem=4 array=fixed bound=3 @@ -57,6 +61,10 @@ table Chunks field blobs id=0x14e2eaab9cde925b kind=14 elem=13 array=map keykind=4 field after id=0xbf82010f6f71eae9 kind=4 +table Crews + field members id=0x79d594675e391090 kind=14 elem=13 array=map keykind=8 + field after id=0xbf82010f6f71eae9 kind=4 + table Depth field one id=0x1a08aa1921ca5caf kind=13 type=Squad field many id=0x1f6459a2cea1fc02 kind=14 elem=13 type=Squad array=bounded bound=3 @@ -83,6 +91,10 @@ table Fleet table Item field count id=0xb1e5e28e4479a274 kind=4 +table Pairs + field slots id=0xe68c2e6bb1ee5646 kind=14 elem=13 array=map keykind=8 + field after id=0xbf82010f6f71eae9 kind=4 + table Row field entries id=0xc5b2a72c0845a253 kind=14 elem=13 array=map keykind=12 keybound=8 field after id=0xbf82010f6f71eae9 kind=4 @@ -112,6 +124,10 @@ table Text field blobs id=0x14e2eaab9cde925b kind=14 elem=13 array=map keykind=4 field after id=0xbf82010f6f71eae9 kind=4 +table Trails + field steps id=0x124250ad5a5b6d14 kind=14 elem=13 array=map keykind=8 + field after id=0xbf82010f6f71eae9 kind=4 + table WideRow field entries id=0xc5b2a72c0845a253 kind=14 elem=13 array=map keykind=8 field after id=0xbf82010f6f71eae9 kind=4 @@ -124,6 +140,14 @@ table b413964e3571a316.c5b2a72c0845a253 field key id=0x3dc94a19365b10ec kind=8 field value id=0x7ce4fd9430e80cea kind=13 type=Item +table b4578774a78fb150.124250ad5a5b6d14 + field key id=0x3dc94a19365b10ec kind=8 + field value id=0x7ce4fd9430e80cea kind=14 elem=17 type=Item array=unbounded + +table c85b940060088651.79d594675e391090 + field key id=0x3dc94a19365b10ec kind=8 + field value id=0x7ce4fd9430e80cea kind=14 elem=17 type=Item array=bounded bound=2 + table e8130af045a036f8.2b7dea192bb7be29 field key id=0x3dc94a19365b10ec kind=9 field value id=0x7ce4fd9430e80cea kind=13 type=Item @@ -170,3 +194,6 @@ union Force ### 2026-09-07 (UTC) — Cells, Runs, Slots, Spans, Docs and Chunks: one map per value kind that carries an extent or names a buffer node (#628) - no compatibility-affecting edits; the wire absorbs the rest + +### 2026-09-07 (UTC) — Pairs, Crews and Trails: one map per value kind whose element is a pointer (#666) +- no compatibility-affecting edits; the wire absorbs the rest diff --git a/test/conformance/harness/maps_test.go b/test/conformance/harness/maps_test.go index e324d4b89..1ad9b8f50 100644 --- a/test/conformance/harness/maps_test.go +++ b/test/conformance/harness/maps_test.go @@ -36,6 +36,10 @@ func TestTheToolWritesTheReferencesMapBytes(t *testing.T) { // unbounded array, a text buffer and a byte buffer, each under a map key {"map_cells", "Cells"}, {"map_runs", "Runs"}, {"map_slots", "Slots"}, {"map_spans", "Spans"}, {"map_docs", "Docs"}, {"map_chunks", "Chunks"}, + // one row per value kind whose ELEMENT is a pointer (#666): a fixed + // array, a counted array and an unbounded array of node references, + // each carrying a shared node and a slot that names none + {"map_pairs", "Pairs"}, {"map_crews", "Crews"}, {"map_trails", "Trails"}, } { name, rootName := tc.name, tc.root data, err := os.ReadFile(root + "testdata/wire/tables/" + name + ".bin") diff --git a/test/tables/maps_main.cpp b/test/tables/maps_main.cpp index 0b3796ff2..f925a3de2 100644 --- a/test/tables/maps_main.cpp +++ b/test/tables/maps_main.cpp @@ -22,6 +22,9 @@ #include "SpansTable.h" #include "DocsTable.h" #include "ChunksTable.h" +#include "PairsTable.h" +#include "CrewsTable.h" +#include "TrailsTable.h" #include "wirebuilder.h" using namespace mapdemo; @@ -2085,6 +2088,350 @@ static void test_byte_blob_values() free( chunk_json ); } +// ---- A MAP WHOSE VALUE IS AN ARRAY OF POINTERS (§2.1, §2.8, §4.2) ---- +// +// The element is a pointer, so the value's storage is the array of `TableRef` +// §4.2 gives an array of pointers at a field, and the handle follows the ARRAY +// FORM and not the element: the array for a `[N]*T`, the ENTRY for a `[..N]*T` +// because the count beside it makes two members, and the list SLOT for a +// `[]*T`. Each slot names a node of the one declaration-order walk, so two +// slots may name one node and a slot may name none. + +static void test_pointer_array_values() +{ + // [2]*Item: ONE member, TableRef value[2], so the handle is a pointer to + // the ARRAY. Key 4 spends both slots on ONE node, key 9 leaves the second + // null, so a share and a null both ride through a map value. + PairsBuilder b; + Pairs * p = b.GetRoot(); + static const uint32_t keys[2] = { 9, 4 }; // OUT OF KEY ORDER + for ( int i = 0; i < 2; i++ ) + { + PairsSlotsEntryValue * slots = PairsSlotsInsert( b.main, p->slots, keys[i] ); + CHECK( slots != NULL ); + if ( slots == NULL ) { return; } + Item * item = ItemEmplace( b.main, ( *slots )[0] ); + CHECK( item != NULL ); + if ( item == NULL ) { return; } + item->count = (int32_t) ( keys[i] * 10 ); + // key 4 names the same node twice; key 9 leaves the second slot null + if ( keys[i] == 4 ) { ( *slots )[1] = ( *slots )[0]; } + } + p->after = 5; + + PairsSlotsEntryValue * found = PairsSlotsFind( b.arena, p->slots, 4u ); + CHECK( found != NULL ); + if ( found != NULL ) + { + const Item * shared = ItemAt( b.arena, ( *found )[0] ); + CHECK( shared != NULL && shared->count == 40 ); + CHECK( ItemAt( b.arena, ( *found )[1] ) == shared ); // two slots, one node + } + + const int64_t measured = PairsMeasure( b ); + static uint8_t wire[1u << 16]; + const int64_t n = PairsSave( b, wire, sizeof( wire ) ); + CHECK_EQ( measured, n ); + pin_golden( "map_pairs", wire, n ); + + const int64_t need = PairsLoadMeasure( wire, n ); + CHECK( need > 0 ); + uint8_t * region = (uint8_t *) MEASURED_CALLOC( need, 0 ); + if ( region == NULL ) { return; } + TableReport report; + const Pairs * loaded = PairsLoad( region, need, wire, n, &report ); + CHECK( loaded != NULL ); + CHECK( !report.malformed ); + if ( loaded != NULL ) + { + const PairsSlotsEntryValue * four = loaded->slots.Find( 4u ); + CHECK( four != NULL ); + if ( four != NULL ) + { + const Item * a = ItemAt( ( *four )[0] ); + CHECK( a != NULL && a->count == 40 ); + CHECK( ItemAt( ( *four )[1] ) == a ); // ONE node in the region too + } + const PairsSlotsEntryValue * nine = loaded->slots.Find( 9u ); + CHECK( nine != NULL ); + if ( nine != NULL ) + { + const Item * only = ItemAt( ( *nine )[0] ); + CHECK( only != NULL && only->count == 90 ); + CHECK( ItemAt( ( *nine )[1] ) == NULL ); // the null slot stayed null + } + CHECK_EQ( loaded->after, 5 ); // and the parent read on past the map + static uint8_t again[1u << 16]; + CHECK_EQ( PairsSave( loaded, again, sizeof( again ) ), n ); + CHECK( memcmp( again, wire, (size_t) n ) == 0 ); + } + free( region ); + + CHECK( b.Lock() ); + const Pairs * locked = b.AsConst(); + CHECK( locked != NULL ); + if ( locked == NULL ) { return; } + const int64_t json_bytes = PairsToJsonMeasure( locked ); + CHECK( json_bytes > 0 ); + char * json = (char *) MEASURED_CALLOC( json_bytes, 1 ); + if ( json == NULL ) { return; } + CHECK_EQ( PairsToJson( locked, json, json_bytes ), json_bytes ); + // the value takes the pointer row inside a JSON array: the node's object, + // `&node` where one node is named twice, `null` where a slot names none + CHECK( strstr( json, "\"&node\": 1" ) != NULL ); + CHECK( strstr( json, "\"count\": 40" ) != NULL ); + CHECK( strstr( json, "\"count\": 90" ) != NULL ); + CHECK( strstr( json, "null" ) != NULL ); + const char * four_at = strstr( json, "\"4\"" ); + const char * nine_at = strstr( json, "\"9\"" ); + CHECK( four_at != NULL && nine_at != NULL && four_at < nine_at ); // ASCENDING + + PairsBuilder into; + TableReport text_report; + CHECK( PairsFromJson( into, json, json_bytes, &text_report ) ); + CHECK( !text_report.malformed ); + static uint8_t from_text[1u << 16]; + CHECK_EQ( PairsSave( into, from_text, sizeof( from_text ) ), n ); + CHECK( memcmp( from_text, wire, (size_t) n ) == 0 ); + free( json ); + + // a DUPLICATE key REPLACES and the value half is reset WHOLE: EVERY slot + // back to null, so the repeat names no node the first insert named + PairsBuilder dup; + Pairs * d = dup.GetRoot(); + PairsSlotsEntryValue * first = PairsSlotsInsert( dup.main, d->slots, 4u ); + CHECK( first != NULL ); + if ( first == NULL ) { return; } + CHECK( ItemEmplace( dup.main, ( *first )[0] ) != NULL ); + CHECK( ItemEmplace( dup.main, ( *first )[1] ) != NULL ); + PairsSlotsEntryValue * repeat = PairsSlotsInsert( dup.main, d->slots, 4u ); + CHECK( repeat == first ); // the same entry, key and address unchanged + CHECK( repeat != NULL && ( *repeat )[0].value == 0 && ( *repeat )[1].value == 0 ); + CHECK_EQ( d->slots.count, 1 ); +} + +static void test_counted_pointer_array_values() +{ + // [..2]*Item: a PAIR, the TableRef array beside its int32 used count, so + // the handle is the ENTRY, which reaches both. + CrewsBuilder b; + Crews * c = b.GetRoot(); + static const uint32_t keys[2] = { 9, 4 }; // OUT OF KEY ORDER + for ( int i = 0; i < 2; i++ ) + { + CrewsMembersEntry * entry = CrewsMembersInsert( b.main, c->members, keys[i] ); + CHECK( entry != NULL ); + if ( entry == NULL ) { return; } + entry->value_count = keys[i] == 4 ? 2 : 1; + for ( int j = 0; j < entry->value_count; j++ ) + { + Item * item = ItemEmplace( b.main, entry->value[j] ); + CHECK( item != NULL ); + if ( item == NULL ) { return; } + item->count = (int32_t) ( keys[i] * 10 + j ); + } + } + c->after = 2; + + const int64_t measured = CrewsMeasure( b ); + static uint8_t wire[1u << 16]; + const int64_t n = CrewsSave( b, wire, sizeof( wire ) ); + CHECK_EQ( measured, n ); + pin_golden( "map_crews", wire, n ); + + const int64_t need = CrewsLoadMeasure( wire, n ); + CHECK( need > 0 ); + uint8_t * region = (uint8_t *) MEASURED_CALLOC( need, 0 ); + if ( region == NULL ) { return; } + TableReport report; + const Crews * loaded = CrewsLoad( region, need, wire, n, &report ); + CHECK( loaded != NULL ); + CHECK( !report.malformed ); + if ( loaded != NULL ) + { + // key 4 was inserted SECOND and carries TWO slots, key 9 first and + // carries one, so a walk that numbered the nodes in insertion order + // rather than in key order would read one entry's node as another's + const CrewsMembersEntry * four = loaded->members.Find( 4u ); + CHECK( four != NULL && four->value_count == 2 ); + if ( four != NULL && four->value_count == 2 ) + { + const Item * a = ItemAt( four->value[0] ); + const Item * second = ItemAt( four->value[1] ); + CHECK( a != NULL && a->count == 40 ); + CHECK( second != NULL && second->count == 41 ); + } + const CrewsMembersEntry * nine = loaded->members.Find( 9u ); + CHECK( nine != NULL && nine->value_count == 1 ); + if ( nine != NULL && nine->value_count == 1 ) + { + const Item * only = ItemAt( nine->value[0] ); + CHECK( only != NULL && only->count == 90 ); + } + CHECK_EQ( loaded->after, 2 ); + static uint8_t again[1u << 16]; + CHECK_EQ( CrewsSave( loaded, again, sizeof( again ) ), n ); + CHECK( memcmp( again, wire, (size_t) n ) == 0 ); + } + free( region ); + + CHECK( b.Lock() ); + const Crews * locked = b.AsConst(); + CHECK( locked != NULL ); + if ( locked == NULL ) { return; } + const int64_t json_bytes = CrewsToJsonMeasure( locked ); + CHECK( json_bytes > 0 ); + char * json = (char *) MEASURED_CALLOC( json_bytes, 1 ); + if ( json == NULL ) { return; } + CHECK_EQ( CrewsToJson( locked, json, json_bytes ), json_bytes ); + // the USED COUNT decides the row's length: two nodes under 4, one under 9 + CHECK( strstr( json, "\"count\": 40" ) != NULL ); + CHECK( strstr( json, "\"count\": 41" ) != NULL ); + CHECK( strstr( json, "\"count\": 90" ) != NULL ); + const char * four_at = strstr( json, "\"4\"" ); + const char * nine_at = strstr( json, "\"9\"" ); + CHECK( four_at != NULL && nine_at != NULL && four_at < nine_at ); // ASCENDING + + CrewsBuilder into; + TableReport text_report; + CHECK( CrewsFromJson( into, json, json_bytes, &text_report ) ); + CHECK( !text_report.malformed ); + static uint8_t from_text[1u << 16]; + CHECK_EQ( CrewsSave( into, from_text, sizeof( from_text ) ), n ); + CHECK( memcmp( from_text, wire, (size_t) n ) == 0 ); + free( json ); + + // a DUPLICATE resets BOTH members of the pair, every slot AND the count + CrewsBuilder dup; + Crews * d = dup.GetRoot(); + CrewsMembersEntry * first = CrewsMembersInsert( dup.main, d->members, 4u ); + CHECK( first != NULL ); + if ( first == NULL ) { return; } + first->value_count = 2; + CHECK( ItemEmplace( dup.main, first->value[0] ) != NULL ); + CrewsMembersEntry * repeat = CrewsMembersInsert( dup.main, d->members, 4u ); + CHECK( repeat == first ); + CHECK( repeat != NULL && repeat->value_count == 0 ); + CHECK( repeat != NULL && repeat->value[0].value == 0 ); + CHECK_EQ( d->members.count, 1 ); +} + +static void test_unbounded_pointer_array_values() +{ + // []*Item: ONE member, the sixteen-byte list slot over TableRef elements, + // so the handle is the slot itself. Its slots ride in the HOLDER'S node + // extent as any list's do, and each names a node of the one walk. + TrailsBuilder b; + Trails * t = b.GetRoot(); + static const uint32_t keys[2] = { 9, 4 }; // OUT OF KEY ORDER + for ( int i = 0; i < 2; i++ ) + { + TableList * steps = TrailsStepsInsert( b.main, t->steps, keys[i] ); + CHECK( steps != NULL ); + if ( steps == NULL ) { return; } + TableRef * slot = TrailsStepsEntryValueAdd( b.main, *steps ); + CHECK( slot != NULL ); + if ( slot == NULL ) { return; } + Item * item = ItemEmplace( b.main, *slot ); + CHECK( item != NULL ); + if ( item == NULL ) { return; } + item->count = (int32_t) ( keys[i] * 10 ); + if ( keys[i] == 4 ) + { + // two slots, one node, and a slot that names none past them + TableRef * share = TrailsStepsEntryValueAdd( b.main, *steps ); + CHECK( share != NULL ); + if ( share == NULL ) { return; } + *share = *slot; + CHECK( TrailsStepsEntryValueAdd( b.main, *steps ) != NULL ); + } + } + t->after = 1; + + const int64_t measured = TrailsMeasure( b ); + static uint8_t wire[1u << 16]; + const int64_t n = TrailsSave( b, wire, sizeof( wire ) ); + CHECK_EQ( measured, n ); + pin_golden( "map_trails", wire, n ); + + // the region: the entry array first, then EACH ENTRY'S list slots, the + // pre-order placement §2.8 states + const int64_t need = TrailsLoadMeasure( wire, n ); + CHECK( need > 0 ); + uint8_t * region = (uint8_t *) MEASURED_CALLOC( need, 0 ); + if ( region == NULL ) { return; } + TableReport report; + const Trails * loaded = TrailsLoad( region, need, wire, n, &report ); + CHECK( loaded != NULL ); + CHECK( !report.malformed ); + if ( loaded != NULL ) + { + // A LIST WHOSE SLOTS WERE NOT PLACED HAS A NULL ELEMENT POINTER, so + // the count is checked and the slots are read only under it: a walk + // that never laid the extent must report on a CHECK and not on a fault + const TableList * four = loaded->steps.Find( 4u ); + CHECK( four != NULL && four->count == 3 ); + if ( four != NULL && four->count == 3 ) + { + const Item * a = ( *four )[0]; + CHECK( a != NULL && a->count == 40 ); + CHECK( ( *four )[1] == a ); // two slots, one node + CHECK( ( *four )[2] == NULL ); // and a slot that names none + } + const TableList * nine = loaded->steps.Find( 9u ); + CHECK( nine != NULL && nine->count == 1 ); + if ( nine != NULL && nine->count == 1 ) + { + const Item * only = ( *nine )[0]; + CHECK( only != NULL && only->count == 90 ); + } + CHECK_EQ( loaded->after, 1 ); + static uint8_t again[1u << 16]; + CHECK_EQ( TrailsSave( loaded, again, sizeof( again ) ), n ); + CHECK( memcmp( again, wire, (size_t) n ) == 0 ); + } + free( region ); + + CHECK( b.Lock() ); + const Trails * locked = b.AsConst(); + CHECK( locked != NULL ); + if ( locked == NULL ) { return; } + const int64_t json_bytes = TrailsToJsonMeasure( locked ); + CHECK( json_bytes > 0 ); + char * json = (char *) MEASURED_CALLOC( json_bytes, 1 ); + if ( json == NULL ) { return; } + CHECK_EQ( TrailsToJson( locked, json, json_bytes ), json_bytes ); + CHECK( strstr( json, "\"&node\": 1" ) != NULL ); + CHECK( strstr( json, "\"count\": 40" ) != NULL ); + CHECK( strstr( json, "\"count\": 90" ) != NULL ); + CHECK( strstr( json, "null" ) != NULL ); + const char * four_at = strstr( json, "\"4\"" ); + const char * nine_at = strstr( json, "\"9\"" ); + CHECK( four_at != NULL && nine_at != NULL && four_at < nine_at ); // ASCENDING + + TrailsBuilder into; + TableReport text_report; + CHECK( TrailsFromJson( into, json, json_bytes, &text_report ) ); + CHECK( !text_report.malformed ); + static uint8_t from_text[1u << 16]; + CHECK_EQ( TrailsSave( into, from_text, sizeof( from_text ) ), n ); + CHECK( memcmp( from_text, wire, (size_t) n ) == 0 ); + free( json ); + + // a DUPLICATE resets the list SLOT, so the repeat starts empty and the + // first insert's nodes are unreachable rather than inherited + TrailsBuilder dup; + Trails * d = dup.GetRoot(); + TableList * first = TrailsStepsInsert( dup.main, d->steps, 4u ); + CHECK( first != NULL ); + if ( first == NULL ) { return; } + CHECK( TrailsStepsEntryValueAdd( dup.main, *first ) != NULL ); + TableList * repeat = TrailsStepsInsert( dup.main, d->steps, 4u ); + CHECK( repeat == first ); + CHECK( repeat != NULL && repeat->count == 0 ); + CHECK_EQ( d->steps.count, 1 ); +} + // ---- the MESSAGE FORM over maps (docs/SPEC-TABLES.md §2.8, §3.3) ---- // // A map rides on the message wire as its thirty-two bit count and its entries @@ -2179,6 +2526,9 @@ int main( int argc, char ** argv ) test_unbounded_values(); test_blob_values(); test_byte_blob_values(); + test_pointer_array_values(); + test_counted_pointer_array_values(); + test_unbounded_pointer_array_values(); test_message_form(); if ( failures != 0 ) diff --git a/testdata/golden/tables/maps/CellsTable.h b/testdata/golden/tables/maps/CellsTable.h index 907c00422..161361a45 100644 --- a/testdata/golden/tables/maps/CellsTable.h +++ b/testdata/golden/tables/maps/CellsTable.h @@ -376,7 +376,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -878,13 +878,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1406,7 +1406,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1424,10 +1424,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1436,6 +1436,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1444,54 +1445,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3429,25 +3439,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -3752,6 +3764,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4128,6 +4150,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -5676,7 +5703,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6434,7 +6461,7 @@ inline bool CellsRowsEntrySaveMessageBody( TableBitWriter & w, const CellsRowsEn for ( int32_t i = 0; i < 3; i++ ) { if ( value.value[i] != 0 ) { rides_value = true; break; } } if ( rides_value ) { - w.put( 10, kTableMessageRefBitsHere ); + w.put( 11, kTableMessageRefBitsHere ); for ( int32_t i = 0; i < 3; i++ ) { w.put( (uint64_t) ( value.value[i] ), 32 ); @@ -6480,15 +6507,13 @@ inline bool CellsRowsEntryLoadMessageBody( TableBitReader & r, const TableVocabu { uint64_t n = 0; if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } - int32_t kept = 0; - if ( n > (uint64_t) 8 ) { kept = 8; report->clamped++; } else { kept = (int32_t) n; } - for ( uint64_t i = 0; i < n; i++ ) - { - uint64_t by = 0; - if ( !r.get( by, 8 ) ) { report->malformed = true; return false; } - if ( (int32_t) i < kept ) { value.key[i] = (char) by; } - } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); value.key[kept] = 0; + r.offset += (int64_t) n * 8; value.key_length = kept; } break; @@ -6869,7 +6894,7 @@ inline bool CellsSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_rows.ok ) { return false; } // the sort could not run if ( order_rows.count > 0 ) { - w.put( 16, kTableMessageRefBitsHere ); + w.put( 17, kTableMessageRefBitsHere ); w.put( (uint64_t) order_rows.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_rows.count; i++ ) { @@ -6880,7 +6905,7 @@ inline bool CellsSaveMessageBody( const Ctx & ctx, const TableNumbering & number } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/ChunksTable.h b/testdata/golden/tables/maps/ChunksTable.h index 6a1d6b153..b6de29938 100644 --- a/testdata/golden/tables/maps/ChunksTable.h +++ b/testdata/golden/tables/maps/ChunksTable.h @@ -376,7 +376,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -878,13 +878,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1406,7 +1406,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1424,10 +1424,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1436,6 +1436,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1444,54 +1445,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3429,25 +3439,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -3752,6 +3764,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4128,6 +4150,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -5676,7 +5703,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6392,7 +6419,7 @@ inline bool ChunksBlobsEntrySaveMessageBody( const Ctx & ctx, const TableNumberi (void) ctx; (void) numbering; (void) index_bits; if ( value.key != 0 ) { - w.put( 11, kTableMessageRefBitsHere ); + w.put( 12, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.key ), 32 ); } { @@ -6776,7 +6803,7 @@ inline bool ChunksSaveMessageBody( const Ctx & ctx, const TableNumbering & numbe if ( !order_blobs.ok ) { return false; } // the sort could not run if ( order_blobs.count > 0 ) { - w.put( 18, kTableMessageRefBitsHere ); + w.put( 19, kTableMessageRefBitsHere ); w.put( (uint64_t) order_blobs.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_blobs.count; i++ ) { @@ -6787,7 +6814,7 @@ inline bool ChunksSaveMessageBody( const Ctx & ctx, const TableNumbering & numbe } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body @@ -7200,7 +7227,7 @@ inline bool ChunksBlobsEntryNumber( const Ctx & ctx, TableNumbering & numbering, TableNodeEntry node; node.node = (const void *) blob; node.type_id = kTableBytesTypeId; - node.type_slot = 49; // its slot in the unit's vocabulary (§3.3) + node.type_slot = 55; // its slot in the unit's vocabulary (§3.3) node.measure = &TableBlobMeasureThunk; node.save = &TableBlobSaveThunk; node.message_measure = &TableBlobMessageMeasureThunk; diff --git a/testdata/golden/tables/maps/CrewsTable.cpp b/testdata/golden/tables/maps/CrewsTable.cpp new file mode 100644 index 000000000..d42cb3cd5 --- /dev/null +++ b/testdata/golden/tables/maps/CrewsTable.cpp @@ -0,0 +1,3448 @@ +// Code generated by the schema compiler from Crews.schema. DO NOT EDIT. +// SPDX-License-Identifier: NONE — this generated output is yours, under terms of +// your choice. See the LICENSE exception in the schema compiler; the compiler is +// AGPL-3.0, its output is not. +// package mapdemo — the TABLE wire's text form (docs/SPEC-TABLES.md §16). +// Compile this file to use FromJson / ToJson; a project that +// never reads or writes a text does not compile it and pays nothing. + +#include "CrewsTable.h" + +#include // the text form: number formatting +#include // the text form: exact number conversion +#include // the text form: the runtime's decimal point + +// The guard is not vestigial. Several mapdemo Table.cpp files may be +// concatenated into ONE translation unit — a unity build — and without it +// each would redefine the walk. It is also why the walk's functions may be +// weak (vague linkage) across separate objects: ODR requires their +// definitions to be token-identical, and the generic-walk gate is what +// proves that, byte for byte, across every generated .cpp. +#ifndef MAPDEMO_SCHEMA_TABLE_JSON +#define MAPDEMO_SCHEMA_TABLE_JSON + +namespace mapdemo { + +// ---- the pointer adapters (docs/SPEC-TABLES.md §16.7) ---- +// +// The walk below is ONE walk, byte-identical in every generated .cpp, and a +// pointer is the one kind it cannot walk alone: reading one needs the +// builder's arena and writing one needs a region's deref, and neither exists +// in a unit that declares no pointer. So the walk calls these three and does +// not define them. A unit with no pointer defines them as stubs no field ever +// reaches; a pointered unit defines them in the graph half that follows the +// walk. + +struct TableJsonIn; +struct TableJsonOut; + +// a pointer field's object, or the `&node` reference standing in for it, into +// the slot; the cursor is on the opening brace +inline bool TableJsonReadPointer( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); +// the node a pointer slot names, in place — or as `&node` when it is shared +inline bool TableJsonWritePointer( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// the FIRST key of an object the walk is skipping begins with `&`: the cursor is +// on its value. A dropped definition still takes its label (§16.7); a fixed reader +// skips the value whole, as it skips everything else it does not place. +inline bool TableJsonSkippedAmpersand( TableJsonIn & in, const char * key, int32_t depth ); + +// ---- the map and list adapters (docs/SPEC-TABLES.md §2.8, §2.9, §16) ---- +// +// A MAP and an UNBOUNDED ARRAY are the other constructs the walk cannot walk +// alone: their arrays live behind a TableMap or a TableList this +// walk has no name for, reading one needs the builder's arena, and neither +// exists in a unit that declares neither construct. Same shape as the +// pointer's three: declared here, defined after the walk by whichever half +// the unit carries. Both are OUT-OF-LINE ARRAYS to the descriptors (§8.1): +// array_bound = 0 is the tell, and the type name says which of the two. + +// a map field: an out-of-line array whose type name spells the map +inline bool TableJsonIsMap( const TableFieldInfo * f ); +// the map as a plain JSON object keyed by the KEY, in ASCENDING key order +inline bool TableJsonWriteMap( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// that object back into the slot, in whatever order the text gives it +inline bool TableJsonReadMap( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); +// an unbounded array: the other out-of-line array +inline bool TableJsonIsList( const TableFieldInfo * f ); +// the list as a JSON array, in INDEX order +inline bool TableJsonWriteList( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// that array back into the slot, every element the text carries +inline bool TableJsonReadList( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); + +// ---- json walk: begin ---- +// +// The TEXT form (docs/SPEC-TABLES.md §16): one table, one text, one walk over the +// reflection descriptors (§8). Reading fills ONE caller-owned instance and +// allocates nothing beyond it; writing targets a caller buffer with the +// wire's measure/write symmetry. Everything AROUND this — which file goes +// with which instance, what key an instance is filed under, how instances +// link into a root table's collections — is a packer's opinion and stays +// with the tool that holds it. +// +// The dialect: trailing commas are accepted on read (the authoring files +// this exists for carry them) and never written; comments are not JSON and +// are refused; unknown keys are skipped and counted; a duplicate key is +// last-wins and counted; a key present with the wrong JSON type is skipped +// and counted, never coerced. + +static const int32_t kTableJsonMaxDepth = 128; + +// A key longer than this cannot name a field, so it is skipped as unknown. +static const int32_t kTableJsonMaxKey = 256; + +// The longest numeric token the walk will convert. Anything longer is a +// value no field can hold and counts as a kind mismatch. +static const int32_t kTableJsonMaxNumber = 512; + +// The decimal point the C runtime is CURRENTLY using. Number conversion is +// the one locale-sensitive corner of the grammar — JSON's point is always +// '.', the runtime's is whatever the program set — so every number crosses +// this one character on the way out and on the way back in. Nothing else in +// the walk consults the locale. +inline char TableJsonDecimalPoint() +{ + const struct lconv * conv = localeconv(); + if ( conv != NULL && conv->decimal_point != NULL && conv->decimal_point[0] != 0 ) + { + return conv->decimal_point[0]; + } + return '.'; +} + +// ---- storage access: the descriptors give an offset and a width, and the +// ---- storage is the HOST's, so every load and store goes through a width +// ---- switch rather than a memcpy into the low bytes of a wider word + +// finite: not a NaN, not an infinity. Written without — the walk's +// runtime surface stays the handful of functions it already names. +// A vocabulary entry the descriptor could not spell. The generated name +// functions answer "???" for a value outside the declared set, and that is +// not a name — writing it would put a spelling in the text that the reader +// then counts as unknown, turning a refusal into a silent loss. +inline bool TableJsonNamed( const char * name ) +{ + return name != NULL && strcmp( name, "???" ) != 0; +} + +inline bool TableJsonFinite( double v ) +{ + return v == v && v <= 1.7976931348623157e308 && v >= -1.7976931348623157e308; +} + +inline uint64_t TableJsonGetRaw( const void * storage, uint32_t width ) +{ + switch ( width ) + { + case 1: { uint8_t v = 0; memcpy( &v, storage, 1 ); return v; } + case 2: { uint16_t v = 0; memcpy( &v, storage, 2 ); return v; } + case 4: { uint32_t v = 0; memcpy( &v, storage, 4 ); return v; } + case 8: { uint64_t v = 0; memcpy( &v, storage, 8 ); return v; } + } + return 0; +} + +inline void TableJsonSetRaw( void * storage, uint32_t width, uint64_t value ) +{ + switch ( width ) + { + case 1: { uint8_t v = (uint8_t) value; memcpy( storage, &v, 1 ); break; } + case 2: { uint16_t v = (uint16_t) value; memcpy( storage, &v, 2 ); break; } + case 4: { uint32_t v = (uint32_t) value; memcpy( storage, &v, 4 ); break; } + case 8: { uint64_t v = value; memcpy( storage, &v, 8 ); break; } + } +} + +inline int64_t TableJsonGetSigned( const void * storage, uint32_t width ) +{ + uint64_t raw = TableJsonGetRaw( storage, width ); + if ( width < 8 ) + { + uint64_t sign = uint64_t( 1 ) << ( width * 8 - 1 ); + if ( ( raw & sign ) != 0 ) + { + raw |= ~( ( sign << 1 ) - 1 ); + } + } + return (int64_t) raw; +} + +// ---- the WIDE kinds (docs/SPEC-TABLES.md §3, §16.2) ---- +// +// The 128-bit integers and the fixed-point family convert EXACTLY, over two +// 64-bit lanes: a 128-bit integer is a decimal integer, a fixed value a +// decimal in WHOLE UNITS (1.0, -0.25, 3.0000152587890625) and nothing +// on either path passes through a double. Nothing here needs a 128-bit type +// either, which is what keeps this walk one text for every unit. +struct TableJsonWide +{ + uint64_t lo; + uint64_t hi; +}; + +inline bool TableJsonKindWide( uint8_t kind ) { return kind >= 18 && kind <= 29; } +inline bool TableJsonKindWideSigned( uint8_t kind ) { return kind == 18 || ( kind >= 20 && kind <= 24 ); } +inline bool TableJsonKindFixed( uint8_t kind ) { return kind >= 20 && kind <= 29; } + +inline bool TableJsonWideZero( TableJsonWide v ) { return v.lo == 0 && v.hi == 0; } +inline bool TableJsonWideNegative( TableJsonWide v ) { return ( v.hi >> 63 ) != 0; } + +inline int TableJsonWideCompare( TableJsonWide a, TableJsonWide b, bool is_signed ) +{ + if ( is_signed && TableJsonWideNegative( a ) != TableJsonWideNegative( b ) ) { return TableJsonWideNegative( a ) ? -1 : 1; } + if ( a.hi != b.hi ) { return a.hi < b.hi ? -1 : 1; } + if ( a.lo != b.lo ) { return a.lo < b.lo ? -1 : 1; } + return 0; +} + +inline TableJsonWide TableJsonWideShl( TableJsonWide v, int n ) +{ + TableJsonWide r = { 0, 0 }; + if ( n <= 0 ) { return v; } + if ( n >= 128 ) { return r; } + if ( n >= 64 ) { r.hi = v.lo << ( n - 64 ); return r; } + r.hi = ( v.hi << n ) | ( v.lo >> ( 64 - n ) ); + r.lo = v.lo << n; + return r; +} + +inline TableJsonWide TableJsonWideShr( TableJsonWide v, int n ) +{ + TableJsonWide r = { 0, 0 }; + if ( n <= 0 ) { return v; } + if ( n >= 128 ) { return r; } + if ( n >= 64 ) { r.lo = v.hi >> ( n - 64 ); return r; } + r.lo = ( v.lo >> n ) | ( v.hi << ( 64 - n ) ); + r.hi = v.hi >> n; + return r; +} + +inline TableJsonWide TableJsonWideNeg( TableJsonWide v ) +{ + TableJsonWide r; + r.lo = ~v.lo + 1; + r.hi = ~v.hi + ( r.lo == 0 ? 1 : 0 ); + return r; +} + +// v = v * m + a; the return is the carry out of 128 bits +inline uint32_t TableJsonWideMulAdd( TableJsonWide * v, uint32_t m, uint32_t a ) +{ + uint64_t limb[4] = { v->lo & 0xffffffffull, v->lo >> 32, v->hi & 0xffffffffull, v->hi >> 32 }; + uint64_t carry = a; + for ( int i = 0; i < 4; i++ ) + { + uint64_t p = limb[i] * m + carry; + limb[i] = p & 0xffffffffull; + carry = p >> 32; + } + v->lo = limb[0] | ( limb[1] << 32 ); + v->hi = limb[2] | ( limb[3] << 32 ); + return (uint32_t) carry; +} + +// v = v / d; the return is the remainder +inline uint32_t TableJsonWideDiv( TableJsonWide * v, uint32_t d ) +{ + uint64_t limb[4] = { v->lo & 0xffffffffull, v->lo >> 32, v->hi & 0xffffffffull, v->hi >> 32 }; + uint64_t rem = 0; + for ( int i = 3; i >= 0; i-- ) + { + uint64_t cur = ( rem << 32 ) | limb[i]; + limb[i] = cur / d; + rem = cur % d; + } + v->lo = limb[0] | ( limb[1] << 32 ); + v->hi = limb[2] | ( limb[3] << 32 ); + return (uint32_t) rem; +} + +// The storage of a wide kind, as lanes. A sixteen-byte storage is serialize's +// pair — native __int128 in the host's byte order, or the emulated struct with +// its low lane first — so the lanes are read in the host's order; a narrower +// storage is one lane, sign-extended for a signed kind. +inline TableJsonWide TableJsonWideLoad( const void * storage, uint32_t width, bool is_signed ) +{ + TableJsonWide v = { 0, 0 }; + if ( width == 16 ) + { + uint64_t half[2]; + memcpy( half, storage, 16 ); + uint16_t probe = 1; + bool little = *(const uint8_t *) &probe == 1; + v.lo = little ? half[0] : half[1]; + v.hi = little ? half[1] : half[0]; + return v; + } + v.lo = is_signed ? (uint64_t) TableJsonGetSigned( storage, width ) : TableJsonGetRaw( storage, width ); + v.hi = ( is_signed && ( v.lo >> 63 ) != 0 ) ? ~uint64_t( 0 ) : 0; + return v; +} + +inline void TableJsonWideStore( void * storage, uint32_t width, TableJsonWide v ) +{ + if ( width == 16 ) + { + uint16_t probe = 1; + bool little = *(const uint8_t *) &probe == 1; + uint64_t half[2]; + half[0] = little ? v.lo : v.hi; + half[1] = little ? v.hi : v.lo; + memcpy( storage, half, 16 ); + return; + } + TableJsonSetRaw( storage, width, v.lo ); +} + +// a counted field's companion: a string's length, a bytes' length, a counted +// array's count. Bounded by the declared extent on the way out, so a storage +// invariant a caller broke cannot walk off the end of the array. +inline int32_t TableJsonCount( const void * base, const TableFieldInfo * f ) +{ + if ( !f->counted ) + { + return f->array_bound; + } + int32_t count = 0; + memcpy( &count, (const uint8_t *) base + f->count_offset, sizeof( count ) ); + if ( count < 0 ) { count = 0; } + if ( count > f->array_bound ) { count = f->array_bound; } + return count; +} + +inline void TableJsonSetCount( void * base, const TableFieldInfo * f, int32_t count ) +{ + if ( f->counted ) + { + memcpy( (uint8_t *) base + f->count_offset, &count, sizeof( count ) ); + } +} + +// ---- what a field's kind expects to see in the text ---- +// +// One classifier, consulted by both directions, so a reader and a writer can +// never disagree about a kind's JSON form. 'o' object, 'a' array, 's' +// string, 'n' number, 'b' boolean. +// +// A vocabulary field is spelled by NAME: an enum is one name, a flags mask +// is the array of the names of its set bits. The two are told apart by the +// id column — an enum variant rides under a wire id, a flags BIT never does +// (docs/SPEC-TABLES.md §4), so a name function with no id function is flags. +// +// bytes(N) is the one kind whose element kind does not decide its form: it +// shares u8 with a plain array of u8, and rides as base64. The schema type +// name settles it, and "bytes" is a keyword no declaration can claim. +inline bool TableJsonIsBytes( const TableFieldInfo * f ) +{ + return f->is_array && f->kind == 6 && strcmp( f->type_name, "bytes" ) == 0; +} + +// An ENUM-KEYED array (docs/SPEC-TABLES.md §2.4): its JSON form is an OBJECT +// keyed by variant name, not a positional array, because that is what the +// storage is — one slot per variant, addressed by the variant. +inline bool TableJsonIsKeyed( const TableFieldInfo * f ) +{ + return f->key_name != NULL; +} + +// THE KEY A STORAGE SLOT HOLDS (§2.4, §8): the storage shifts left, so slot i +// holds the key i + 1 and nothing is stored for None. This is the ONE place +// the walker spells the shift. +inline uint64_t TableJsonKeyedSlotKey( int64_t slot ) +{ + return (uint64_t) ( slot + 1 ); +} + +// A slot whose key names a variant of the keying enum. Every slot in +// [0, array_bound) does, unless the enum carries max-headroom variants outside +// a table closure, where a reserved value names nothing and its key id is 0 — +// the reserved id no declared name can fold to (§5). +inline bool TableJsonKeyedSlotValid( const TableFieldInfo * f, int64_t slot ) +{ + return f->key_id( TableJsonKeyedSlotKey( slot ) ) != 0; +} + +inline bool TableJsonIsFlags( const TableFieldInfo * f ) +{ + return f->enum_name != NULL && f->variant_id == NULL; +} + +inline bool TableJsonIsEnum( const TableFieldInfo * f ) +{ + return f->variant_id != NULL && f->arms == NULL; +} + +inline char TableJsonShape( const TableFieldInfo * f ) +{ + if ( TableJsonIsMap( f ) ) return 'o'; // a MAP: an object keyed by the KEY (§2.8) + if ( f->kind == 12 ) return 's'; // string + if ( f->kind == 33 ) return 's'; // wstring: the same text, transcoded (§16.2) + if ( TableJsonIsBytes( f ) ) return 's'; // bytes: base64 + if ( TableJsonIsKeyed( f ) ) return 'o'; // an object keyed by variant NAME + if ( f->is_array ) return 'a'; + if ( f->arms != NULL ) return 'o'; // union: an object with ONE key + if ( f->kind == 13 ) return 'o'; // nested table or type + if ( f->kind == 17 ) return f->table != NULL ? 'o' : 's'; // a pointer: the pointee's object in place, or null (§16.7); a byte buffer's string (§2.5) + if ( TableJsonIsEnum( f ) ) return 's'; + if ( TableJsonIsFlags( f ) ) return 'a'; + if ( f->kind == 1 ) return 'b'; + return 'n'; +} + +// the ELEMENT shape of an array field — the same classifier one level down +inline char TableJsonElementShape( const TableFieldInfo * f ) +{ + if ( f->arms != NULL ) return 'o'; // an element of an array of unions: one key, the arm (§2.6) + if ( f->kind == 13 ) return 'o'; + if ( TableJsonIsEnum( f ) ) return 's'; + if ( TableJsonIsFlags( f ) ) return 'a'; + if ( f->kind == 1 ) return 'b'; + return 'n'; +} + +// A guarded group rides only when its guard reads true — the wire's own +// elision (§4), carried into the text so a text and a wire written from one +// instance say the same thing. The guard is spelled as its branch condition +// over bool fields of the SAME type ("at_rest", "!at_rest", +// "active && has_target"), so evaluating it is a walk of the same +// descriptor. Nothing is inferred in the other direction: reading places +// every key it can name, and the guard is a plain bool key (§16.2). +inline bool TableJsonGuardHolds( const void * base, const TableTypeInfo * info, const char * guard ) +{ + const char * p = guard; + for ( ;; ) + { + while ( *p == ' ' || *p == '&' ) { p++; } + if ( *p == 0 ) { return true; } + bool want = true; + if ( *p == '!' ) { want = false; p++; } + const char * start = p; + while ( *p != 0 && *p != ' ' && *p != '&' ) { p++; } + size_t length = (size_t) ( p - start ); + bool value = false; + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + const TableFieldInfo * f = &info->fields[i]; + if ( strlen( f->name ) == length && strncmp( f->name, start, length ) == 0 ) + { + value = TableJsonGetRaw( (const uint8_t *) base + f->offset, f->elem_size ) != 0; + break; + } + } + if ( value != want ) { return false; } + } +} + +// ---- writing ---- + +// The writer sink MEASURES when the buffer is NULL and WRITES when it is +// not, over one code path — so measure and write agree byte for byte, the +// wire's invariant (§9) carried across. +struct TableJsonOut +{ + char * buffer; + int64_t capacity; + int64_t offset; + bool overflow; + void * graph; // the pointered write's identity map (§16.7); NULL for a fixed table + + void raw( const char * data, int64_t count ) + { + if ( buffer != NULL ) + { + if ( offset + count > capacity ) { overflow = true; return; } + memcpy( buffer + offset, data, (size_t) count ); + } + offset += count; + } + void put( char c ) { raw( &c, 1 ); } + void text( const char * s ) { raw( s, (int64_t) strlen( s ) ); } + void line( int32_t depth ) + { + put( '\n' ); + for ( int32_t i = 0; i < depth; i++ ) { raw( " ", 2 ); } + } +}; + +inline const char * TableJsonBase64Alphabet() +{ + return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +} + +inline void TableJsonWriteBase64( TableJsonOut & out, const uint8_t * data, int32_t length ) +{ + const char * alphabet = TableJsonBase64Alphabet(); + out.put( '"' ); + int32_t i = 0; + for ( ; i + 3 <= length; i += 3 ) + { + uint32_t triple = ( uint32_t( data[i] ) << 16 ) | ( uint32_t( data[i+1] ) << 8 ) | uint32_t( data[i+2] ); + char quad[4] = { alphabet[ ( triple >> 18 ) & 0x3f ], alphabet[ ( triple >> 12 ) & 0x3f ], + alphabet[ ( triple >> 6 ) & 0x3f ], alphabet[ triple & 0x3f ] }; + out.raw( quad, 4 ); + } + if ( i < length ) + { + int32_t left = length - i; + uint32_t triple = uint32_t( data[i] ) << 16; + if ( left == 2 ) { triple |= uint32_t( data[i+1] ) << 8; } + char quad[4] = { alphabet[ ( triple >> 18 ) & 0x3f ], alphabet[ ( triple >> 12 ) & 0x3f ], '=', '=' }; + if ( left == 2 ) { quad[2] = alphabet[ ( triple >> 6 ) & 0x3f ]; } + out.raw( quad, 4 ); + } + out.put( '"' ); +} + +// One UTF-8 sequence at s, or -1 when the bytes there are not one. Rejects +// the lot: a stray continuation, an overlong form, a surrogate half, and +// anything past U+10FFFF. +inline int32_t TableJsonUtf8( const char * s, int32_t remaining, int32_t * width ) +{ + unsigned char lead = (unsigned char) s[0]; + int32_t want = 0; + int32_t code = 0; + if ( lead < 0x80 ) { *width = 1; return lead; } + else if ( lead >= 0xc2 && lead <= 0xdf ) { want = 2; code = lead & 0x1f; } + else if ( lead >= 0xe0 && lead <= 0xef ) { want = 3; code = lead & 0x0f; } + else if ( lead >= 0xf0 && lead <= 0xf4 ) { want = 4; code = lead & 0x07; } + else { return -1; } + if ( remaining < want ) { return -1; } + for ( int32_t i = 1; i < want; i++ ) + { + unsigned char next = (unsigned char) s[i]; + if ( ( next & 0xc0 ) != 0x80 ) { return -1; } + code = ( code << 6 ) | ( next & 0x3f ); + } + if ( want == 3 && code < 0x800 ) { return -1; } // overlong + if ( want == 4 && code < 0x10000 ) { return -1; } // overlong + if ( code >= 0xd800 && code <= 0xdfff ) { return -1; } // a surrogate half + if ( code > 0x10ffff ) { return -1; } + *width = want; + return code; +} + +// The inverse: one code point encoded as UTF-8 into unit, its length +// answered. Both text kinds' writers reach it, and so does the escape +// grammar's U+FFFD replacement. +inline int32_t TableJsonEncodeUtf8( uint32_t code, char * unit ) +{ + if ( code < 0x80 ) { unit[0] = (char) code; return 1; } + if ( code < 0x800 ) + { + unit[0] = (char) ( 0xc0 | ( code >> 6 ) ); + unit[1] = (char) ( 0x80 | ( code & 0x3f ) ); + return 2; + } + if ( code < 0x10000 ) + { + unit[0] = (char) ( 0xe0 | ( code >> 12 ) ); + unit[1] = (char) ( 0x80 | ( ( code >> 6 ) & 0x3f ) ); + unit[2] = (char) ( 0x80 | ( code & 0x3f ) ); + return 3; + } + unit[0] = (char) ( 0xf0 | ( code >> 18 ) ); + unit[1] = (char) ( 0x80 | ( ( code >> 12 ) & 0x3f ) ); + unit[2] = (char) ( 0x80 | ( ( code >> 6 ) & 0x3f ) ); + unit[3] = (char) ( 0x80 | ( code & 0x3f ) ); + return 4; +} + +// A JSON text MUST be valid UTF-8 (RFC 8259 §8.1). The read path is +// byte-transparent — the wire imposes no encoding (§3) and a string may hold +// anything — so the WRITER is where that obligation is met: a byte that is +// not part of a well-formed sequence is written as U+FFFD, one per bad byte, +// and never raw. A text this walk writes is therefore readable by any +// conforming parser, which a raw byte would not be. The cost is stated +// plainly: for a string holding invalid UTF-8, the round trip is NOT +// byte-identical, because the alternative is emitting a text that is not +// JSON. +inline void TableJsonWriteString( TableJsonOut & out, const char * s, int32_t length ) +{ + static const char hex[] = "0123456789abcdef"; + out.put( '"' ); + for ( int32_t i = 0; i < length; i++ ) + { + unsigned char c = (unsigned char) s[i]; + switch ( c ) + { + case '"': out.raw( "\\\"", 2 ); break; + case '\\': out.raw( "\\\\", 2 ); break; + case '\b': out.raw( "\\b", 2 ); break; + case '\f': out.raw( "\\f", 2 ); break; + case '\n': out.raw( "\\n", 2 ); break; + case '\r': out.raw( "\\r", 2 ); break; + case '\t': out.raw( "\\t", 2 ); break; + default: + if ( c < 0x20 ) + { + char escape[6] = { '\\', 'u', '0', '0', hex[ c >> 4 ], hex[ c & 0xf ] }; + out.raw( escape, 6 ); + } + else if ( c < 0x80 ) + { + out.put( (char) c ); + } + else + { + int32_t width = 0; + if ( TableJsonUtf8( s + i, length - i, &width ) < 0 ) + { + out.raw( "\xef\xbf\xbd", 3 ); // U+FFFD, one per bad byte + } + else + { + out.raw( s + i, width ); + i += width - 1; + } + } + break; + } + } + out.put( '"' ); +} + +// A WIDE field's text: the code units transcoded back to UTF-8 (§16.2). A +// SURROGATE PAIR is one code point; an UNPAIRED SURROGATE is not a code point +// at all, encodes to nothing, and writes one U+FFFD per ill-formed unit; a +// ZERO UNIT is U+0000, which JSON has an escape for, and writes \u0000 +// (§16.3). No wire can put either into storage (§3), so both answer for +// storage a PROGRAM built. +inline void TableJsonWriteWString( TableJsonOut & out, const char16_t * s, int32_t length ) +{ + static const char hex[] = "0123456789abcdef"; + out.put( '"' ); + for ( int32_t i = 0; i < length; i++ ) + { + uint32_t code = (uint32_t) (uint16_t) s[i]; + if ( code >= 0xd800 && code <= 0xdbff && i + 1 < length ) + { + const uint32_t low = (uint32_t) (uint16_t) s[i + 1]; + if ( low >= 0xdc00 && low <= 0xdfff ) + { + code = 0x10000 + ( ( code - 0xd800 ) << 10 ) + ( low - 0xdc00 ); + i++; + } + } + if ( code >= 0xd800 && code <= 0xdfff ) { code = 0xfffd; } // an unpaired surrogate + switch ( code ) + { + case '"': out.raw( "\\\"", 2 ); continue; + case '\\': out.raw( "\\\\", 2 ); continue; + case '\b': out.raw( "\\b", 2 ); continue; + case '\f': out.raw( "\\f", 2 ); continue; + case '\n': out.raw( "\\n", 2 ); continue; + case '\r': out.raw( "\\r", 2 ); continue; + case '\t': out.raw( "\\t", 2 ); continue; + default: break; + } + if ( code < 0x20 ) + { + char escape[6] = { '\\', 'u', '0', '0', hex[ code >> 4 ], hex[ code & 0xf ] }; + out.raw( escape, 6 ); + continue; + } + char encoded[4]; + const int32_t encoded_length = TableJsonEncodeUtf8( code, encoded ); + out.raw( encoded, encoded_length ); + } + out.put( '"' ); +} + +inline void TableJsonWriteUnsigned( TableJsonOut & out, uint64_t value ) +{ + char digits[24]; + int32_t n = 0; + do + { + digits[n++] = (char) ( '0' + (int) ( value % 10 ) ); + value /= 10; + } while ( value != 0 ); + char text[24]; + for ( int32_t i = 0; i < n; i++ ) { text[i] = digits[n - 1 - i]; } + out.raw( text, n ); +} + +inline void TableJsonWriteSigned( TableJsonOut & out, int64_t value ) +{ + if ( value < 0 ) + { + out.put( '-' ); + TableJsonWriteUnsigned( out, uint64_t( 0 ) - (uint64_t) value ); + return; + } + TableJsonWriteUnsigned( out, (uint64_t) value ); +} + +// A wide kind writes its raw storage as §16.2's text: a 128-bit integer as a +// decimal integer; a fixed value in WHOLE UNITS as the shortest exact decimal +// with at least one fractional digit (1.0, -0.25), the spelling the schema text +// gives a fixed default. The fraction terminates because a dyadic fraction has +// a finite decimal expansion — at most F digits. +inline void TableJsonWriteWide( TableJsonOut & out, const void * storage, const TableFieldInfo * f ) +{ + bool is_signed = TableJsonKindWideSigned( f->kind ); + TableJsonWide v = TableJsonWideLoad( storage, f->elem_size, is_signed ); + if ( is_signed && TableJsonWideNegative( v ) ) + { + out.put( '-' ); + v = TableJsonWideNeg( v ); + } + int frac = f->frac_bits; + TableJsonWide whole = TableJsonWideShr( v, frac ); + char digits[40]; + int32_t n = 0; + do + { + digits[n++] = (char) ( '0' + (int) TableJsonWideDiv( &whole, 10 ) ); + } while ( !TableJsonWideZero( whole ) ); + char text[40]; + for ( int32_t i = 0; i < n; i++ ) { text[i] = digits[n - 1 - i]; } + out.raw( text, n ); + if ( !TableJsonKindFixed( f->kind ) ) { return; } + out.put( '.' ); + // the fraction bits alone: v with everything at and above bit F cleared + TableJsonWide fraction = v; + if ( frac < 64 ) { fraction.hi = 0; fraction.lo &= ( uint64_t( 1 ) << frac ) - 1; } + else { fraction.hi &= ( uint64_t( 1 ) << ( frac - 64 ) ) - 1; } + if ( frac == 0 ) { fraction.lo = 0; } + if ( TableJsonWideZero( fraction ) ) + { + out.put( '0' ); + return; + } + while ( !TableJsonWideZero( fraction ) ) + { + // ×10: the digit is what lands at and above bit F, including the + // carry out of 128 bits when F leaves no room for it below + uint32_t carry = TableJsonWideMulAdd( &fraction, 10, 0 ); + uint64_t digit = TableJsonWideShr( fraction, frac ).lo; + if ( frac > 64 ) { digit |= uint64_t( carry ) << ( 128 - frac ); } + out.put( (char) ( '0' + (int) digit ) ); + if ( frac < 64 ) { fraction.hi = 0; fraction.lo &= ( uint64_t( 1 ) << frac ) - 1; } + else { fraction.hi &= ( uint64_t( 1 ) << ( frac - 64 ) ) - 1; } + } +} + +// A float writes at the SHORTEST precision that reads back as the same value +// at the field's own width, so a round trip is exact and a text stays +// readable. Non-finite values have no JSON spelling at all, and the writer +// REFUSES rather than losing one silently — the same rule measure and save +// already apply to an enum value no variant names (§5). +inline bool TableJsonWriteFloat( TableJsonOut & out, double value, bool single ) +{ + if ( !TableJsonFinite( value ) ) { return false; } + char text[64]; + int low = single ? 6 : 15; + int high = single ? 9 : 17; + int length = 0; + for ( int digits = low; ; digits++ ) + { + length = snprintf( text, sizeof( text ), "%.*g", digits, value ); + if ( length <= 0 || length >= (int) sizeof( text ) ) { return false; } + if ( digits >= high ) { break; } + // the round-trip check runs BEFORE the decimal point is normalised: + // the token still carries whatever point snprintf just produced + if ( single ) + { + if ( (double) strtof( text, NULL ) == value ) { break; } + } + else + { + if ( strtod( text, NULL ) == value ) { break; } + } + } + char point = TableJsonDecimalPoint(); + if ( point != '.' ) + { + for ( int i = 0; i < length; i++ ) + { + if ( text[i] == point ) { text[i] = '.'; } + } + } + out.raw( text, length ); + return true; +} + +inline bool TableJsonWriteValue( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth ); +// a UNION ARM that names no declaration writes through the field walk one key +// down (docs/SPEC-TABLES.md §2.6, §16.2), which is defined below +inline bool TableJsonWriteField( TableJsonOut & out, const void * base, const TableFieldInfo * f, int32_t depth ); + +// one scalar, at one storage address: a nested object, a union, a +// vocabulary, or a number +inline bool TableJsonWriteScalar( TableJsonOut & out, const void * storage, const TableFieldInfo * f, int32_t depth ) +{ + if ( f->arms != NULL ) + { + // a union is an object with ONE key, the arm's name; None is {} + const TableUnionInfo * arms = f->arms(); + uint64_t tag = TableJsonGetRaw( (const uint8_t *) storage + arms->tag_offset, arms->tag_size ); + if ( tag == 0 ) + { + out.raw( "{}", 2 ); + return true; + } + if ( (int64_t) tag > f->enum_max ) + { + return false; // a tag no arm names, exactly as measure refuses it + } + const char * arm = f->enum_name( tag ); + // and refuse on the NAME, not merely on the bound: §16.2 says a value + // no variant NAMES is refused, so the check is the name. Writing + // whatever came back would emit "???", a spelling the reader counts + // as unknown — a silent round-trip loss in place of a refusal. + if ( !TableJsonNamed( arm ) ) { return false; } + out.put( '{' ); + out.line( depth + 1 ); + TableJsonWriteString( out, arm, (int32_t) strlen( arm ) ); + out.raw( ": ", 2 ); + // THE ARM'S VALUE TAKES THE ARM'S OWN ROW (§16.2): an arm that names + // no declaration carries the FIELD descriptor a field of its type + // would carry, offsets taken inside the union storage (§2.6), so the + // value walks through the field writer one key down. + if ( arms->arms[tag].field != NULL ) + { + if ( !TableJsonWriteField( out, storage, arms->arms[tag].field, depth + 1 ) ) + { + return false; + } + } + else if ( arms->arms[tag].table == NULL ) + { + out.raw( "null", 4 ); // a payload-free arm: the name selects it (§2.6) + } + else if ( !TableJsonWriteValue( out, (const uint8_t *) storage + arms->arms[tag].offset, arms->arms[tag].table, depth + 1 ) ) + { + return false; + } + out.line( depth ); + out.put( '}' ); + return true; + } + if ( f->kind == 13 ) + { + return TableJsonWriteValue( out, storage, f->table, depth ); + } + if ( TableJsonIsEnum( f ) ) + { + uint64_t value = TableJsonGetRaw( storage, f->elem_size ); + // a value no variant names has no text spelling, exactly as it has no + // wire identity: the writer REFUSES rather than writing None over it, + // the rule measure and save already apply (docs/SPEC-TABLES.md §5) + if ( (int64_t) value > f->enum_max ) { return false; } + if ( value != 0 && f->variant_id( value ) == 0 ) { return false; } + const char * name = f->enum_name( value ); + if ( !TableJsonNamed( name ) ) { return false; } + TableJsonWriteString( out, name, (int32_t) strlen( name ) ); + return true; + } + if ( TableJsonIsFlags( f ) ) + { + uint64_t bits = TableJsonGetRaw( storage, f->elem_size ); + if ( bits == 0 ) + { + out.raw( "[]", 2 ); + return true; + } + out.put( '[' ); + bool first = true; + for ( int64_t bit = 0; bit < 64; bit++ ) + { + if ( ( bits & ( uint64_t( 1 ) << bit ) ) == 0 ) { continue; } + if ( bit > f->enum_max ) + { + return false; // a bit no variant names has no text spelling + } + const char * name = f->enum_name( (uint64_t) bit ); + if ( !TableJsonNamed( name ) ) { return false; } + if ( !first ) { out.put( ',' ); } + first = false; + out.line( depth + 1 ); + TableJsonWriteString( out, name, (int32_t) strlen( name ) ); + } + out.line( depth ); + out.put( ']' ); + return true; + } + switch ( f->kind ) + { + case 1: + out.text( TableJsonGetRaw( storage, f->elem_size ) != 0 ? "true" : "false" ); + return true; + case 10: + { + float v = 0.0f; + memcpy( &v, storage, sizeof( v ) ); + return TableJsonWriteFloat( out, (double) v, true ); + } + case 11: + { + double v = 0.0; + memcpy( &v, storage, sizeof( v ) ); + return TableJsonWriteFloat( out, v, false ); + } + case 2: case 3: case 4: case 5: + TableJsonWriteSigned( out, TableJsonGetSigned( storage, f->elem_size ) ); + return true; + default: + if ( TableJsonKindWide( f->kind ) ) + { + TableJsonWriteWide( out, storage, f ); + return true; + } + TableJsonWriteUnsigned( out, TableJsonGetRaw( storage, f->elem_size ) ); + return true; + } +} + +inline bool TableJsonWriteField( TableJsonOut & out, const void * base, const TableFieldInfo * f, int32_t depth ) +{ + const uint8_t * storage = (const uint8_t *) base + f->offset; + if ( TableJsonIsMap( f ) ) + { + return TableJsonWriteMap( out, (const void *) storage, f, depth ); + } + if ( TableJsonIsList( f ) ) + { + return TableJsonWriteList( out, (const void *) storage, f, depth ); + } + if ( f->kind == 17 && !f->is_array ) + { + return TableJsonWritePointer( out, storage, f, depth ); + } + if ( f->kind == 17 ) + { + // an ARRAY OF POINTERS (§2.1): the pointer row per element — the + // pointee's object in place, null, or `&node` for a shared one (§16.7) + int32_t count = TableJsonCount( base, f ); + if ( count == 0 ) { out.raw( "[]", 2 ); return true; } + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + if ( !TableJsonWritePointer( out, storage + (int64_t) i * f->elem_size, f, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( ']' ); + return true; + } + if ( f->kind == 12 ) + { + TableJsonWriteString( out, (const char *) storage, TableJsonCount( base, f ) ); + return true; + } + if ( f->kind == 33 ) + { + TableJsonWriteWString( out, (const char16_t *) (const void *) storage, TableJsonCount( base, f ) ); + return true; + } + if ( TableJsonIsBytes( f ) ) + { + TableJsonWriteBase64( out, storage, TableJsonCount( base, f ) ); + return true; + } + if ( TableJsonIsKeyed( f ) ) + { + // one entry per SLOT, keyed by the variant that owns it, so inserting + // a variant next season moves nothing in the text either. Slot i holds + // the key i + 1: nothing is stored for None, so nothing is written for it. + out.put( '{' ); + bool first = true; + for ( int64_t slot = 0; slot < f->array_bound; slot++ ) + { + if ( !TableJsonKeyedSlotValid( f, slot ) ) { continue; } + if ( !first ) { out.put( ',' ); } + first = false; + out.line( depth + 1 ); + const char * key = f->key_name( TableJsonKeyedSlotKey( slot ) ); + TableJsonWriteString( out, key, (int32_t) strlen( key ) ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteScalar( out, storage + slot * f->elem_size, f, depth + 1 ) ) + { + return false; + } + } + if ( first ) { out.raw( "}", 1 ); return true; } + out.line( depth ); + out.put( '}' ); + return true; + } + if ( f->is_array ) + { + int32_t count = TableJsonCount( base, f ); + if ( count == 0 ) + { + out.raw( "[]", 2 ); + return true; + } + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + if ( !TableJsonWriteScalar( out, storage + (int64_t) i * f->elem_size, f, depth + 1 ) ) + { + return false; + } + } + out.line( depth ); + out.put( ']' ); + return true; + } + return TableJsonWriteScalar( out, storage, f, depth ); +} + +// One instance's fields, in DECLARATION ORDER, defaults included — a text is +// for people and tools, and a text that elides is a text a reader has to know +// the schema to complete. `any` says whether the object is already open on +// entry — a shared node's `&node` opens it before the fields (§16.7) — and +// whether it is open on return. +inline bool TableJsonWriteFields( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth, bool & any ) +{ + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + const TableFieldInfo * f = &info->fields[i]; + if ( f->guard[0] != 0 && !TableJsonGuardHolds( base, info, f->guard ) ) { continue; } + // an ABSENT optional writes no key: presence of the key IS the + // presence (§16.2), so an absent field is an absent key and nothing + // else would read back as absent + if ( f->optional && + TableJsonGetRaw( (const uint8_t *) base + f->present_offset, 1 ) == 0 ) + { + continue; + } + if ( !any ) { out.put( '{' ); } + else { out.put( ',' ); } + any = true; + out.line( depth + 1 ); + TableJsonWriteString( out, f->json, (int32_t) strlen( f->json ) ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteField( out, base, f, depth + 1 ) ) { return false; } + } + return true; +} + +// One instance as one object. The writer carries the reader's depth cap +// (§16.2): a pointer chain nests as deep as it is long (§16.7), and a text the +// writer produced past the cap would be a text the reader refuses. +inline bool TableJsonWriteValue( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { return false; } + bool any = false; + if ( !TableJsonWriteFields( out, base, info, depth, any ) ) { return false; } + if ( !any ) + { + out.raw( "{}", 2 ); + return true; + } + out.line( depth ); + out.put( '}' ); + return true; +} + +// ---- reading ---- + +struct TableJsonIn +{ + const char * text; + int64_t size; + int64_t pos; + TableReport * report; + bool bad; // the text is not JSON: the walk stops and keeps what it placed + void * graph; // the pointered read's builder and label map (§16.7); NULL for a fixed table +}; + +inline void TableJsonSpace( TableJsonIn & in ) +{ + while ( in.pos < in.size ) + { + char c = in.text[in.pos]; + if ( c == ' ' || c == '\t' || c == '\n' || c == '\r' ) { in.pos++; continue; } + // COMMENTS ARE ACCEPTED ON READ AND NEVER WRITTEN (docs/SPEC-TABLES.md + // §16.2): a line comment runs to the end of the line or of the input, + // a block comment to its closing delimiter, which does not nest, and + // an UNCLOSED block comment is malformed on the terms an unclosed + // string is. Both are legal wherever whitespace is; a lone slash is not JSON. + if ( c == '/' && in.pos + 1 < in.size && in.text[in.pos + 1] == '/' ) + { + in.pos += 2; + while ( in.pos < in.size && in.text[in.pos] != '\n' ) { in.pos++; } + continue; + } + if ( c == '/' && in.pos + 1 < in.size && in.text[in.pos + 1] == '*' ) + { + int64_t at = in.pos + 2; + while ( at + 1 < in.size && !( in.text[at] == '*' && in.text[at + 1] == '/' ) ) { at++; } + if ( at + 1 >= in.size ) { in.bad = true; in.pos = in.size; return; } + in.pos = at + 2; + continue; + } + if ( c == '/' ) { in.bad = true; } + return; + } +} + +inline char TableJsonPeek( TableJsonIn & in ) +{ + TableJsonSpace( in ); + return in.pos < in.size ? in.text[in.pos] : 0; +} + +// the shape of the value sitting at the cursor, without consuming it +inline char TableJsonValueShape( TableJsonIn & in ) +{ + char c = TableJsonPeek( in ); + switch ( c ) + { + case '{': return 'o'; + case '[': return 'a'; + case '"': return 's'; + case 't': case 'f': return 'b'; + case 'n': return 'z'; + case 0: return 0; + default: return 'n'; + } +} + +inline bool TableJsonLiteral( TableJsonIn & in, const char * word ) +{ + int64_t length = (int64_t) strlen( word ); + if ( in.pos + length > in.size || memcmp( in.text + in.pos, word, (size_t) length ) != 0 ) + { + in.bad = true; + return false; + } + in.pos += length; + return true; +} + +// one \uXXXX escape body; -1 when the four hex digits are not there +inline int TableJsonHex4( TableJsonIn & in ) +{ + if ( in.pos + 4 > in.size ) { return -1; } + int value = 0; + for ( int i = 0; i < 4; i++ ) + { + char c = in.text[in.pos + i]; + int digit; + if ( c >= '0' && c <= '9' ) { digit = c - '0'; } + else if ( c >= 'a' && c <= 'f' ) { digit = c - 'a' + 10; } + else if ( c >= 'A' && c <= 'F' ) { digit = c - 'A' + 10; } + else { return -1; } + value = ( value << 4 ) | digit; + } + in.pos += 4; + return value; +} + + +// One STRING BODY CHARACTER at the cursor, encoded into unit as UTF-8 and +// its length answered: an escape's code point, or a UTF-8 sequence read +// whole. It is ONE grammar serving both text kinds — the narrow scan places +// these bytes and the wide scan converts them back to code units — so the +// escape table, the lone-surrogate rule and the U+FFFD replacement are stated +// once. false means the text is not JSON and in.bad says so; a returned +// length of 0 means the closing quote was consumed and the string is done. +inline bool TableJsonScanUnit( TableJsonIn & in, char * unit, int32_t * unit_length_out ) +{ + int32_t unit_length = 0; + *unit_length_out = 0; + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos]; + if ( c == '"' ) { in.pos++; return true; } + if ( c == '\\' ) + { + in.pos++; + if ( in.pos >= in.size ) { in.bad = true; return false; } + char escape = in.text[in.pos++]; + switch ( escape ) + { + case '"': unit[0] = '"'; unit_length = 1; break; + case '\\': unit[0] = '\\'; unit_length = 1; break; + case '/': unit[0] = '/'; unit_length = 1; break; + case 'b': unit[0] = '\b'; unit_length = 1; break; + case 'f': unit[0] = '\f'; unit_length = 1; break; + case 'n': unit[0] = '\n'; unit_length = 1; break; + case 'r': unit[0] = '\r'; unit_length = 1; break; + case 't': unit[0] = '\t'; unit_length = 1; break; + case 'u': + { + int high = TableJsonHex4( in ); + if ( high < 0 ) { in.bad = true; return false; } + uint32_t code = (uint32_t) high; + if ( high >= 0xd800 && high <= 0xdbff && in.pos + 2 <= in.size && + in.text[in.pos] == '\\' && in.text[in.pos + 1] == 'u' ) + { + int64_t mark = in.pos; + in.pos += 2; + int low = TableJsonHex4( in ); + if ( low >= 0xdc00 && low <= 0xdfff ) + { + code = 0x10000 + ( ( (uint32_t) high - 0xd800 ) << 10 ) + ( (uint32_t) low - 0xdc00 ); + } + else + { + in.pos = mark; // a lone lead surrogate rides as itself + } + } + // a surrogate half that never found its partner has no + // UTF-8 encoding: encoding it anyway would manufacture + // CESU-8 — invalid UTF-8 — out of input that was valid + // JSON, so it reads as the replacement character + if ( code >= 0xd800 && code <= 0xdfff ) { code = 0xfffd; } + unit_length = TableJsonEncodeUtf8( code, unit ); + break; + } + default: in.bad = true; return false; + } + } + else if ( (unsigned char) c < 0x20 ) + { + in.bad = true; // a raw control character is not a JSON string body + return false; + } + else + { + // a UTF-8 sequence read WHOLE, so the clamp below can only land + // between code points. Only bytes that ACTUALLY look like + // continuations are taken: the wire imposes no encoding (§3), so + // a string may legitimately hold a stray lead byte, and one at + // the end of a text must not swallow the closing quote. + unsigned char lead = (unsigned char) c; + int32_t want = 1; + if ( ( lead & 0xe0 ) == 0xc0 ) { want = 2; } + else if ( ( lead & 0xf0 ) == 0xe0 ) { want = 3; } + else if ( ( lead & 0xf8 ) == 0xf0 ) { want = 4; } + unit[0] = c; + in.pos++; + unit_length = 1; + while ( unit_length < want && in.pos < in.size && + ( (unsigned char) in.text[in.pos] & 0xc0 ) == 0x80 ) + { + unit[unit_length++] = in.text[in.pos++]; + } + // A SEQUENCE THAT IS NOT A CODE POINT READS AS U+FFFD, which is + // §16.3's rule at the point the defect ENTERS rather than at the + // point it leaves: a lone surrogate escape already reads that way + // above, RFC 8259 requires a JSON text to be valid UTF-8, and a + // kind 12 payload is well-formed UTF-8 (§3), so storage the text + // form built has to be storage the wire can carry (§5). + if ( !TableUtf8Valid( (const uint8_t *) unit, unit_length ) ) + { + unit_length = TableJsonEncodeUtf8( 0xfffd, unit ); + } + } + } + *unit_length_out = unit_length; + return true; +} + +// Scan one JSON string into a caller buffer. Bytes are appended ONE CODE +// POINT AT A TIME — an escape's encoding, or a UTF-8 sequence read whole — +// so a string longer than the field is clamped AT A CODE POINT BOUNDARY and +// never cut through a multi-byte character. Clamping is counted, never +// fatal, exactly as it is on the wire (§4). A NULL destination scans past a +// string without keeping it. +// +// A CALLER THAT TAKES clamped_out OWNS THE COUNTER. The value paths leave it +// NULL, and the clamp is a value's clamp, counted here. A MAP KEY takes it, +// because a key never clamps: a key this buffer could not hold whole is not a +// shorter key, and its entry drops instead (§2.8). +inline bool TableJsonScanString( TableJsonIn & in, char * out, int32_t capacity, int32_t * length, + bool * clamped_out = NULL ) +{ + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + int32_t placed = 0; + bool clamped = false; + for ( ;; ) + { + char unit[4]; + int32_t unit_length = 0; + if ( !TableJsonScanUnit( in, unit, &unit_length ) ) { return false; } + if ( unit_length == 0 ) { break; } + if ( out == NULL ) + { + placed += unit_length; // measured and not kept: a byte buffer's read sizes its node this way (§2.5) + } + else if ( !clamped && placed + unit_length <= capacity ) + { + memcpy( out + placed, unit, (size_t) unit_length ); + placed += unit_length; + } + else + { + // A CLAMP IS A PREFIX. Once one code point does not fit, the scan + // stops placing: a later SHORTER code point sliding into the room + // the long one left would store a string the input never spelled, + // and one clamped count cannot tell the two apart. + clamped = true; + } + } + if ( clamped_out != NULL ) { *clamped_out = clamped; } + else if ( clamped ) { in.report->clamped++; } + if ( length != NULL ) { *length = placed; } + return true; +} + +// One UTF-8 sequence back to its CODE POINT, over bytes TableJsonScanUnit +// produced and therefore already well formed. It is the inverse of +// TableJsonEncodeUtf8 and nothing more. +inline uint32_t TableJsonDecodeUtf8( const char * unit, int32_t unit_length ) +{ + const unsigned char lead = (unsigned char) unit[0]; + if ( unit_length == 1 ) { return lead; } + uint32_t code = lead & ( unit_length == 2 ? 0x1fu : ( unit_length == 3 ? 0x0fu : 0x07u ) ); + for ( int32_t i = 1; i < unit_length; i++ ) + { + code = ( code << 6 ) | (uint32_t) ( (unsigned char) unit[i] & 0x3f ); + } + return code; +} + +// Scan one JSON string into a caller buffer of UTF-16 CODE UNITS: the wstring +// row of §16.2, the text TRANSCODED at the boundary. It shares +// TableJsonScanUnit with the narrow scan, so the escape grammar and the +// lone-surrogate rule are one grammar, and appends ONE CODE POINT AT A TIME — +// one unit below U+10000 and a surrogate PAIR above it. A string longer than +// the field is therefore clamped at N code units with a pair never split, and +// a high surrogate left without its low half is dropped with it, which is the +// same sentence the wire's clamp takes (§3). Clamping is counted, never fatal. +inline bool TableJsonScanWString( TableJsonIn & in, char16_t * out, int32_t capacity, int32_t * length ) +{ + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + int32_t placed = 0; + bool clamped = false; + for ( ;; ) + { + char unit[4]; + int32_t unit_length = 0; + if ( !TableJsonScanUnit( in, unit, &unit_length ) ) { return false; } + if ( unit_length == 0 ) { break; } + const uint32_t code = TableJsonDecodeUtf8( unit, unit_length ); + char16_t units[2]; + int32_t units_length = 1; + if ( code < 0x10000 ) + { + units[0] = (char16_t) code; + } + else + { + const uint32_t rest = code - 0x10000; + units[0] = (char16_t) ( 0xd800 + ( rest >> 10 ) ); + units[1] = (char16_t) ( 0xdc00 + ( rest & 0x3ff ) ); + units_length = 2; + } + if ( out == NULL ) + { + placed += units_length; + } + else if ( !clamped && placed + units_length <= capacity ) + { + for ( int32_t i = 0; i < units_length; i++ ) { out[placed + i] = units[i]; } + placed += units_length; + } + else + { + // A CLAMP IS A PREFIX, the narrow scan's own rule: once one code + // point does not fit, the scan stops placing. A pair is placed + // whole or not at all, so no clamp can leave an unpaired + // surrogate in storage. + clamped = true; + } + } + if ( clamped ) { in.report->clamped++; } + if ( length != NULL ) { *length = placed; } + return true; +} + + +// the numeric token at the cursor, copied out whole; false = not a number +// Scan one number, to JSON's OWN grammar (RFC 8259 §6) and not to a run of +// number-ish characters: +// +// number = [ "-" ] int [ frac ] [ exp ] +// int = "0" / ( digit1-9 *digit ) +// frac = "." 1*digit +// exp = ( "e" / "E" ) [ "-" / "+" ] 1*digit +// +// Scanning the production is what makes a typo in an authoring file a +// DIAGNOSTIC rather than a value: "1-2" scans as 1 and leaves "-2" where the +// object expects a comma, so the text is malformed — which is what §16.2 +// already promises. A permissive scan would hand "1-2" to a digit loop and +// report a clamp, and a config pipeline would never hear about it. Leading +// "+", leading zeros, ".5" and "3." are not JSON either. +inline bool TableJsonWalkNumber( TableJsonIn & in, bool * integral ) +{ + TableJsonSpace( in ); + bool whole = true; + if ( in.pos < in.size && in.text[in.pos] == '-' ) { in.pos++; } + // int: a lone zero, or a non-zero digit and any digits after it + if ( in.pos >= in.size ) { return false; } + if ( in.text[in.pos] == '0' ) + { + in.pos++; + } + else if ( in.text[in.pos] >= '1' && in.text[in.pos] <= '9' ) + { + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + } + else + { + return false; + } + // frac + if ( in.pos < in.size && in.text[in.pos] == '.' ) + { + in.pos++; + if ( in.pos >= in.size || in.text[in.pos] < '0' || in.text[in.pos] > '9' ) { return false; } + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + whole = false; + } + // exp + if ( in.pos < in.size && ( in.text[in.pos] == 'e' || in.text[in.pos] == 'E' ) ) + { + in.pos++; + if ( in.pos < in.size && ( in.text[in.pos] == '-' || in.text[in.pos] == '+' ) ) { in.pos++; } + if ( in.pos >= in.size || in.text[in.pos] < '0' || in.text[in.pos] > '9' ) { return false; } + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + whole = false; + } + *integral = whole; + return true; +} + +// the same production, with the token kept for conversion +inline bool TableJsonScanNumber( TableJsonIn & in, char * token, int32_t capacity, int32_t * length, bool * integral ) +{ + TableJsonSpace( in ); + int64_t start = in.pos; + if ( !TableJsonWalkNumber( in, integral ) ) { return false; } + int64_t count = in.pos - start; + if ( count <= 0 || count >= capacity ) { return false; } + memcpy( token, in.text + start, (size_t) count ); + token[count] = 0; + *length = (int32_t) count; + return true; +} + +// the token's exact double, through the runtime's own converter — which +// speaks the LOCALE's decimal point, so the token crosses back over that +// character on its way in +inline double TableJsonTokenDouble( const char * token, int32_t length, bool single ) +{ + char work[kTableJsonMaxNumber]; + memcpy( work, token, (size_t) length ); + work[length] = 0; + char point = TableJsonDecimalPoint(); + if ( point != '.' ) + { + for ( int32_t i = 0; i < length; i++ ) + { + if ( work[i] == '.' ) { work[i] = point; } + } + } + if ( single ) { return (double) strtof( work, NULL ); } + return strtod( work, NULL ); +} + +// ---- ONE CHECKED NUMERIC INTERPRETATION (docs/SPEC-TABLES.md §16.2) ---- +// +// JSON HAS ONE NUMBER TYPE, so every integer target reads a token the same way +// and the VALUE decides rather than the spelling: 2, 2.0 and 1e3 are the +// integers 2, 2 and 1000. What comes out of a token is a SIGN, a MAGNITUDE and +// a STATUS, and nothing on the way is cast through a type that cannot hold what +// it is handed. A uint64 magnitude past INT64_MAX is a magnitude and never a +// negative, and a double is consulted only for a spelling the digit path cannot +// read exactly. +// +// TWO POLICIES SIT ON TOP OF THE ONE VALUE and neither reinterprets the token: +// an ordinary FIELD clamps to its domain and counts, and a MAP KEY rejects the +// whole entry, because a key is an identity and a clamped one is two entries +// merged. That difference is the only difference between them. +// +// THE KEY READS ITS TOKEN EXACTLY AND A FIELD READS IT THROUGH THE DOUBLE, and +// that is the one place the two interpretations part. A key is an IDENTITY, so +// two spellings a 53-bit mantissa cannot tell apart are two keys and the key +// path carries TableJsonInterpretExact below. A field's value is a quantity +// under a clamp, and its interpretation is the one the C, Go and Rust ports +// read the same texts with, so it lives here and reads as they read. +struct TableJsonInteger +{ + uint64_t magnitude; // |value|, exact for every integral token 64 bits hold + bool negative; + bool fractional; // a genuinely fractional VALUE: the wrong shape for an integer + bool saturated; // a magnitude past what 64 bits hold, held at that edge + bool finite; // false: no integer target holds it at all +}; + +// THE FIELD'S INTERPRETATION: the token, parsed digit by digit so no width and +// no locale can move it, and through the runtime's converter only where the +// spelling carries a fraction or an exponent +inline TableJsonInteger TableJsonInterpret( const char * token, int32_t length, bool integral ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = false; + out.fractional = false; + out.saturated = false; + out.finite = true; + if ( integral ) + { + int32_t i = 0; + if ( i < length && token[i] == '-' ) // WalkNumber refuses a leading plus + { + out.negative = true; + i++; + } + for ( ; i < length; i++ ) + { + const uint64_t digit = (uint64_t) ( token[i] - '0' ); + if ( out.magnitude > ( UINT64_MAX - digit ) / 10 ) + { + out.magnitude = UINT64_MAX; + out.saturated = true; + break; + } + out.magnitude = out.magnitude * 10 + digit; + } + if ( out.magnitude == 0 ) { out.negative = false; } // -0 IS zero + return out; + } + const double d = TableJsonTokenDouble( token, length, false ); + if ( !TableJsonFinite( d ) ) { out.finite = false; return out; } + out.negative = d < 0; + const double whole = out.negative ? -d : d; + // THE DOMAIN IS ESTABLISHED BEFORE THE CAST: a magnitude past what sixty-four + // bits hold is answered here, so no value ever reaches a conversion that is + // undefined for it + if ( whole >= 18446744073709551616.0 ) + { + out.magnitude = UINT64_MAX; + out.saturated = true; + return out; + } + const uint64_t truncated = (uint64_t) whole; + if ( (double) truncated != whole ) { out.fractional = true; return out; } + out.magnitude = truncated; + if ( out.magnitude == 0 ) { out.negative = false; } + return out; +} + +// THE DECIMAL BAND a token is answered in without arithmetic: 10^20 is above +// UINT64_MAX whatever the digits are, so a point past it saturates and a token +// spelling 1e999999999 costs nothing to refuse. +const int64_t kTableJsonDecimalBand = 20; + +// THE MAP KEY'S INTERPRETATION, and no other path's: the token's own digits, +// read where they stand, so no 53-bit mantissa decides the identity of a 64-bit +// key. The int and frac runs are one digit string with the point after "point" +// of them, and the exponent moves the point rather than the digits, which is +// the normalization the wide kinds already use over one 64-bit lane. A zero +// fraction is the integer the token spells at every magnitude the kind holds, +// so 9007199254740993.0 is that key rather than the one a double rounds it to. +// No exact reader has an infinity, so finite is true here always. +inline TableJsonInteger TableJsonInterpretExact( const char * token, int32_t length ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = false; + out.fractional = false; + out.saturated = false; + out.finite = true; + int32_t i = 0; + if ( i < length && token[i] == '-' ) { out.negative = true; i++; } // WalkNumber refuses a leading plus + const char * int_digits = token + i; + int32_t int_len = 0; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { int_len++; i++; } + const char * frac_digits = token + i; + int32_t frac_len = 0; + if ( i < length && token[i] == '.' ) + { + i++; + frac_digits = token + i; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { frac_len++; i++; } + } + int64_t exp = 0; + if ( i < length && ( token[i] == 'e' || token[i] == 'E' ) ) + { + i++; + bool exp_negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { exp_negative = token[i] == '-'; i++; } + while ( i < length && token[i] >= '0' && token[i] <= '9' ) + { + if ( exp < 100000 ) { exp = exp * 10 + ( token[i] - '0' ); } + i++; + } + if ( exp_negative ) { exp = -exp; } + } + // leading and trailing zeros stripped, so the last digit kept is significant + int32_t start = 0, end = int_len + frac_len; + int64_t point = (int64_t) int_len + exp; + while ( start < end && ( start < int_len ? int_digits[start] : frac_digits[start - int_len] ) == '0' ) { start++; point--; } + while ( end > start && ( end - 1 < int_len ? int_digits[end - 1] : frac_digits[end - 1 - int_len] ) == '0' ) { end--; } + const int64_t digits = end - start; + if ( digits == 0 ) { out.negative = false; return out; } // the value is zero, and -0 IS zero + if ( point < digits ) { out.fractional = true; return out; } // a significant digit below the point + if ( point > kTableJsonDecimalBand ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + for ( int32_t k = start; k < end; k++ ) + { + const uint64_t digit = (uint64_t) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ); + if ( out.magnitude > ( UINT64_MAX - digit ) / 10 ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + out.magnitude = out.magnitude * 10 + digit; + } + for ( int64_t k = digits; k < point; k++ ) // the point's own zeros, which no digit spells + { + if ( out.magnitude > UINT64_MAX / 10 ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + out.magnitude *= 10; + } + return out; +} + +// a declared range bound as the same value. A bound is inside the field's own +// domain by construction, so nothing here saturates. +inline TableJsonInteger TableJsonIntegerOf( double bound ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = bound < 0; + out.fractional = false; + out.saturated = false; + out.finite = true; + const double whole = out.negative ? -bound : bound; + out.magnitude = whole >= 18446744073709551616.0 ? UINT64_MAX : (uint64_t) whole; + if ( out.magnitude == 0 ) { out.negative = false; } + return out; +} + +// THE TARGET DOMAIN, established before the value reaches storage: the bytes of +// storage, signed or not. Answers what the target holds and whether the domain +// MOVED it. An unsigned magnitude above INT64_MAX rides out as its bit pattern, +// which is the storage's own image of it and not a negative number. +inline int64_t TableJsonIntegerInDomain( const TableJsonInteger & number, bool is_signed, int32_t bytes, bool & moved ) +{ + moved = false; + uint64_t magnitude = number.magnitude; + if ( is_signed ) + { + const uint64_t high = bytes >= 8 ? (uint64_t) INT64_MAX : ( ( uint64_t( 1 ) << ( bytes * 8 - 1 ) ) - 1 ); + if ( number.negative ) + { + const uint64_t low = high + 1; // the floor's magnitude + if ( magnitude > low ) { magnitude = low; moved = true; } + return (int64_t) ( ~magnitude + 1 ); // two's complement, INT64_MIN included + } + if ( magnitude > high ) { magnitude = high; moved = true; } + return (int64_t) magnitude; + } + // A NEGATIVE TOKEN IN AN UNSIGNED FIELD CLAMPS TO ZERO, and -0 is zero, + // which is why the sign is dropped at a zero magnitude above + if ( number.negative ) { moved = true; return 0; } + const uint64_t high = bytes >= 8 ? UINT64_MAX : ( ( uint64_t( 1 ) << ( bytes * 8 ) ) - 1 ); + if ( magnitude > high ) { magnitude = high; moved = true; } + return (int64_t) magnitude; +} + +// A number token into a wide kind's raw storage (docs/SPEC-TABLES.md §16.2). A +// 128-bit integer takes any token whose VALUE is integral; a fixed field any +// token whose value is EXACTLY representable in its Q I.F — a finer fraction +// is the wrong shape for the field, counted as a kind mismatch and never +// rounded, the rule SPEC.md §4.6 gives a fixed default. A magnitude past 128 +// bits saturates and counts as a clamp, as an int64 field saturates at +// INT64_MAX; the declared range clamps after it, on the RAW scale, as it does +// for every bounded scalar. +// +// The token is normalized to its digits with the decimal point after "point" +// of them. An integer part past 40 digits is above 2^128 whatever the digits +// are, and a value below 10^-40 is finer than 2^-127, the finest fraction any +// F can spell — so outside that band the answer is known without the +// arithmetic, and a token spelling 1e999999999 costs nothing to refuse. +inline bool TableJsonReadWide( TableJsonIn & in, const char * token, int32_t length, void * storage, const TableFieldInfo * f ) +{ + bool is_signed = TableJsonKindWideSigned( f->kind ); + int frac = f->frac_bits; + int32_t i = 0; + bool negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { negative = token[i] == '-'; i++; } + const char * int_digits = token + i; + int32_t int_len = 0; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { int_len++; i++; } + const char * frac_digits = token + i; + int32_t frac_len = 0; + if ( i < length && token[i] == '.' ) + { + i++; + frac_digits = token + i; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { frac_len++; i++; } + } + int64_t exp = 0; + if ( i < length && ( token[i] == 'e' || token[i] == 'E' ) ) + { + i++; + bool exp_negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { exp_negative = token[i] == '-'; i++; } + while ( i < length && token[i] >= '0' && token[i] <= '9' ) + { + if ( exp < 100000 ) { exp = exp * 10 + ( token[i] - '0' ); } + i++; + } + if ( exp_negative ) { exp = -exp; } + } + // the digits, with the point after "point" of them; leading and trailing + // zeros stripped. digit( k ) reads the k-th of the int and frac runs. + int32_t start = 0, end = int_len + frac_len; + int64_t point = int_len + exp; + while ( start < end && ( start < int_len ? int_digits[start] : frac_digits[start - int_len] ) == '0' ) { start++; point--; } + while ( end > start && ( end - 1 < int_len ? int_digits[end - 1] : frac_digits[end - 1 - int_len] ) == '0' ) { end--; } + + TableJsonWide raw = { 0, 0 }; + bool saturated = false; + TableJsonWide signed_max = { ~uint64_t( 0 ), ~uint64_t( 0 ) >> 1 }; + TableJsonWide signed_min = { 0, uint64_t( 1 ) << 63 }; + TableJsonWide unsigned_max = { ~uint64_t( 0 ), ~uint64_t( 0 ) }; + if ( start == end ) + { + // zero, and -0 IS zero + } + else if ( point > 40 ) + { + saturated = true; + if ( !negative ) { raw = is_signed ? signed_max : unsigned_max; } + else if ( is_signed ) { raw = signed_min; } + } + else if ( point < -40 ) + { + in.report->kind_mismatch++; // finer than any F can spell + return true; + } + else + { + // the fraction FIRST, so an inexact value is the wrong shape whatever + // its magnitude: its digits, with the zeros a negative point puts in + // front, doubled F times; each doubling's carry is the next bit, and + // the value is exact iff nothing is left after the last one + char fd[kTableJsonMaxNumber + 48]; + int32_t fn = 0; + for ( int64_t z = point; z < 0; z++ ) { fd[fn++] = 0; } + for ( int32_t k = (int32_t) ( point > 0 ? point : 0 ) + start; k < end; k++ ) + { + fd[fn++] = (char) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ); + } + TableJsonWide fraction = { 0, 0 }; + for ( int b = 0; b < frac; b++ ) + { + int carry = 0; + for ( int32_t k = fn - 1; k >= 0; k-- ) + { + int d = fd[k] * 2 + carry; + fd[k] = (char) ( d % 10 ); + carry = d / 10; + } + fraction = TableJsonWideShl( fraction, 1 ); + fraction.lo |= (uint64_t) carry; + } + for ( int32_t k = 0; k < fn; k++ ) + { + if ( fd[k] != 0 ) + { + in.report->kind_mismatch++; + return true; + } + } + // then the whole part, saturating past 128 bits + TableJsonWide whole = { 0, 0 }; + for ( int64_t k = start; k < start + point && !saturated; k++ ) + { + uint32_t digit = k < end ? (uint32_t) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ) : 0; + if ( TableJsonWideMulAdd( &whole, 10, digit ) != 0 ) { saturated = true; } + } + if ( !saturated && frac > 0 && !TableJsonWideZero( TableJsonWideShr( whole, 128 - frac ) ) ) { saturated = true; } + if ( !saturated ) + { + raw = TableJsonWideShl( whole, frac ); + raw.lo |= fraction.lo; + raw.hi |= fraction.hi; + } + if ( is_signed ) + { + if ( !saturated && !negative && TableJsonWideNegative( raw ) ) { saturated = true; } + if ( !saturated && negative && TableJsonWideCompare( raw, signed_min, false ) > 0 ) { saturated = true; } + if ( saturated ) { raw = negative ? signed_min : signed_max; } + else if ( negative ) { raw = TableJsonWideNeg( raw ); } + } + else + { + if ( saturated ) { raw = unsigned_max; } + if ( negative && !TableJsonWideZero( raw ) ) { raw.lo = 0; raw.hi = 0; saturated = true; } + } + } + if ( saturated ) { in.report->clamped++; } + if ( f->wide != NULL ) + { + TableJsonWide lo = { f->wide->lo[0], f->wide->lo[1] }; + TableJsonWide hi = { f->wide->hi[0], f->wide->hi[1] }; + if ( TableJsonWideCompare( raw, lo, is_signed ) < 0 ) { raw = lo; in.report->clamped++; } + else if ( TableJsonWideCompare( raw, hi, is_signed ) > 0 ) { raw = hi; in.report->clamped++; } + } + TableJsonWideStore( storage, f->elem_size, raw ); + return true; +} + +inline bool TableJsonSkipValue( TableJsonIn & in, int32_t depth ); + +inline bool TableJsonSkipContainer( TableJsonIn & in, char close, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; // the opening bracket + bool first = true; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == close ) { in.pos++; return true; } + if ( c == 0 ) { in.bad = true; return false; } + if ( close == '}' ) + { + // the key is kept, because a skipped OBJECT may still be a + // pointer's: an `&node` opening it names a node the storage could + // not hold, and the numbering has to survive the drop (§16.7). + // Anywhere but first, the prefix is the reserved key out of place + // — in a pointered unit; a fixed unit skips the value whole. + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + if ( key[0] == '&' && in.graph != NULL ) + { + if ( !first ) { in.report->malformed = true; in.bad = true; return false; } + if ( !TableJsonSkippedAmpersand( in, key, depth ) ) { return false; } + first = false; + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == close ) { in.pos++; return true; } + in.bad = true; + return false; + } + } + first = false; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == close ) { in.pos++; return true; } + in.bad = true; + return false; + } +} + +inline bool TableJsonSkipValue( TableJsonIn & in, int32_t depth ) +{ + char c = TableJsonPeek( in ); + switch ( c ) + { + case '{': return TableJsonSkipContainer( in, '}', depth ); + case '[': return TableJsonSkipContainer( in, ']', depth ); + case '"': return TableJsonScanString( in, NULL, 0, NULL ); + case 't': return TableJsonLiteral( in, "true" ); + case 'f': return TableJsonLiteral( in, "false" ); + case 'n': return TableJsonLiteral( in, "null" ); + case 0: in.bad = true; return false; + default: + { + // consumed, never converted: skipping needs no buffer, and this + // is the one walk a hostile text drives to the depth cap. It is + // the SAME production the value path scans, so an unknown key + // cannot smuggle past a number a named key would refuse. + bool integral = false; + if ( !TableJsonWalkNumber( in, &integral ) ) { in.bad = true; return false; } + return true; + } + } +} + +inline bool TableJsonReadTable( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth ); +// a UNION ARM that names no declaration reads through the field walk one key +// down (docs/SPEC-TABLES.md §2.6, §16.2), which is defined below +inline bool TableJsonReadField( TableJsonIn & in, void * base, const TableFieldInfo * f, int32_t depth ); + +// place one scalar at one storage address +inline bool TableJsonReadScalar( TableJsonIn & in, void * storage, const TableFieldInfo * f, int32_t depth ) +{ + if ( f->arms != NULL ) + { + // a union is an object with ONE key, the arm's name; {} is None, and + // two keys is a text this walk will not guess at + const TableUnionInfo * arms = f->arms(); + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + TableJsonSetRaw( (uint8_t *) storage + arms->tag_offset, arms->tag_size, 0 ); + if ( TableJsonPeek( in ) == '}' ) { in.pos++; return true; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t tag = 0; + for ( int64_t t = 1; t <= f->enum_max; t++ ) + { + if ( strcmp( f->enum_name( (uint64_t) t ), key ) == 0 ) { tag = t; break; } + } + if ( tag == 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + void * payload = (uint8_t *) storage + arms->arms[tag].offset; + const TableFieldInfo * arm = arms->arms[tag].field; + bool placed = true; + if ( arm != NULL ) + { + // THE ARM'S VALUE TAKES THE ARM'S OWN ROW (§16.2). A value of + // the wrong shape for that row is a KIND MISMATCH: the union + // reads None, the event is counted, and the enclosing object + // continues — the rule a FIELD's value lives under, one key + // down. A pointer arm's null is a null pointer, not a shape + // error, exactly as a pointer field's is (§16.7). + char got = TableJsonValueShape( in ); + if ( arm->kind == 17 && !arm->is_array && got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + memset( payload, 0, (size_t) arms->arms[tag].size ); + } + else if ( got != TableJsonShape( arm ) ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else if ( arm->kind == 17 && !arm->is_array ) + { + // A POINTER ARM'S VALUE IS THE POINTEE IN PLACE, or a + // node reference to one (§16.7) — the read a pointer + // FIELD takes, which is not the scalar walk + memset( payload, 0, (size_t) arms->arms[tag].size ); + if ( !TableJsonReadPointer( in, payload, arm, depth + 1 ) ) { return false; } + } + else + { + // SELECTION ZERO-ESTABLISHES THE ARM (SPEC §5): an arm + // takes no specified default, so zero is the establish + memset( payload, 0, (size_t) arms->arms[tag].size ); + if ( !TableJsonReadField( in, storage, arm, depth + 1 ) ) { return false; } + } + } + else if ( arms->arms[tag].table != NULL ) + { + if ( TableJsonValueShape( in ) != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else + { + arms->arms[tag].table->reset( payload ); + if ( !TableJsonReadTable( in, payload, arms->arms[tag].table, depth + 1 ) ) { return false; } + } + } + else + { + // A PAYLOAD-FREE ARM'S VALUE IS null (§2.6): the arm name + // selects it and there is nothing to place + if ( TableJsonValueShape( in ) != 'z' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else if ( !TableJsonLiteral( in, "null" ) ) + { + return false; + } + } + if ( placed ) + { + TableJsonSetRaw( (uint8_t *) storage + arms->tag_offset, arms->tag_size, (uint64_t) tag ); + } + } + char c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; c = TableJsonPeek( in ); } + if ( c == '}' ) { in.pos++; return true; } + in.bad = true; // a second key: a one-of with two arms is not a value + return false; + } + if ( f->kind == 13 ) + { + f->table->reset( storage ); + return TableJsonReadTable( in, storage, f->table, depth + 1 ); + } + if ( TableJsonIsEnum( f ) ) + { + char name[kTableJsonMaxKey]; + int32_t name_length = 0; + if ( !TableJsonScanString( in, name, kTableJsonMaxKey - 1, &name_length ) ) { return false; } + name[name_length] = 0; + for ( int64_t v = 0; v <= f->enum_max; v++ ) + { + if ( strcmp( f->enum_name( (uint64_t) v ), name ) == 0 ) + { + TableJsonSetRaw( storage, f->elem_size, (uint64_t) v ); + return true; + } + } + // a name this build cannot name reads as None and counts as unknown, + // exactly as an unknown variant id does on the wire (§4) + TableJsonSetRaw( storage, f->elem_size, 0 ); + in.report->unknown++; + return true; + } + if ( TableJsonIsFlags( f ) ) + { + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + in.pos++; + uint64_t bits = 0; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + if ( c != '"' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + char name[kTableJsonMaxKey]; + int32_t name_length = 0; + if ( !TableJsonScanString( in, name, kTableJsonMaxKey - 1, &name_length ) ) { return false; } + name[name_length] = 0; + bool found = false; + for ( int64_t bit = 0; bit <= f->enum_max; bit++ ) + { + if ( strcmp( f->enum_name( (uint64_t) bit ), name ) == 0 ) + { + bits |= uint64_t( 1 ) << bit; + found = true; + break; + } + } + if ( !found ) { in.report->unknown++; } + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + TableJsonSetRaw( storage, f->elem_size, bits ); + return true; + } + if ( f->kind == 1 ) + { + char c = TableJsonPeek( in ); + if ( c == 't' ) { if ( !TableJsonLiteral( in, "true" ) ) { return false; } TableJsonSetRaw( storage, f->elem_size, 1 ); return true; } + if ( !TableJsonLiteral( in, "false" ) ) { return false; } + TableJsonSetRaw( storage, f->elem_size, 0 ); + return true; + } + char token[kTableJsonMaxNumber]; + int32_t length = 0; + bool integral = false; + if ( !TableJsonScanNumber( in, token, kTableJsonMaxNumber, &length, &integral ) ) + { + in.bad = true; + return false; + } + if ( TableJsonKindWide( f->kind ) ) + { + return TableJsonReadWide( in, token, length, storage, f ); + } + if ( f->kind == 10 || f->kind == 11 ) + { + bool single = f->kind == 10; + double value = TableJsonTokenDouble( token, length, single ); + // A magnitude the field's format cannot hold is the WRONG SHAPE for + // the kind, and it never reaches storage: 1e400 is not a float64 and + // 1e300 is not a float32. Storing the infinity the conversion + // produced would leave an instance this walk called CLEAN that + // ToJsonMeasure then refuses forever (a non-finite float has no JSON + // spelling), and §16.1's one invariant is that a text which reads + // clean writes back. + if ( !TableJsonFinite( value ) ) + { + in.report->kind_mismatch++; + return true; + } + if ( f->has_range ) + { + if ( value < f->range_min ) { value = f->range_min; in.report->clamped++; } + else if ( value > f->range_max ) { value = f->range_max; in.report->clamped++; } + } + if ( single ) + { + float narrow = (float) value; + if ( !TableJsonFinite( (double) narrow ) ) + { + in.report->kind_mismatch++; + return true; + } + memcpy( storage, &narrow, sizeof( narrow ) ); + } + else + { + memcpy( storage, &value, sizeof( value ) ); + } + return true; + } + // AN ORDINARY FIELD'S POLICY over the one interpreted value: it CLAMPS to + // its domain and counts. JSON has one number type, so 2.0 IS the integer 2 + // and 1e3 IS 1000. A library that round-trips numbers through a double + // emits them that way, and this walker's own float writer emits 1e+21. Only + // a genuinely fractional value is the wrong shape for the kind. + const bool is_signed = f->kind >= 2 && f->kind <= 5; + const TableJsonInteger number = TableJsonInterpret( token, length, integral ); + if ( !number.finite || number.fractional ) + { + in.report->kind_mismatch++; + return true; + } + if ( number.saturated ) { in.report->clamped++; } // past what sixty-four bits hold + // THE DECLARED RANGE FIRST, THEN THE STORAGE WIDTH, the wire's order (§4), + // so a text and a wire loaded from the same data land the same instance. + // The comparison is on the value's OWN scale, correctly signed past + // INT64_MAX, where the storage's bit pattern is not a number to compare. + TableJsonInteger bounded = number; + if ( f->has_range ) + { + const double scale = number.negative ? -(double) number.magnitude : (double) number.magnitude; + if ( scale < f->range_min ) { bounded = TableJsonIntegerOf( f->range_min ); in.report->clamped++; } + else if ( scale > f->range_max ) { bounded = TableJsonIntegerOf( f->range_max ); in.report->clamped++; } + } + bool moved = false; + const int64_t value = TableJsonIntegerInDomain( bounded, is_signed, (int32_t) f->elem_size, moved ); + if ( moved ) { in.report->clamped++; } + TableJsonSetRaw( storage, f->elem_size, (uint64_t) value ); + return true; +} + +inline bool TableJsonReadField( TableJsonIn & in, void * base, const TableFieldInfo * f, int32_t depth ) +{ + uint8_t * storage = (uint8_t *) base + f->offset; + if ( TableJsonIsMap( f ) ) + { + return TableJsonReadMap( in, (void *) storage, f, depth ); + } + if ( TableJsonIsList( f ) ) + { + return TableJsonReadList( in, (void *) storage, f, depth ); + } + + if ( f->kind == 12 ) + { + int32_t length = 0; + if ( !TableJsonScanString( in, (char *) storage, f->array_bound, &length ) ) { return false; } + storage[length] = 0; + TableJsonSetCount( base, f, length ); + return true; + } + if ( f->kind == 33 ) + { + char16_t * units = (char16_t *) (void *) storage; + int32_t length = 0; + if ( !TableJsonScanWString( in, units, (int32_t) f->array_bound, &length ) ) { return false; } + units[length] = 0; // the terminating zero UNIT at index length (§7.2, SPEC.md §4.12) + TableJsonSetCount( base, f, length ); + return true; + } + if ( TableJsonIsBytes( f ) ) + { + // base64 decodes STRAIGHT INTO the field's storage, six bits at a + // time — no window, no temporary, so a bytes(N) of any declared + // extent reads the same way. A base64 body carries no escapes, so a + // backslash in one is simply not an alphabet character. + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + memset( storage, 0, (size_t) f->array_bound ); + TableJsonSetCount( base, f, 0 ); + const char * alphabet = TableJsonBase64Alphabet(); + int32_t placed = 0; + uint32_t accumulator = 0; + int32_t held = 0; + bool clamped = false; + bool malformed = false; + for ( ;; ) + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos++]; + if ( c == '"' ) { break; } + if ( c == '=' || malformed ) { continue; } + const char * at = c != 0 ? strchr( alphabet, c ) : NULL; + if ( at == NULL ) { malformed = true; continue; } + accumulator = ( accumulator << 6 ) | (uint32_t) ( at - alphabet ); + held += 6; + if ( held >= 8 ) + { + held -= 8; + if ( placed < f->array_bound ) + { + storage[placed++] = (uint8_t) ( ( accumulator >> held ) & 0xff ); + } + else + { + clamped = true; + } + } + } + if ( malformed ) + { + // a body that is not base64 is the wrong shape for the kind: the + // field keeps its default and the event is counted + in.report->kind_mismatch++; + return true; + } + if ( clamped ) { in.report->clamped++; } + TableJsonSetCount( base, f, placed ); + return true; + } + if ( TableJsonIsKeyed( f ) ) + { + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + // every slot back to its declared defaults first, so a key the text + // omits keeps them and a repeated field key cannot leave an earlier + // occurrence's slots standing + for ( int32_t i = 0; i < f->array_bound; i++ ) + { + void * slot = storage + (int64_t) i * f->elem_size; + if ( f->kind == 13 ) { f->table->reset( slot ); } + else { memset( slot, 0, (size_t) f->elem_size ); } + } + char shape = TableJsonElementShape( f ); + // A KEYED OBJECT'S KEYS ARE KEYS: a variant named twice is a duplicate + // key like any other, last-wins and counted (§16.2). Tracked the way + // a table's own field keys are — a bounded, allocation-free bitmask; + // a vocabulary wider than this still reads, its repeats simply stop + // being counted. + uint64_t seen[8] = {}; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t slot = -1; + for ( int64_t v = 0; v < f->array_bound; v++ ) + { + // nothing is stored for None, so "None" finds no slot and is + // an unknown key like any other name this reader cannot place + if ( !TableJsonKeyedSlotValid( f, v ) ) { continue; } + if ( strcmp( f->key_name( TableJsonKeyedSlotKey( v ) ), key ) == 0 ) { slot = v; break; } + } + if ( slot >= 0 && slot < 512 ) + { + uint64_t bit = uint64_t( 1 ) << ( slot & 63 ); + if ( ( seen[slot >> 6] & bit ) != 0 ) { in.report->duplicate++; } + seen[slot >> 6] |= bit; + } + if ( slot < 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( TableJsonValueShape( in ) != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadScalar( in, storage + slot * f->elem_size, f, depth + 1 ) ) + { + return false; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; + } + if ( f->is_array ) + { + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + in.pos++; + // LAST WINS has to be true of a repeated ARRAY key too, and it is + // wire-visible: a fixed array writes every slot, so a second, shorter + // occurrence overlaying a prefix would leave the first occurrence's + // tail standing. The field goes back to its declared defaults before + // this occurrence's elements are placed — the re-establishment a nested + // table and a union arm already get. A table element's defaults are + // its own (the reset hook); every other element kind's storage + // default is zero, which is what the generated array declares. + if ( f->kind == 13 ) + { + for ( int32_t i = 0; i < f->array_bound; i++ ) + { + f->table->reset( storage + (int64_t) i * f->elem_size ); + } + } + else + { + memset( storage, 0, (size_t) f->array_bound * (size_t) f->elem_size ); + } + TableJsonSetCount( base, f, 0 ); + int32_t placed = 0; + char shape = TableJsonElementShape( f ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + if ( placed >= f->array_bound ) + { + // more elements than the reader's bound: the bounded prefix + // is kept and the excess counts, the wire's rule (§4) + in.report->clamped++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( f->kind == 17 ) + { + // an element of an ARRAY OF POINTERS (§2.1): null is a null slot, an + // object is the pointee in place or an `&node` reference (§16.7) + char got = TableJsonValueShape( in ); + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( storage + (int64_t) placed * f->elem_size, f->elem_size, 0 ); + } + else if ( got != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, storage + (int64_t) placed * f->elem_size, f, depth + 1 ) ) { return false; } + placed++; + } + else if ( TableJsonValueShape( in ) != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed++; + } + else + { + if ( !TableJsonReadScalar( in, storage + (int64_t) placed * f->elem_size, f, depth + 1 ) ) { return false; } + placed++; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + // a fixed array's tail keeps the defaults the prefill left there, + // exactly as a short wire count does + TableJsonSetCount( base, f, placed ); + return true; + } + return TableJsonReadScalar( in, storage, f, depth ); +} + +inline bool TableJsonReadTableKeys( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth, const char * first_key ); + +// ONE table object: keys are field keys, unknown ones are skipped and +// counted, a repeated key is last-wins and counted. The instance is already +// at its declared defaults when this is entered, so a key the text never +// mentions keeps the default an absent field takes on the wire (§4). +inline bool TableJsonReadTable( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + return TableJsonReadTableKeys( in, base, info, depth, NULL ); +} + +// The keys of an object whose brace is already consumed. A pointer's object +// opens the same way a table's does, but its FIRST key may be `&node` (§16.7) +// and the adapter that reads it has to scan the key to know — so it hands the +// key it scanned in as `first_key`, with the colon consumed, and this places +// it before scanning the rest. +inline bool TableJsonReadTableKeys( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth, const char * first_key ) +{ + // duplicate tracking, bounded and allocation-free: a table with more + // fields than this still reads, its repeats simply stop being counted + uint64_t seen[8] = {}; + for ( ;; ) + { + char key[kTableJsonMaxKey]; + char c = 0; + if ( first_key != NULL ) + { + memcpy( key, first_key, strlen( first_key ) + 1 ); // scanned into a buffer this size by the caller + first_key = NULL; + } + else + { + c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; return true; } + if ( c == 0 ) { in.bad = true; return false; } + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + } + int32_t index = -1; + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + if ( strcmp( info->fields[i].json, key ) == 0 ) { index = i; break; } + } + if ( key[0] == '&' ) + { + // THE AMPERSAND PREFIX IS RESERVED TO THE FORM (docs/SPEC-TABLES.md + // §16.7). No declaration may take a key beginning with it, so this + // is never a field this build lacks — it is the sharing construct + // somewhere it cannot stand: `&node` is the FIRST key of a pointer's + // object and nothing else, and the adapter that reads a pointer + // has consumed it before these keys are read. MALFORMED, refused + // and counted; never counted as unknown, never skipped. + in.report->malformed = true; + in.bad = true; + return false; + } + if ( index < 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + const TableFieldInfo * f = &info->fields[index]; + if ( index < 512 ) + { + uint64_t bit = uint64_t( 1 ) << ( index & 63 ); + if ( ( seen[index >> 6] & bit ) != 0 ) { in.report->duplicate++; } + seen[index >> 6] |= bit; + } + // PRESENCE OF THE KEY IS THE PRESENCE (§16.2): reaching this line + // is the key being present, so an optional is set present + // whatever its value — with one exception the page names: a JSON + // null, which reads as ABSENT rather than as a value. + char got = TableJsonValueShape( in ); + if ( f->kind == 17 && !f->is_array ) + { + // a pointer: null is a null pointer, an object is the pointee + // in place or an `&node` reference to one (§16.7), a string is + // a BYTE BUFFER's bytes (§2.5), and anything else is the wrong + // shape for the kind + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) base + f->offset, f->elem_size, 0 ); + } + else if ( got != TableJsonShape( f ) ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, (uint8_t *) base + f->offset, f, depth ) ) + { + return false; + } + } + else if ( f->optional && got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + // absent, and back at its defaults: a repeated key whose last + // occurrence is null must not leave an earlier value standing + if ( f->table != NULL ) { f->table->reset( (uint8_t *) base + f->offset ); } + else { memset( (uint8_t *) base + f->offset, 0, (size_t) f->elem_size ); } + TableJsonSetRaw( (uint8_t *) base + f->present_offset, 1, 0 ); + } + else + { + if ( got != TableJsonShape( f ) ) + { + // the wrong JSON type for the kind: skipped, never coerced + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadField( in, base, f, depth ) ) + { + return false; + } + if ( f->optional ) + { + TableJsonSetRaw( (uint8_t *) base + f->present_offset, 1, 1 ); + } + } + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; return true; } + in.bad = true; + return false; + } +} + +// ---- the two entry points the per-table wrappers name ---- + +inline bool TableJsonRead( void * value, const TableTypeInfo * info, const char * text, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableJsonIn in; + in.text = text; + in.size = bytes; + in.pos = 0; + in.report = report != NULL ? report : &ignored; + in.bad = false; + in.graph = NULL; + info->reset( value ); + if ( text == NULL || bytes < 0 ) + { + in.report->malformed = true; + return false; + } + bool ok = TableJsonReadTable( in, value, info, 0 ); + if ( ok ) + { + TableJsonSpace( in ); + if ( in.pos != in.size ) { in.bad = true; } // trailing rubbish is not one text + } + if ( in.bad || !ok ) + { + in.report->malformed = true; + return false; + } + return true; +} + +inline int64_t TableJsonWrite( const void * value, const TableTypeInfo * info, char * buffer, int64_t capacity ) +{ + TableJsonOut out; + out.buffer = buffer; + out.capacity = capacity; + out.offset = 0; + out.overflow = false; + out.graph = NULL; + if ( !TableJsonWriteValue( out, value, info, 0 ) ) { return -1; } + // THE CANONICAL TEXT ENDS WITH EXACTLY ONE NEWLINE (docs/SPEC-TABLES.md + // §16.1). Every writer emits it — this walk, the C# walk and + // "schema unpack" — and every reader accepts a text with or without one, + // because the trailing whitespace a read already skips is what makes the + // two the same text. It is a byte of the FORM rather than a file + // convention: a text that is written to a file, pasted into a diff and + // handed back through a pipe has to be one text in all three places, and a + // buffer whose last byte is a closing brace is the one shape that is not. + out.put( '\n' ); + if ( out.overflow ) { return -1; } + return out.offset; +} + +// ---- json walk: end ---- + +// ---- json graph walk: begin ---- +// +// THE VARIABLE CLASS's half of the text form (docs/SPEC-TABLES.md §16.7). The +// walk above places every kind but one; this defines the three adapters it +// calls for that one, and the two entry points a pointered table's wrappers +// name. The text is the fixed class's — a pointee is an object in place — and a +// node named more than once carries `&node`: defined once, with its fields, +// and referenced after by `{ "&node": N }` alone. + +// ---- the identity map ---- +// +// ONE map shape serves both directions. Writing keys it by a node's ADDRESS and +// counts the slots that name the node, so the second pass knows at a node's +// first occurrence whether it will be named again; reading keys it by the +// text's own label and answers the node it defined. Open addressing, a +// multiply-shift hash and quadrupling growth — TablePackMap's shape (§6.2), on +// the same terms: proportional to nodes, never to bytes, on the authoring +// side, and released before the call returns. + +struct TableJsonGraphEntry +{ + uint64_t key; // a node's address (write) or a label (read); 0 is an empty slot + int64_t count; // write: how many slots name this node + int64_t label; // write: the `&node` label assigned at its first write, 0 until then + uint8_t open; // the descent is still open: a reference here is a cycle (write), a self-reference (read) + uint32_t node; // read: the node's arena offset; 0 for a definition the reader dropped + const TableTypeInfo * type; // read: the node's table; NULL for a dropped one +}; + +struct TableJsonGraphMap +{ + TableJsonGraphEntry * entries; + int64_t capacity; // a power of two, or zero while empty + int64_t count; + TableAllocator allocator; // the caller's pair (§6.5): the builder's on read, the one handed to ToJson on write +}; + +inline void TableJsonGraphMapInit( TableJsonGraphMap & map, TableAllocator allocator ) +{ + map.entries = NULL; + map.capacity = 0; + map.count = 0; + map.allocator = allocator; +} + +inline void TableJsonGraphMapShutdown( TableJsonGraphMap & map ) +{ + map.allocator.free( map.allocator.context, map.entries ); + TableJsonGraphMapInit( map, map.allocator ); +} + +inline int64_t TableJsonGraphMapSlot( const TableJsonGraphMap & map, uint64_t key ) +{ + uint64_t hash = key * 0x9E3779B97F4A7C15ull; + hash ^= hash >> 29; + int64_t mask = map.capacity - 1; + int64_t at = (int64_t) ( hash & (uint64_t) mask ); + while ( map.entries[at].key != 0 && map.entries[at].key != key ) + { + at = ( at + 1 ) & mask; + } + return at; +} + +inline TableJsonGraphEntry * TableJsonGraphMapFind( TableJsonGraphMap & map, uint64_t key ) +{ + if ( map.capacity == 0 ) { return NULL; } + TableJsonGraphEntry * entry = &map.entries[ TableJsonGraphMapSlot( map, key ) ]; + return entry->key == key ? entry : NULL; +} + +inline bool TableJsonGraphMapGrow( TableJsonGraphMap & map ) +{ + TableJsonGraphMap grown; + grown.allocator = map.allocator; + grown.capacity = map.capacity != 0 ? map.capacity * 4 : 64; + grown.count = 0; + grown.entries = (TableJsonGraphEntry *) map.allocator.alloc( map.allocator.context, grown.capacity * (int64_t) sizeof( TableJsonGraphEntry ) ); // zeroed, by the pair's contract + if ( grown.entries == NULL ) { return false; } + for ( int64_t i = 0; i < map.capacity; i++ ) + { + if ( map.entries[i].key == 0 ) { continue; } + grown.entries[ TableJsonGraphMapSlot( grown, map.entries[i].key ) ] = map.entries[i]; + grown.count++; + } + map.allocator.free( map.allocator.context, map.entries ); + map = grown; + return true; +} + +// the entry for a key, made if it was not there; `taken` says which. NULL is the +// allocator refusing, and the walk refuses with it. +inline TableJsonGraphEntry * TableJsonGraphMapReach( TableJsonGraphMap & map, uint64_t key, bool & taken ) +{ + if ( ( map.count + 1 ) * 4 >= map.capacity * 3 ) // keep the load factor under three quarters + { + if ( !TableJsonGraphMapGrow( map ) ) { return NULL; } + } + TableJsonGraphEntry * entry = &map.entries[ TableJsonGraphMapSlot( map, key ) ]; + taken = entry->key != key; + if ( taken ) + { + entry->key = key; + map.count++; + } + return entry; +} + +// ---- reading: into a builder ---- + +struct TableJsonGraphIn +{ + TableWorker * worker; // where every node comes from + TableJsonGraphMap labels; // a label -> the node it defined +}; + +// `&node`'s value, the LABEL: a positive integer spelled as one — digits, no sign, no +// fraction, no exponent, no leading zero (§16.7). Anything else is malformed. +inline bool TableJsonScanLabel( TableJsonIn & in, uint64_t & label ) +{ + TableJsonSpace( in ); + if ( in.pos >= in.size || in.text[in.pos] < '1' || in.text[in.pos] > '9' ) + { + in.report->malformed = true; + in.bad = true; + return false; + } + uint64_t value = 0; + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) + { + uint64_t digit = (uint64_t) ( in.text[in.pos] - '0' ); + if ( value > ( UINT64_MAX - digit ) / 10 ) + { + in.report->malformed = true; + in.bad = true; + return false; + } + value = value * 10 + digit; + in.pos++; + } + label = value; + return true; +} + +// A BYTE BUFFER's text (docs/SPEC-TABLES.md §2.5, §16.2): a string. For a +// *string the string's bytes become the blob; for a *bytes the string is base64 +// and its decoded bytes do. The blob is allocated at EXACTLY the decoded +// length — the string is scanned once without keeping it to learn the length, +// and once into the node — so a blob of any size reads with no window and no +// bound to clamp against. A *bytes body that is not base64 is the wrong shape +// for the kind: the reference stays null and the event is counted. +inline bool TableJsonReadBlob( TableJsonIn & in, void * slot, const TableFieldInfo * f ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + TableRef * ref = (TableRef *) slot; + ref->value = 0; + if ( strcmp( f->type_name, "string" ) == 0 ) + { + const int64_t mark = in.pos; + int32_t length = 0; + if ( !TableJsonScanString( in, NULL, 0, &length ) ) { return false; } + in.pos = mark; + char * data = TableStringEmplace( *graph->worker, *ref, NULL, (int64_t) length ); + if ( data == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + int32_t placed = 0; + return TableJsonScanString( in, data, length, &placed ); + } + // base64: the alphabet characters decide the length, six bits apiece + const char * alphabet = TableJsonBase64Alphabet(); + const int64_t mark = in.pos + 1; + int64_t symbols = 0; + bool malformed = false; + in.pos++; + for ( ;; ) + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos++]; + if ( c == '"' ) { break; } + if ( c == '=' || malformed ) { continue; } + if ( c == 0 || strchr( alphabet, c ) == NULL ) { malformed = true; continue; } + symbols++; + } + if ( malformed ) + { + in.report->kind_mismatch++; + return true; + } + const int64_t length = ( symbols * 6 ) / 8; + uint8_t * data = TableBytesEmplace( *graph->worker, *ref, length ); + if ( data == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + int64_t placed = 0; + uint32_t accumulator = 0; + int32_t held = 0; + for ( int64_t at = mark; ; at++ ) + { + char c = in.text[at]; + if ( c == '"' ) { break; } + const char * symbol = c != '=' ? strchr( alphabet, c ) : NULL; + if ( symbol == NULL ) { continue; } + accumulator = ( accumulator << 6 ) | (uint32_t) ( symbol - alphabet ); + held += 6; + if ( held >= 8 ) + { + held -= 8; + if ( placed < length ) { data[placed++] = (uint8_t) ( ( accumulator >> held ) & 0xff ); } + } + } + return true; +} + +// A pointer's object. Its FIRST key decides what it is: `&node` naming a label not +// yet defined, with fields after it, is a DEFINITION; `&node` naming one already +// defined, alone, is a REFERENCE; any other key is a node named once, its +// object in place. The node comes from the +// builder's arena, and the slot holds its arena offset (§6.3). A pointer whose +// target is a BYTE BUFFER — no table — takes a string instead (§2.5). +inline bool TableJsonReadPointer( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( f->table == NULL ) { return TableJsonReadBlob( in, slot, f ); } + // the pointee nests one level down, exactly as a by-value table does, and + // takes the same cap: a chain nests as deep as it is long (§16.7) + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + char c = TableJsonPeek( in ); + if ( c == '}' ) + { + // an empty object: a node at its defaults, named once + in.pos++; + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + return true; + } + if ( c == 0 ) { in.bad = true; return false; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + if ( strcmp( key, "&node" ) != 0 ) + { + // a node named once: the pointee's object in place, and this key is + // its first field — unless it is the reserved prefix under a spelling + // this form does not have, which ReadTableKeys refuses + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } + return TableJsonReadTableKeys( in, node, f->table, depth + 1, key ); + } + uint64_t label = 0; + if ( !TableJsonScanLabel( in, label ) ) { return false; } + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph->labels, label, taken ); + if ( entry == NULL ) { in.report->malformed = true; in.bad = true; return false; } + // ONE SPELLING, and what follows the label says which half it is: fields + // after a label the text has not defined DEFINE it, and a label alone that + // the text has defined REFERS to it. The other two are malformed — a label + // alone that the text never defined, which would otherwise read as a default + // node under a silent report, and a field after a label already defined, + // which would be a second definition. That is what keeps a typo loud. + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; c = TableJsonPeek( in ); } + bool bare = c == '}'; + if ( bare == taken ) { in.report->malformed = true; in.bad = true; return false; } + if ( bare ) + { + // A REFERENCE. A label is defined when its object CLOSES, so a + // reference met inside its own definition — at any depth of by-value + // nesting — names a node whose descent is still open: the cycle the + // wire refuses (§3.1), refused here where it is written. A definition + // the reader dropped names no node, so the slot stays null with + // nothing more counted — the drop was counted where it happened. A + // node of another table than the slot declares is a kind mismatch, as + // on the wire. + in.pos++; + if ( entry->open != 0 ) { in.report->malformed = true; in.bad = true; return false; } + TableRef ref; + if ( entry->type == NULL ) + { + memcpy( slot, &ref, sizeof( ref ) ); + return true; + } + if ( entry->type != f->table ) + { + memcpy( slot, &ref, sizeof( ref ) ); + in.report->kind_mismatch++; + return true; + } + ref.value = (int64_t) entry->node; + memcpy( slot, &ref, sizeof( ref ) ); + return true; + } + // A DEFINITION: the node is allocated, the label is its, and the keys after + // `&node` are its fields. The entry is OPEN until the object closes, so a + // reference to the label from inside the node's own fields is refused as + // the cycle it is; the node and its table are filled in at the close. + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } + entry->open = 1; + if ( !TableJsonReadTableKeys( in, node, f->table, depth + 1, NULL ) ) { return false; } + entry = TableJsonGraphMapFind( graph->labels, label ); // the map may have grown under the descent + if ( entry == NULL ) { in.report->malformed = true; in.bad = true; return false; } + TableRef ref; + memcpy( &ref, slot, sizeof( ref ) ); + entry->node = (uint32_t) ref.value; + entry->type = f->table; + entry->open = 0; + return true; +} + +// An `&`-prefixed key opening an object the walk is SKIPPING — a value past an +// array's bound, an unknown key's value, a value of the wrong shape. A +// definition in there still takes its label, so the numbering survives whatever +// the storage could not hold (§16.7): the label is registered with no node, and a +// reference to it reads null. Any other prefixed key is the reserved prefix +// out of place. +inline bool TableJsonSkippedAmpersand( TableJsonIn & in, const char * key, int32_t ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL || strcmp( key, "&node" ) != 0 ) { in.report->malformed = true; in.bad = true; return false; } + uint64_t label = 0; + if ( !TableJsonScanLabel( in, label ) ) { return false; } + bool taken = false; + if ( TableJsonGraphMapReach( graph->labels, label, taken ) == NULL ) { in.report->malformed = true; in.bad = true; return false; } + return true; // a fresh entry is node 0, type NULL: a definition with no node +} + +// ---- writing: from a region's const root ---- + +struct TableJsonGraphOut +{ + TableJsonGraphMap nodes; // a node's address -> how many slots name it, and its `&node` once assigned + bool counting; // PASS ONE: count the references, refuse a cycle, emit nothing + int64_t next_label; +}; + +// The node a slot names: null as `null`, a node named once as its object in +// place, and a node named more than once under the construct. Which of the +// last two it is was learned in pass one; pass two spells it. +inline bool TableJsonWritePointer( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphOut * graph = (TableJsonGraphOut *) out.graph; + if ( graph == NULL ) { return false; } + const void * node = f->resolve( slot ); + if ( node == NULL ) + { + out.raw( "null", 4 ); + return true; + } + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph->nodes, (uint64_t) (uintptr_t) node, taken ); + if ( entry == NULL ) { return false; } + if ( f->table == NULL ) + { + // A BYTE BUFFER (§2.5, §16.7): its text is a string, which has no + // first key to carry `&node`, so a blob named from more than one + // slot has no spelling this form can carry and the graph is refused — + // as a shared node with nothing to write is. A blob named once is its + // bytes in place: base64 for a *bytes, the string itself for a *string. + if ( graph->counting ) { entry->count++; return true; } + if ( entry->count > 1 ) { return false; } + const TableBlob * blob = (const TableBlob *) node; + if ( blob->length > (uint32_t) 0x7fffffff ) { return false; } + if ( strcmp( f->type_name, "string" ) == 0 ) { TableJsonWriteString( out, (const char *) ( blob + 1 ), (int32_t) blob->length ); } + else { TableJsonWriteBase64( out, (const uint8_t *) ( blob + 1 ), (int32_t) blob->length ); } + return true; + } + if ( graph->counting ) + { + // PASS ONE: one visit per node, every slot that names it counted, and + // a reference to a node whose descent is still open is a cycle — + // refused here as the wire refuses it (§3.1) + entry->count++; + if ( !taken ) { return entry->open == 0; } + entry->open = 1; + if ( !TableJsonWriteValue( out, node, f->table, depth ) ) { return false; } + entry = TableJsonGraphMapFind( graph->nodes, (uint64_t) (uintptr_t) node ); // the map may have grown under the descent + if ( entry == NULL ) { return false; } + entry->open = 0; + return true; + } + // PASS TWO: a node named once is its object in place; a node named more + // than once is DEFINED at its first occurrence — `&node` first, then its + // fields — and REFERENCED by `&node` alone after that, spelled the same way at + // every site. Labels run from 1 in first-write order and are the text's own, + // so a stray number in a hand-edited text is most often one never defined. + if ( entry->count <= 1 ) + { + return TableJsonWriteValue( out, node, f->table, depth ); + } + if ( depth > kTableJsonMaxDepth ) { return false; } + if ( entry->label != 0 ) + { + out.put( '{' ); + out.line( depth + 1 ); + out.raw( "\"&node\": ", 9 ); + TableJsonWriteUnsigned( out, (uint64_t) entry->label ); + out.line( depth ); + out.put( '}' ); + return true; + } + entry->label = ++graph->next_label; + out.put( '{' ); + out.line( depth + 1 ); + out.raw( "\"&node\": ", 9 ); + TableJsonWriteUnsigned( out, (uint64_t) entry->label ); + bool any = true; + int64_t before = out.offset; + if ( !TableJsonWriteFields( out, node, f->table, depth, any ) ) { return false; } + // a definition carries at least one field, because a label alone is a + // reference: a shared node with nothing to write has no definition this + // form can spell, and the writer refuses it as it refuses any value it + // cannot spell (§16.3) + if ( out.offset == before ) { return false; } + out.line( depth ); + out.put( '}' ); + return true; +} + +// ---- the two entry points a pointered table's wrappers name ---- + +// The text into the builder's root. Every node the text names is allocated in +// the builder's arena through the field's own Emplace; the label map is the +// walk's, released before this returns. The root itself takes no label — nothing +// may name it (§16.7) — so an `&node` at the root is refused like any other key +// of the prefix. +inline bool TableJsonReadGraph( TableWorker & worker, void * root, const TableTypeInfo * info, const char * text, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + if ( worker.arena == NULL ) { if ( report != NULL ) { report->malformed = true; } return false; } + TableJsonGraphIn graph; + graph.worker = &worker; + TableJsonGraphMapInit( graph.labels, worker.arena->allocator ); + TableJsonIn in; + in.text = text; + in.size = bytes; + in.pos = 0; + in.report = report != NULL ? report : &ignored; + in.bad = false; + in.graph = &graph; + info->reset( root ); + if ( text == NULL || bytes < 0 ) + { + in.report->malformed = true; + return false; + } + bool ok = TableJsonReadTable( in, root, info, 0 ); + if ( ok ) + { + TableJsonSpace( in ); + if ( in.pos != in.size ) { in.bad = true; } // trailing rubbish is not one text + } + TableJsonGraphMapShutdown( graph.labels ); + if ( in.bad || !ok ) + { + in.report->malformed = true; + return false; + } + return true; +} + +// The text of a region's const root: measured when the buffer is NULL, written +// when it is not, over one code path. Two passes over one walk — the first +// counts how many slots name each node and refuses a cycle, the second writes +// — so a node's first occurrence knows whether it will be named again. The +// ROOT's entry is open for the whole first pass, so a reference back at it is +// the cycle it is (§3.1), and it takes no label. +inline int64_t TableJsonWriteGraph( const void * root, const TableTypeInfo * info, char * buffer, int64_t capacity, TableAllocator allocator ) +{ + if ( root == NULL ) { return -1; } + TableJsonGraphOut graph; + TableJsonGraphMapInit( graph.nodes, allocator ); + graph.counting = true; + graph.next_label = 0; + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph.nodes, (uint64_t) (uintptr_t) root, taken ); + if ( entry == NULL ) { TableJsonGraphMapShutdown( graph.nodes ); return -1; } + entry->open = 1; + TableJsonOut count; + count.buffer = NULL; + count.capacity = 0; + count.offset = 0; + count.overflow = false; + count.graph = &graph; + bool ok = TableJsonWriteValue( count, root, info, 0 ); + graph.counting = false; + TableJsonOut out; + out.buffer = buffer; + out.capacity = capacity; + out.offset = 0; + out.overflow = false; + out.graph = &graph; + if ( ok ) { ok = TableJsonWriteValue( out, root, info, 0 ); } + TableJsonGraphMapShutdown( graph.nodes ); + if ( !ok ) { return -1; } + out.put( '\n' ); // the canonical text ends with exactly one newline (§16.1) + if ( out.overflow ) { return -1; } + return out.offset; +} + +// ---- json graph walk: end ---- + +// ---- the out-of-line array's slot (docs/SPEC-TABLES.md §8.1) ---- + +inline int32_t TableJsonExtentCount( const void * slot ) +{ + int32_t count = 0; + memcpy( &count, (const uint8_t *) slot + 8, sizeof( count ) ); + return count < 0 ? 0 : count; +} + +inline const uint8_t * TableJsonExtentElements( const void * slot ) +{ + int64_t delta = 0; + memcpy( &delta, slot, sizeof( delta ) ); + return delta != 0 ? (const uint8_t *) slot + delta : NULL; +} + +// ---- json map walk: begin ---- + +inline bool TableJsonIsMap( const TableFieldInfo * f ) +{ + return f->is_array && f->array_bound == 0 && strncmp( f->type_name, "map[", 4 ) == 0; +} + +// the entry's two rows: fields[0] IS the key and fields[1] IS the value, which +// is what makes a user's own table of pairs the same bytes (§2.8) +inline const TableFieldInfo * TableJsonMapKeyField( const TableFieldInfo * f ) { return &f->table->fields[0]; } +inline const TableFieldInfo * TableJsonMapValueField( const TableFieldInfo * f ) { return &f->table->fields[1]; } + +inline bool TableJsonMapKeyIsString( const TableFieldInfo * key ) { return key->kind == 12; } +inline bool TableJsonMapKeySigned( const TableFieldInfo * key ) { return key->kind >= 2 && key->kind <= 5; } + +// AN INTEGER KEY IS THE INTEGER'S DECIMAL SPELLING, QUOTED, because a JSON +// object's keys are strings. Written digit by digit so no locale can move it. +inline void TableJsonWriteMapIntegerKey( TableJsonOut & out, const void * storage, const TableFieldInfo * key ) +{ + uint64_t magnitude = 0; + bool negative = false; + if ( TableJsonMapKeySigned( key ) ) + { + int64_t value = 0; + switch ( key->kind ) + { + case 2: value = (int64_t) *(const int8_t *) storage; break; + case 3: value = (int64_t) *(const int16_t *) storage; break; + case 4: value = (int64_t) *(const int32_t *) storage; break; + default: value = *(const int64_t *) storage; break; + } + negative = value < 0; + magnitude = negative ? ( ~(uint64_t) value ) + 1 : (uint64_t) value; + } + else + { + switch ( key->kind ) + { + case 6: magnitude = (uint64_t) *(const uint8_t *) storage; break; + case 7: magnitude = (uint64_t) *(const uint16_t *) storage; break; + case 8: magnitude = (uint64_t) *(const uint32_t *) storage; break; + default: magnitude = *(const uint64_t *) storage; break; + } + } + char digits[24]; + int32_t at = (int32_t) sizeof( digits ); + do { digits[--at] = (char) ( '0' + ( magnitude % 10 ) ); magnitude /= 10; } while ( magnitude != 0 ); + if ( negative ) { digits[--at] = '-'; } + TableJsonWriteString( out, digits + at, (int32_t) sizeof( digits ) - at ); +} + +inline void TableJsonWriteMapKey( TableJsonOut & out, const void * entry, const TableFieldInfo * key ) +{ + const uint8_t * storage = (const uint8_t *) entry + key->offset; + if ( TableJsonMapKeyIsString( key ) ) + { + // A STRING KEY IS THE STRING (§2.8): every JSON key of a map object is + // a KEY OF THE MAP and none is a field key, so the `&` prefix §16.7 + // reserves for field keys is ordinary data here. + TableJsonWriteString( out, (const char *) storage, *(const int32_t *) ( (const uint8_t *) entry + key->count_offset ) ); + return; + } + TableJsonWriteMapIntegerKey( out, (const void *) storage, key ); +} + +// ToJson WRITES ENTRIES IN ASCENDING KEY ORDER, so unpack then pack is +// byte-stable and a diff of two texts is a diff of two maps (§2.8, §17.2). +// A region holds them in that order already, so this is the array in place. +inline bool TableJsonWriteMap( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + const int32_t count = TableJsonExtentCount( slot ); + if ( count == 0 ) { out.raw( "{}", 2 ); return true; } + const TableFieldInfo * key = TableJsonMapKeyField( f ); + const TableFieldInfo * value = TableJsonMapValueField( f ); + const uint8_t * entries = TableJsonExtentElements( slot ); + out.put( '{' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + const void * entry = (const void *) ( entries + (int64_t) i * f->elem_size ); + TableJsonWriteMapKey( out, entry, key ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteField( out, entry, value, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( '}' ); + return true; +} + +// AN INTEGER KEY IS READ BY §16.2's INTEGER RULE AND BY NOTHING ELSE, so +// "2.0" and "1e3" are the integers 2 and 1000 and "-0" is zero. The token is +// walked as a JSON number over its own bytes; a token that rule calls +// malformed makes the KEY malformed, and a genuinely fractional value, or one +// outside the key kind's range, is kind_mismatch for that entry. +// +// THE KEY IS THE SPELLING AND NOTHING AROUND IT. The number walk steps over +// leading whitespace and comments, which is right BETWEEN tokens and wrong +// INSIDE one: a key is an identity, and a padded spelling that resolved to the +// same integer would be a second name for one entry. So the walk must begin at +// the token's first byte, and a token with anything before the number is not a +// JSON number at all, which is malformed on the terms "1-2" is. +inline bool TableJsonMapKeyValue( const char * token, int32_t length, const TableFieldInfo * key, + int64_t & value, bool & fits ) +{ + fits = false; + TableReport scratch; + TableJsonIn probe = { token, (int64_t) length, 0, &scratch, false, NULL }; + bool integral = false; + TableJsonSpace( probe ); + if ( probe.pos != 0 ) { return false; } // whitespace is never part of a key + if ( !TableJsonWalkNumber( probe, &integral ) ) { return false; } + if ( probe.pos != (int64_t) length ) { return false; } // trailing bytes: not a number + // A MAP KEY'S POLICY over the one interpreted value: it REJECTS THE WHOLE + // ENTRY. A key is an identity, so a clamped one is two entries merged, and + // a value the key kind does not hold is kind_mismatch for that entry, + // dropped and counted, never clamped. + const TableJsonInteger number = TableJsonInterpretExact( token, length ); + if ( number.fractional || number.saturated ) { return true; } + bool moved = false; + value = TableJsonIntegerInDomain( number, TableJsonMapKeySigned( key ), (int32_t) key->elem_size, moved ); + fits = !moved; + return true; +} + +// FromJson READS KEYS IN WHATEVER ORDER THE TEXT GIVES THEM. A repeated key is +// last-wins and counted duplicate, the object rule (§16.2) applied inside the +// map. An empty object is an empty map, and null is kind_mismatch. +inline bool TableJsonReadMap( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; + const TableFieldInfo * key = TableJsonMapKeyField( f ); + const TableFieldInfo * value = TableJsonMapValueField( f ); + const char shape = TableJsonShape( value ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + char token[kTableJsonMaxKey]; + int32_t token_length = 0; + bool key_over = false; // longer than THIS buffer: never truncated into a key + if ( !TableJsonScanString( in, token, kTableJsonMaxKey - 1, &token_length, &key_over ) ) { return false; } + token[token_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t key_value = 0; + bool place = true; + if ( !TableJsonMapKeyIsString( key ) ) + { + // AN INTEGER KEY PAST THIS SCAN'S BUFFER DROPS AS kind_mismatch, + // here and in the tool's walker. The bytes kept are a PREFIX, and a + // prefix is a different token, so the entry drops rather than a + // truncation being read as a value. The length alone does not + // settle it: a token that long can still spell a number an integer + // kind holds, "1" padded by an exponent of zeroes for one, and this + // read declines to find out. + bool fits = false; + if ( key_over ) { in.report->kind_mismatch++; place = false; } + else if ( !TableJsonMapKeyValue( token, token_length, key, key_value, fits ) ) + { + // A MALFORMED KEY STOPS THE READ where §16.1's rule stops it, + // with the instance holding what was placed before the stop. + in.report->malformed = true; + in.bad = true; + return false; + } + else if ( !fits ) { in.report->kind_mismatch++; place = false; } + } + else if ( key_over || token_length > key->array_bound ) + { + // A KEY LONGER THAN N DROPS ITS ENTRY AND COUNTS clamped, the + // wire's rule, because a clamped key is a merged entry (§2.8). The + // BOUND IS THE WALKER'S, tested here against the key field's own + // descriptor, so placement is left with one failure to report. A + // key past this scan's own buffer is the SAME event, because a + // truncated key is the merged entry the rule exists to prevent. + in.report->clamped++; + place = false; + } + const int32_t before = TableJsonExtentCount( (const void *) slot ); + void * entry = place ? f->place( *graph->worker, slot, token, token_length, key_value ) : NULL; + if ( place && entry == NULL ) + { + // AN ALLOCATION FAILURE IS NOT AN OVERSIZED KEY (§2.8, §16.1). The + // key was checked above, so the arena is what refused, and the read + // stops where the list, blob and pointer paths stop on one rather + // than handing back an instance short of entries the text spelled + // and calling itself clean. + in.report->malformed = true; + in.bad = true; + return false; + } + else if ( entry != NULL && TableJsonExtentCount( (const void *) slot ) == before ) + { + in.report->duplicate++; // last-wins, the object rule inside the map + } + const char got = TableJsonValueShape( in ); + if ( entry == NULL ) + { + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( value->kind == 17 && !value->is_array ) + { + // A POINTER VALUE IS SHARED EXACTLY AS A POINTER FIELD IS (§2.8): + // null is a null slot, an object is the pointee in place or an + // &node reference to one (§16.7), anything else is the wrong shape — + // the same three the field-key loop gives a pointer field, because + // an entry's value IS a field line. + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) entry + value->offset, value->elem_size, 0 ); + } + else if ( got != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, (uint8_t *) entry + value->offset, value, depth + 1 ) ) + { + return false; + } + } + else if ( got != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadField( in, entry, value, depth + 1 ) ) + { + return false; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; +} + +// ---- json map walk: end ---- + +// ---- json list walk: begin ---- + +// an unbounded array is the out-of-line array that is not a map (§8.1) +inline bool TableJsonIsList( const TableFieldInfo * f ) +{ + return f->is_array && f->array_bound == 0 && !TableJsonIsMap( f ); +} + +// ToJson WRITES THE ELEMENTS IN INDEX ORDER, which is the only order there is, +// so unpack then pack is byte-stable without a rule of its own (§2.9, §17.2). +// A region holds the array in place, so this steps it at the descriptor's pitch. +inline bool TableJsonWriteList( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + const int32_t count = TableJsonExtentCount( slot ); + if ( count == 0 ) { out.raw( "[]", 2 ); return true; } + const uint8_t * elements = TableJsonExtentElements( slot ); + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + const uint8_t * element = elements + (int64_t) i * f->elem_size; + if ( f->kind == 17 ) + { + // a []*T's elements take the pointer row (§16.7): the pointee's + // object in place, null, or `&node` for a shared one + if ( !TableJsonWritePointer( out, element, f, depth + 1 ) ) { return false; } + } + else if ( !TableJsonWriteScalar( out, element, f, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( ']' ); + return true; +} + +// FromJson READS EVERY ELEMENT THE TEXT CARRIES, appending each through the +// descriptor's place resolver: `[]` is an empty list, and null is +// kind_mismatch, the array row's own rule (§16.2). LAST WINS holds for a +// repeated key: the list goes back to EMPTY before this occurrence's elements +// land, the builder's storage being reclaimed at reset (§2.9). +inline bool TableJsonReadList( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; + TableJsonSetRaw( (uint8_t *) slot, 8, 0 ); + TableJsonSetRaw( (uint8_t *) slot + 8, 4, 0 ); + const char shape = TableJsonElementShape( f ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + void * element = f->place( *graph->worker, slot, NULL, 0, 0 ); + if ( element == NULL ) + { + // NOT ADDED: the arena could not carve another segment, or the + // count met the int32 cap. The text cannot be placed whole, and + // the read stops where §16.1's rule stops it. + in.report->malformed = true; + in.bad = true; + return false; + } + if ( f->kind == 17 ) + { + // an element of a []*T (§2.9): null is a null slot, an object is the + // pointee in place or an `&node` reference (§16.7) + char got = TableJsonValueShape( in ); + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) element, f->elem_size, 0 ); + } + else if ( got != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, element, f, depth + 1 ) ) { return false; } + } + else if ( TableJsonValueShape( in ) != shape ) + { + // the wrong shape for the element kind: the slot keeps its + // defaults and the event counts, the array row's rule (§16.2) + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadScalar( in, element, f, depth + 1 ) ) { return false; } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; +} + +// ---- json list walk: end ---- + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_JSON + +namespace mapdemo { + +bool CrewsFromJson( CrewsBuilder & builder, const char * text, int64_t bytes, TableReport * report ) +{ + Crews * root = builder.GetRoot(); + if ( root == NULL ) { if ( report != NULL ) { report->malformed = true; } return false; } // locked, or the root allocation failed + return TableJsonReadGraph( builder.main, root, CrewsTableType(), text, bytes, report ); +} + +int64_t CrewsToJsonMeasure( const Crews * root, TableAllocator allocator ) +{ + return TableJsonWriteGraph( root, CrewsTableType(), NULL, 0, allocator ); +} + +int64_t CrewsToJson( const Crews * root, char * buffer, int64_t capacity, TableAllocator allocator ) +{ + return TableJsonWriteGraph( root, CrewsTableType(), buffer, capacity, allocator ); +} + +} // namespace mapdemo diff --git a/testdata/golden/tables/maps/CrewsTable.h b/testdata/golden/tables/maps/CrewsTable.h new file mode 100644 index 000000000..0acb2ac69 --- /dev/null +++ b/testdata/golden/tables/maps/CrewsTable.h @@ -0,0 +1,9466 @@ +// Code generated by the schema compiler from Crews.schema. DO NOT EDIT. +// SPDX-License-Identifier: NONE — this generated output is yours, under terms of +// your choice. See the LICENSE exception in the schema compiler; the compiler is +// AGPL-3.0, its output is not. +// package mapdemo — protocol id 0x1ac124decde5b2aa (packets only: tables version by field id, not by protocol id) +// The TABLE wire (evolution-tolerant, docs/SPEC-TABLES.md): no serialize +// dependency — includable from any TU. + +#pragma once + +#include +#include // the prefill's scalar-array fills +#include // offsetof, for the reflection descriptors + +// ---- the hooks (docs/USAGE.md, "the C++ table runtime's hooks") ---- +// +// schema_assert — the runtime's own assert, and the refusal a debugger reads. +// NDEBUG removes it, exactly as it removes assert. A caller who already routes +// serialize's asserts writes `#define schema_assert serialize_assert` before +// including this header and both halves land in one handler. +#ifndef schema_assert +#include +#define schema_assert assert +#endif // #ifndef schema_assert + +// schema_fatal — what stands after the assert on a path that cannot continue. +// NDEBUG does not remove it. Supply it and is never included. +#ifndef schema_fatal +#include // abort +#define schema_fatal abort +#endif // #ifndef schema_fatal + +// schema_allocate / schema_release — what "no allocator handed in" means for +// this program. schema_allocate hands back ZEROED bytes and NULL on failure: +// an arena segment is copied whole, padding included, so anything left +// uninitialized here would reach a packed region. Supply both and +// is never included; hand a TableAllocator to a builder to route one +// structure's allocations somewhere else again. +#ifndef schema_allocate +#include // calloc, free +#define schema_allocate( bytes ) calloc( (size_t) 1, (size_t) ( bytes ) ) +#define schema_release( pointer ) free( pointer ) +#endif // #ifndef schema_allocate +#include // a node's lifetime starts in arena storage (placement new) +#include // one atomic per slab: the arena is lock-free by ownership + +#include "Crews.h" +#include "FleetTable.h" + +#ifndef MAPDEMO_SCHEMA_TABLE_PRIMITIVES +#define MAPDEMO_SCHEMA_TABLE_PRIMITIVES + +// THE CODEC DOES NOT DEPEND ON THE COMPILER'S INLINING BUDGET. A table of a +// realistic field count emits one large body per type, and the cursor a body +// writes through lives in the caller's `TableWriter`: across a call boundary +// that cursor round-trips through memory, and a `uint8_t *` store may alias the +// writer itself, so every put reloads it. When a budget runs out mid-body the +// codec silently degrades to that shape. Forcing the primitives and the +// fixed-class bodies inline is what keeps the cursor in registers and lets +// adjacent constant framing bytes merge into one store. +#if defined( _MSC_VER ) +#define MAPDEMO_TABLE_INLINE __forceinline +#elif defined( __GNUC__ ) || defined( __clang__ ) +#define MAPDEMO_TABLE_INLINE inline __attribute__(( always_inline )) +#else +#define MAPDEMO_TABLE_INLINE inline +#endif + +namespace mapdemo { + +// WHY A READ WAS REFUSED, by name (docs/SPEC-TABLES.md §3.3, §11). A REFUSAL +// is not one of §4's events: nothing is decoded, no counter moves and no +// damage is reported, so five zero counters and a false flag are what a clean +// read prints too and only the verdict tells them apart. The reason says which +// refusal it was. +// +// This is the MESSAGE PATH's vocabulary and not the cooked form's (§7.4): a +// caller meeting one of these has been refused a MESSAGE on a connection, +// which is a different recovery with a different owner than a file a header +// match turned down. +enum TableMessageReason +{ + newer_form, // a FORM BYTE this reader does not carry (§3) + no_vocabulary, // no table for this connection: the message arrived before the announcement, or after a refused one + second_announcement, // a second announcement on a connection: it sets nothing, amends nothing, and the connection closes + vocabulary_too_large, // an announcement above the receiver's declared bound, refused before an entry is touched + message_form_as_file, // a form 2 wire where a FILE was expected: its table is somewhere else + batch_too_large // a batch of more than 256 bodies on the write side, or of more than the caller has room for on the read side: nothing is written or decoded, and the count says what the wire carries +}; + +// The table-wire read report — the permissive contract's ledger. Silence +// (all zero) means the data matched this reader's schema exactly. +struct TableReport +{ + int32_t unknown = 0; // unknown field ids skipped (newer data) + int32_t kind_mismatch = 0; // known id, changed type — skipped, never misdecoded + // a kind that GREW since the writer (docs/SPEC-TABLES.md §4): an integer + // kind read into a wider one of the same signedness, or f32 into f64, + // decoded EXACTLY. One count per field or per map. It is the one counter + // that names no loss: the bytes were not the shape this reader declares, + // and the number survived. + int32_t widened = 0; + int32_t clamped = 0; // out-of-range values clamped to declared bounds + // a key the TEXT form saw twice: last wins, and the repeat is counted + // (docs/SPEC-TABLES.md §16.2). The wire never raises it — a body carrying an + // id twice is legal input whose last occurrence wins, silently (§3). + int32_t duplicate = 0; + bool malformed = false; // framing damage; decode stopped, partial result kept + // THE REFUSAL VERDICT, which is not one of §4's events and moves no counter + // (docs/SPEC-TABLES.md §3): a FORM BYTE this reader does not carry. Five + // zero counters and a false flag are what a clean read prints too, so the + // verdict is what tells the two apart. + bool refused = false; + // WHICH refusal, and it is read only when refused is set: a read that + // was not refused has no reason, and this member is the one the caller + // must not look at then (docs/SPEC-TABLES.md §3.3). + TableMessageReason reason = newer_form; + // RETAIN-UNKNOWN's pair (docs/SPEC-TABLES.md §6.6), on the same struct for + // the reason duplicate is: a caller has one report type and not two. Both + // are ZERO in every read that did not opt in, and retention moves no + // counter above. A retained field still counts unknown, because unknown + // says what a READER could not name and that stays true. + int32_t retained = 0; // unknown fields whose bytes were kept + int32_t retain_lost = 0; // every unknown this load or save could not keep +}; + + +// WHY A FILE WAS REFUSED, by name (docs/SPEC-TABLES.md §6.5, §7, §19.2): the +// one vocabulary a cook's Open, a block's BlockOpen and a load measure's -1 +// share, because a caller asking "why can I not have this file" is +// asking one question whichever call refused it. The FIRST failing clause names the +// reason, in the order §7 enumerates, so one file answers one value in every +// language. A refusal moves no counter, and a match writes nothing: the +// out-parameter is touched on the refusal path only. +// +// It is not the MESSAGE FORM's vocabulary (TableMessageReason, §3.3): a caller +// meeting one of these has been refused a FILE, by a header match or by a +// measure. +enum TableRefuseReason +{ + ok, // no clause failed: the only value beside a non-null root (§7) + not_a_cook, // the magic is neither this build's constant nor its byte reversal, or the byte-order word contradicts the magic + foreign_order, // the magic byte-reversed: a cook of the other byte order (§7.1) + wrong_build_version, // the build_version word is not this build's (§20) + reserved_not_zero, // a reserved header word is not zero (§7.1) + bad_alignment, // the alignment word is not a power of two, is below eight, is above sixty-four, or is not a multiple of the root's own alignof + truncated, // the part lengths against the caller's length, or a data part too short to hold the root + unaligned_base, // the pointer the caller passed is not aligned for the region: the caller's defect, not the file's + bad_layout, // BlockOpen (§19.2): a pitch, a count, an offset or an extent that disagrees with this build's or leaves the block + unknown_form, // at a MEASURE (§3, §6.5): a form byte this build does not carry, refused before any read + count_over_length, // an array or map count whose elements cannot fit the field's own L (§2.8, §2.9) + count_over_extent_cap, // a count above the int32 extent cap (§2.2), which no region can hold whatever its size + blob_over_size_cap, // a blob whose length is past the derived-size cap (§3.1, §11) + data_cycle // a data cycle reached from a builder: the AUTHORING side's -1 (§3.1, §7.6) +}; +// ---- reflection (tables only, docs/SPEC-TABLES.md) ---- +// +// Static field descriptors for every type in the table closure: name, wire +// id/kind, storage offset, bounds, ranges, enum names and branch guards — +// enough to walk, print, diff, edit or bind any table value at runtime with +// no RTTI and no schema files. TableType() returns X's descriptor. + +struct TableTypeInfo; + +// One arm of a union field: where its payload sits inside the union's storage +// and what its payload looks like. The arm's NAME and its table-wire id come +// from the field's enum_name/variant_id functions at the same tag, so nothing +// is spelled twice (docs/SPEC-TABLES.md §8). +struct TableFieldInfo; + +struct TableUnionArmInfo +{ + uint32_t offset; // offsetof the arm's payload within the union storage + const TableTypeInfo * table; // the arm payload's descriptor, or NULL + // AN ARM IS A FIELD LINE (docs/SPEC-TABLES.md §2.6): an arm that names no + // declared type or table carries the FIELD descriptor a field of that + // type would carry instead — offsets taken within the union storage — so + // a generic walk meets an arm's kind, width, bounds and companions where + // it meets a field's. Exactly one of the two is non-NULL on a set arm. + const TableFieldInfo * field; + uint32_t size; // the arm's whole storage, which selection zero-establishes +}; + +// A union field's shape: the tag, and the arms indexed by it. Arms run +// [0, enum_max]; index 0 is the EMPTY arm and carries no payload. +struct TableUnionInfo +{ + uint32_t tag_offset; // offsetof the tag within the union storage + uint32_t tag_size; // sizeof the tag + const TableUnionArmInfo * arms; +}; + +// The exact raw range of a wide-kind field (docs/SPEC-TABLES.md §8.2): two 128-bit +// values as 64-bit lanes, low lane first, two's complement for the signed kinds. +struct TableWideRange +{ + uint64_t lo[2]; + uint64_t hi[2]; +}; + +// THE SHARED EMPTY DOC (docs/SPEC-TABLES.md §8.1): a declaration with no /// +// block carries a doc column pointing at this one object, so absence costs a +// unit no string data and a printer concatenates doc columns with no null +// test. One definition for the whole unit: every absent doc compares equal by +// address. +inline const char TableDocNone[1] = ""; + +// the arena's allocation front, defined with the variable-length runtime +// below; a descriptor names it only through a pointer parameter. +struct TableWorker; + +struct TableFieldInfo +{ + const char * name; // schema field name, e.g. "health" + const char * json; // the TEXT form's key: the json = "key" attribute, else name (§16.3) + const char * type_name; // schema type name, e.g. "float32", "Grade" + uint64_t id; // table-wire field id: fnv1a64 of the name, of the was alias after a rename (§5) + uint8_t kind; // table-wire kind; for arrays/strings/bytes, the ELEMENT kind + bool is_array; // fixed or counted array (bytes included) + bool is_pointer; // a *T pointer field: storage is an 8-byte TableRef; the target is a table + // THE TWO THE TEXT FORM NEEDS (docs/SPEC-TABLES.md §16.7), and they + // are here for the same reason is_pointer is: the walk is ONE walk + // over descriptors and cannot spell a target's own At or + // Emplace. `resolve` reads a slot in a REGION and answers the + // node it names, or NULL; `emplace` allocates one in a BUILDER's + // arena and points the slot at it. NULL on every field that is not + // a pointer, and emitted only in a unit that declares one. + const void * (*resolve)( const void * slot ); + void * (*emplace)( TableWorker & worker, void * slot ); + bool counted; // a _count/_length int32 companion exists (counted arrays, strings, bytes) + bool optional; // a ?T field: a _present bool companion decides whether it rides + int32_t array_bound; // array capacity / string max length; 0 for plain scalars + uint32_t offset; // offsetof the storage member + uint32_t elem_size; // sizeof the member (element size for arrays) + uint32_t count_offset; // offsetof the _count/_length companion, or 0xffffffff + uint32_t present_offset; // offsetof the _present companion, or 0xffffffff + const TableTypeInfo * table; // nested table's descriptor, or NULL + bool has_range; // a declared [min, max] (int or float) + double range_min; // NOTE: int64 ranges beyond 2^53 lose precision here + double range_max; + // the WIDE kinds (18-29, docs/SPEC-TABLES.md §3, §8.2): frac_bits is a fixed + // field's F — its storage holds units × 2^F — and wide is the declared + // range on that RAW scale, exact, as two 128-bit two's-complement values + // in 64-bit lanes (low lane first). NULL where the declaration bounds + // nothing (a bare uint128) and for every other kind; frac_bits is 0 for + // every kind that is not fixed-point. range_min/range_max still carry + // the declared bounds as doubles — whole units for a fixed field — for + // a walker that only shows them. + uint8_t frac_bits; + const TableWideRange * wide; + int64_t enum_max; // enums: highest valid value (None = 0 always valid); + // unions: the arm count (tag range [0, enum_max]); + // flags: the highest declared BIT INDEX; else -1 + // the vocabulary's names, indexed the same way enum_max bounds: an enum's + // value -> name, a union's tag -> arm name, a FLAGS field's bit index -> + // variant name. NULL for every other kind. + const char * (*enum_name)( uint64_t value ); + // the TABLE-WIRE id of one variant (docs/SPEC-TABLES.md §5): for an enum, the + // hash of the variant's name; for a union, the hash of the arm's name. + // 0 is the reserved id — an enum's None, a union's empty. NULL for every + // other kind — a FLAGS field's variants have no per-variant wire id (§4), + // so a NULL here beside a non-NULL enum_name is what says "flags". + // Walk [0, enum_max] to enumerate a vocabulary and its ids. + uint64_t (*variant_id)( uint64_t value ); + // an ENUM-KEYED array (docs/SPEC-TABLES.md §2.4): the array has one slot per + // variant of key_type_name, indexed by the variant's value, and its slots + // ride under variant ids rather than positions. key_name and key_id are + // the key's vocabulary — walk [0, array_bound) to print slots by name. + // NULL on every other field. + const char * key_type_name; + const char * (*key_name)( uint64_t value ); + uint64_t (*key_id)( uint64_t value ); + // union fields: the tag and its arms, behind a function so the whole + // descriptor stays CONSTANT-INITIALISED (a captureless lambda converts to + // a function pointer at compile time; the arms themselves are a static + // inside it). NULL for every other kind. + const TableUnionInfo * (*arms)(); + // an OUT-OF-LINE array (docs/SPEC-TABLES.md §8.1): place one element and + // hand it back at its defaults. A MAP places BY KEY, a string key comes + // in as the bytes and the length, an integer key as the value, and NULL + // is NOT INSERTED: a key past the bound, or an arena that could not carve + // another segment. A LIST ignores the key and APPENDS, NULL at the arena + // or the int32 cap. NULL on every field that is neither. + void * ( * place )( TableWorker & worker, void * slot, const char * key, int32_t key_length, int64_t key_value ); + const char * guard; // branch guard, e.g. "at_rest" or "!at_rest"; "" if unguarded + // what a PERSON wrote about the field (docs/SPEC-TABLES.md §8.1): the /// + // block above it, verbatim (SPEC §4.1). It is TableDocNone when there is + // none, never NULL. Its tags (SPEC §4.2) follow in declared order, and an + // untagged field is 0 beside NULL. Static, constant-initialized, + // allocating nothing. + const char * doc; + int32_t num_tags; + const char * const * tags; +}; + +struct TableTypeInfo +{ + const char * name; // schema type name + uint32_t size; // sizeof the storage struct + int32_t num_fields; + const TableFieldInfo * fields; + // put one instance back at its declared defaults, in place. A generic + // walker that fills a value has to be able to establish the defaults an + // absent field takes, and it holds no type to spell — this is the one + // thing the descriptors could not express without it. Placement-new + // value-init, exactly what the wire's read path does, and no temporary. + void (*reset)( void * storage ); + // the DERIVED mode (docs/SPEC-TABLES.md): false = fixed-size, a plain + // relocatable struct; true = variable-length, built through a Builder + // and read through a region root. Nobody declares it; the compiler + // works it out. + bool variable; + // the declaration's own doc and tags, on the same terms as a field's + // (docs/SPEC-TABLES.md §8.1) + const char * doc; + int32_t num_tags; + const char * const * tags; +}; + +struct TableWriter +{ + uint8_t * buffer; + int64_t capacity; + int64_t offset = 0; + bool overflow = false; + + // the parameters do not repeat the member names: a parameter that hides a + // member is a warning the estate's compilers disagree about (gcc's + // -Wshadow and cl's C4458 refuse it, clang's -Wshadow does not), and this + // is a header a consumer compiles under its OWN flags + TableWriter( uint8_t * to_buffer, int64_t to_capacity ) : buffer( to_buffer ), capacity( to_capacity ) {} + + MAPDEMO_TABLE_INLINE void raw( const void * data, int64_t bytes ) + { + if ( offset + bytes > capacity ) { overflow = true; return; } + memcpy( buffer + offset, data, (size_t) bytes ); + offset += bytes; + } + MAPDEMO_TABLE_INLINE void put8( uint8_t v ) { raw( &v, 1 ); } + MAPDEMO_TABLE_INLINE void put16( uint16_t v ) { uint8_t b[2] = { uint8_t( v ), uint8_t( v >> 8 ) }; raw( b, 2 ); } + MAPDEMO_TABLE_INLINE void put32( uint32_t v ) { uint8_t b[4] = { uint8_t( v ), uint8_t( v >> 8 ), uint8_t( v >> 16 ), uint8_t( v >> 24 ) }; raw( b, 4 ); } + MAPDEMO_TABLE_INLINE void put64( uint64_t v ) { put32( uint32_t( v ) ); put32( uint32_t( v >> 32 ) ); } + // a 128-bit value as two lanes, the low half first (docs/SPEC-TABLES.md §3) + MAPDEMO_TABLE_INLINE void put128( uint64_t lo, uint64_t hi ) { put64( lo ); put64( hi ); } + // EVERY LENGTH, COUNT, INDEX AND ID REFERENCE IS ONE CANONICAL UNSIGNED + // LEB128 (docs/SPEC-TABLES.md §3): seven value bits a byte, the lowest + // group first, the high bit set on every byte but the last. One value has + // one spelling, so two conforming writers agree byte for byte. + MAPDEMO_TABLE_INLINE void putleb( uint64_t v ) + { + while ( v >= 0x80 ) { put8( uint8_t( v ) | 0x80 ); v >>= 7; } + put8( uint8_t( v ) ); + } +}; + +// TableLebBytes is one value's spelling length, which a MEASURE needs before +// the bytes exist — the length of a body has to be known before it is written, +// because a length whose own width moves cannot be patched in place. +inline int64_t TableLebBytes( uint64_t v ) +{ + int64_t n = 1; + while ( v >= 0x80 ) { v >>= 7; n++; } + return n; +} + +// THE ID TABLE, WRITER SIDE (docs/SPEC-TABLES.md §3). It holds every id the +// body used, once each, in FIRST-USE order over the whole wire, and the body +// names them by position: reference k is the kth entry, counted from 1, and +// reference 0 names NO ID. +// +// Its capacity is a COMPILE-TIME fact of the unit — the distinct names its +// table closure can spell — so a save allocates nothing: the table is a local +// of Measure and of Save. The bucket chain makes ref constant time and makes +// truncate constant time too, which is what an ELIDED field needs: a field +// that turns out not to ride costs nothing in the id table either, so the walk +// interns its id, builds the payload that decides, and undoes the entry when +// nothing rides. +struct TableIds +{ + static const int32_t kCapacity = 76; + static const int32_t kBuckets = 256; + + uint64_t ids[ kCapacity ]; + int32_t chain[ kCapacity ]; + int32_t head[ kBuckets ]; + int32_t count; + bool overflow; + + TableIds() : count( 0 ), overflow( false ) + { + for ( int32_t i = 0; i < kBuckets; i++ ) { head[i] = -1; } + } + + static MAPDEMO_TABLE_INLINE uint32_t bucket_of( uint64_t id ) + { + return uint32_t( ( id * 0x9E3779B97F4A7C15ull ) >> 56 ) & uint32_t( kBuckets - 1 ); + } + + // the reference an id takes: the file's own first-use entry, appended on + // first use. The MESSAGE form names no id at all: its references are + // compile-time slots of the announced vocabulary (docs/SPEC-TABLES.md §3.3). + MAPDEMO_TABLE_INLINE uint64_t ref( uint64_t id ) + { + const uint32_t b = bucket_of( id ); + for ( int32_t i = head[b]; i >= 0; i = chain[i] ) + { + if ( ids[i] == id ) { return uint64_t( i ) + 1; } + } + if ( count >= kCapacity ) { overflow = true; return 1; } + ids[count] = id; chain[count] = head[b]; head[b] = count; count++; + return uint64_t( count ); + } + + // undo every entry appended since mark. An entry removed is the most + // recent one in its bucket, so it sits at that bucket's head. + void truncate( int32_t mark ) + { + while ( count > mark ) + { + count--; + head[ bucket_of( ids[count] ) ] = chain[count]; + } + } +}; + +// TableIdsBytes is the trailer's own size: the entries, each a fixed +// little-endian u64, and the ENTRY COUNT, the one fixed-width number on the +// wire (docs/SPEC-TABLES.md §3). +inline int64_t TableIdsBytes( const TableIds & ids ) { return int64_t( ids.count ) * 8 + 8; } + +// TableIdsWrite puts the trailer where the walk ended: a writer never patches, +// because first-use order is known only when the walk ends. +inline void TableIdsWrite( TableWriter & w, const TableIds & ids ) +{ + for ( int32_t i = 0; i < ids.count; i++ ) { w.put64( ids.ids[i] ); } + w.put64( uint64_t( ids.count ) ); +} + +// THE ID TABLE, READER SIDE (docs/SPEC-TABLES.md §3). A reader locates it from +// the END of the wire and resolves it ONCE, at open: the entries are eight +// bytes each and a body names them by position, so every field dispatches +// through an index rather than through a search over hashes. +struct TableIdTable +{ + const uint8_t * entries = NULL; + int64_t count = 0; + + // the id a reference names. ref is 1-based and bounds-checked by the + // caller: a reference ABOVE the entry count is framing damage on the body + // that carries it, and 0 names no id at all. + uint64_t at( uint64_t ref ) const + { + const uint8_t * e = entries + ( ref - 1 ) * 8; + uint64_t lo = uint64_t( e[0] ) | uint64_t( e[1] ) << 8 | uint64_t( e[2] ) << 16 | uint64_t( e[3] ) << 24; + uint64_t hi = uint64_t( e[4] ) | uint64_t( e[5] ) << 8 | uint64_t( e[6] ) << 16 | uint64_t( e[7] ) << 24; + return lo | ( hi << 32 ); + } +}; + +struct TableReader +{ + const uint8_t * buffer; + int64_t size; + int64_t offset = 0; + TableReport * report; + const TableIdTable * ids = NULL; + // ONLY THE ROOT BODY CARRIES THE NODE TABLE (docs/SPEC-TABLES.md §3.1), so + // a body has to know which it is: the reserved id inside a NESTED body is + // malformed, because a second numbering cannot exist. Every reader made + // for a payload is nested; the two the wire surfaces make for a root say so. + bool nested = true; + + TableReader( const uint8_t * from_buffer, int64_t from_size, TableReport * to_report ) + : buffer( from_buffer ), size( from_size ), report( to_report ) {} + + TableReader( const uint8_t * from_buffer, int64_t from_size, TableReport * to_report, const TableIdTable * to_ids ) + : buffer( from_buffer ), size( from_size ), report( to_report ), ids( to_ids ) {} + + MAPDEMO_TABLE_INLINE bool has( int64_t bytes ) const { return offset + bytes <= size; } + // A LENGTH IS A 64-BIT NUMBER AND A BUFFER IS NOT (docs/SPEC-TABLES.md + // §3): every length, count and index on this wire has sixty-four bits of + // capability, so one past what remains must be compared UNSIGNED. Casting + // it to int64 first turns 0xFFFFFFFFFFFFFFFF into -1, and a negative + // length looks like room. + MAPDEMO_TABLE_INLINE bool room( uint64_t bytes ) const { return bytes <= (uint64_t) ( size - offset ); } + MAPDEMO_TABLE_INLINE uint8_t get8() { return buffer[offset++]; } + MAPDEMO_TABLE_INLINE uint16_t get16() { uint16_t v = uint16_t( buffer[offset] ) | uint16_t( buffer[offset+1] ) << 8; offset += 2; return v; } + MAPDEMO_TABLE_INLINE uint32_t get32() { uint32_t v = uint32_t( buffer[offset] ) | uint32_t( buffer[offset+1] ) << 8 | uint32_t( buffer[offset+2] ) << 16 | uint32_t( buffer[offset+3] ) << 24; offset += 4; return v; } + MAPDEMO_TABLE_INLINE uint64_t get64() { uint64_t lo = get32(); uint64_t hi = get32(); return lo | ( hi << 32 ); } + MAPDEMO_TABLE_INLINE void get128( uint64_t & lo, uint64_t & hi ) { lo = get64(); hi = get64(); } + + // ONE CANONICAL UNSIGNED LEB128 (docs/SPEC-TABLES.md §3), and a + // non-minimal spelling is MALFORMED: 0x80 0x00 and 0x00 both spell zero, + // and only the second is legal input. An encoding past ten bytes, or a + // tenth byte with a bit above the 64th value bit, is malformed on the same + // rule. false = framing damage on the body carrying it. + bool getleb( uint64_t & value ) + { + // A NUMBER THIS READER REFUSES LEAVES THE CURSOR WHERE IT WAS. The + // caller's next question is often "did this body end exactly at its + // L", and a rejected number that had moved the cursor would answer + // that question with the damage already stepped over. + const int64_t at = offset; + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( !has( 1 ) ) { offset = at; return false; } + const uint8_t b = get8(); + if ( i == 9 && b > 1 ) { offset = at; return false; } + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) + { + if ( i > 0 && b == 0 ) { offset = at; return false; } // a redundant continuation + return true; + } + shift += 7; + } + offset = at; + return false; + } + + // resolve one id reference against the file's table. false = a reference + // ABOVE the entry count, or a 0 where an id is required, both of which + // are framing damage on the body that carries it. + bool getid( uint64_t & id ) + { + uint64_t ref = 0; + if ( !getleb( ref ) ) { return false; } + if ( ref == 0 || ids == NULL || ref > (uint64_t) ids->count ) { return false; } + id = ids->at( ref ); + return true; + } + + // skip one payload by kind; false = framing damage. FOUR RULES COVER THE + // SET (docs/SPEC-TABLES.md §3), and a kind outside it is not skippable — + // which is why the set is closed and why kind 31 exists. + bool skip( uint8_t kind ) + { + switch ( kind ) + { + // the fixed-width kinds, each by its width: 18-29 are the 128-bit integers and + // the fixed-point family at every storage width (docs/SPEC-TABLES.md §3) + case 1: case 2: case 6: case 20: case 25: return has( 1 ) ? ( offset += 1, true ) : false; + case 3: case 7: case 21: case 26: return has( 2 ) ? ( offset += 2, true ) : false; + case 4: case 8: case 10: case 22: case 27: return has( 4 ) ? ( offset += 4, true ) : false; + case 5: case 9: case 11: case 23: case 28: return has( 8 ) ? ( offset += 8, true ) : false; + case 18: case 19: case 24: case 29: return has( 16 ) ? ( offset += 16, true ) : false; + case 17: case 30: // a NODE INDEX (§3.1) and an ENUM's variant reference: one LEB128 and stop + { + uint64_t ignored = 0; + return getleb( ignored ); + } + case 12: case 13: case 14: case 16: case 31: case 32: case 33: // 31 is the ESCAPE, 32 the payload-free kind, 33 wide text + { + uint64_t n = 0; + if ( !getleb( n ) ) return false; + return room( n ) ? ( offset += (int64_t) n, true ) : false; + } + case 15: // union: the arm id reference, then its kind, its L and its payload (reference 0 = empty) + { + uint64_t arm = 0; + if ( !getleb( arm ) ) return false; + if ( arm == 0 ) return true; + if ( !has( 1 ) ) return false; + offset += 1; // the arm's kind byte + uint64_t n = 0; + if ( !getleb( n ) ) return false; + return room( n ) ? ( offset += (int64_t) n, true ) : false; + } + // KIND 34 IS RESERVED FOR float16 AND IS NOT PART OF THIS MAJOR (§3): + // no writer emits it and no reader has a rule for it, so a reader + // meets it only as DAMAGE, exactly as it meets 35 or 200. A bare 34 + // is a writer that ignored the escape kind 31. + case 34: return false; + } + return false; + } +}; + + +// WIDENING (docs/SPEC-TABLES.md §4): a payload under a kind BELOW the reader's +// on the same ladder decodes exactly. The signed ladder is kinds 2, 3, 4, 5, +// 18, the unsigned one 6, 7, 8, 9, 19, and 10 into 11 is the float rung. Every +// other pair is a kind mismatch. The declared kind is a constant at every call +// site, so this folds to one or two comparisons on the mismatch path and to +// nothing on the matching one. +inline bool TableKindWidens( uint8_t kind, uint8_t declared ) +{ + switch ( declared ) + { + case 3: case 4: case 5: return kind >= 2 && kind < declared; + case 18: return kind >= 2 && kind <= 5; + case 7: case 8: case 9: return kind >= 6 && kind < declared; + case 19: return kind >= 6 && kind <= 9; + case 11: return kind == 10; + } + return false; +} + +// a fixed-width kind's payload width, for the one place the width is a +// runtime fact: an arm whose kind byte the reader widens, whose L must be the +// wire kind's own width (§3) +inline int64_t TableKindWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + } + return 0; +} + +// the payload of a kind on the SIGNED ladder (2 to 5), sign-extended to +// sixty-four bits; false = the body cannot cover it, which is framing damage +inline bool TableReadSignedAt( TableReader & r, uint8_t kind, int64_t & out ) +{ + switch ( kind ) + { + case 2: if ( !r.has( 1 ) ) { return false; } out = (int8_t) r.get8(); return true; + case 3: if ( !r.has( 2 ) ) { return false; } out = (int16_t) r.get16(); return true; + case 4: if ( !r.has( 4 ) ) { return false; } out = (int32_t) r.get32(); return true; + default: if ( !r.has( 8 ) ) { return false; } out = (int64_t) r.get64(); return true; + } +} + +// the payload of a kind on the UNSIGNED ladder (6 to 9), zero-extended +inline bool TableReadUnsignedAt( TableReader & r, uint8_t kind, uint64_t & out ) +{ + switch ( kind ) + { + case 6: if ( !r.has( 1 ) ) { return false; } out = r.get8(); return true; + case 7: if ( !r.has( 2 ) ) { return false; } out = r.get16(); return true; + case 8: if ( !r.has( 4 ) ) { return false; } out = r.get32(); return true; + default: if ( !r.has( 8 ) ) { return false; } out = r.get64(); return true; + } +} + +// f32 into f64, exact: a NaN's payload is data and rides on the bits, since +// the hardware conversion would set the quiet bit (§4) +inline double TableWidenF32( uint32_t bits ) +{ + if ( ( bits & 0x7F800000u ) == 0x7F800000u && ( bits & 0x007FFFFFu ) != 0 ) + { + const uint64_t sign = (uint64_t) ( bits >> 31 ) << 63; + const uint64_t payload = (uint64_t) ( bits & 0x007FFFFFu ) << 29; + const uint64_t nan_bits = sign | 0x7FF0000000000000ull | payload; + double d; memcpy( &d, &nan_bits, 8 ); return d; + } + float f; memcpy( &f, &bits, 4 ); return (double) f; +} + +// ILL-FORMED TEXT IS DAMAGE (docs/SPEC-TABLES.md §3, §4): a kind 12 payload is +// well-formed UTF-8 with no zero byte among its bytes, checked AS IT ARRIVES +// and before the reader's own bound, because a payload that is not text is not +// text at whatever length the reader would have kept. Rejects a zero byte, a +// truncated sequence, a bare continuation, an overlong encoding, a surrogate +// and a code point past U+10FFFF, which is SPEC.md §4.7's rule in this wire's +// idiom: the field reads its declared default, one malformed counts, and the +// parent reads on past L. +// +// A LENGTH IS A 64-BIT NUMBER (§3), so it arrives as one: a payload length is +// whatever the wire spelled, and narrowing it to a signed count would read +// 0xFFFFFFFFFFFFFFFF as an empty payload. +inline bool TableUtf8Valid( const uint8_t * bytes, uint64_t length ) +{ + uint64_t i = 0; + while ( i < length ) + { + const uint8_t lead = bytes[i]; + uint64_t continuations; + uint32_t code_point; + if ( lead == 0 ) { return false; } + if ( lead < 0x80 ) { i++; continue; } + else if ( ( lead & 0xE0 ) == 0xC0 ) { continuations = 1; code_point = lead & 0x1F; } + else if ( ( lead & 0xF0 ) == 0xE0 ) { continuations = 2; code_point = lead & 0x0F; } + else if ( ( lead & 0xF8 ) == 0xF0 ) { continuations = 3; code_point = lead & 0x07; } + else { return false; } + if ( i + continuations >= length ) { return false; } + for ( uint64_t k = 1; k <= continuations; k++ ) + { + if ( ( bytes[i + k] & 0xC0 ) != 0x80 ) { return false; } + code_point = ( code_point << 6 ) | uint32_t( bytes[i + k] & 0x3F ); + } + if ( continuations == 1 && code_point < 0x80 ) { return false; } + if ( continuations == 2 && ( code_point < 0x800 || ( code_point >= 0xD800 && code_point <= 0xDFFF ) ) ) { return false; } + if ( continuations == 3 && ( code_point < 0x10000 || code_point > 0x10FFFF ) ) { return false; } + i += 1 + continuations; + } + return true; +} + +// A CLAMP CUTS AT A CODE POINT BOUNDARY (§3, §16.2): the last whole code point +// that fits within the bound, over a payload the check above already accepted, +// so a clamp can never invent ill-formed storage. +// +// THE ANSWER IS NEVER ABOVE THE BOUND. The length arrives as the wire's own +// 64-bit number and the caller turns the answer back into the size of a copy, +// so a length no reader could have bounded has to leave here bounded: taken as +// a signed count, 0xFFFFFFFFFFFFFFFF is -1, -1 is under every bound, and the +// copy would run at SIZE_MAX. +inline int64_t TableUtf8Clamp( const uint8_t * bytes, uint64_t length, int64_t bound ) +{ + if ( length <= (uint64_t) bound ) { return (int64_t) length; } + int64_t cut = bound; + while ( cut > 0 && ( bytes[cut] & 0xC0 ) == 0x80 ) { cut--; } + return cut; +} + +// ONE CODE UNIT off the wire: two bytes LITTLE-ENDIAN, this wire's order for +// every fixed-width number (docs/SPEC-TABLES.md §3). No unit can exceed +// 0xFFFF, because two bytes cannot spell one. +inline uint16_t TableUtf16Unit( const uint8_t * bytes, int64_t index ) +{ + return uint16_t( uint16_t( bytes[index * 2] ) | ( uint16_t( bytes[index * 2 + 1] ) << 8 ) ); +} + +// ILL-FORMED WIDE TEXT IS DAMAGE (docs/SPEC-TABLES.md §3, §4): a kind 33 +// payload carrying an UNPAIRED SURROGATE or a ZERO CODE UNIT among its units, +// checked AS IT ARRIVES and before the reader's own bound, on the rule kind 12 +// takes for UTF-8. An ODD L is framing damage and the caller rejects it ahead +// of this, because units is L / 2. SPEC.md §4.12 refuses the same content +// TERMINALLY on the packet wire; here the field reads its declared default, +// one malformed counts, and the parent reads on past L. +inline bool TableUtf16Valid( const uint8_t * bytes, int64_t units ) +{ + int64_t i = 0; + while ( i < units ) + { + const uint16_t unit = TableUtf16Unit( bytes, i ); + if ( unit == 0 ) { return false; } + if ( unit >= 0xD800 && unit <= 0xDBFF ) + { + if ( i + 1 >= units ) { return false; } // a high surrogate with no low half + const uint16_t low = TableUtf16Unit( bytes, i + 1 ); + if ( low < 0xDC00 || low > 0xDFFF ) { return false; } + i += 2; + continue; + } + if ( unit >= 0xDC00 && unit <= 0xDFFF ) { return false; } // a low surrogate first + i++; + } + return true; +} + +// A CLAMP CUTS AT A CODE UNIT BOUNDARY AND NEVER SPLITS A PAIR (§3, §16.2): +// the first bound units of a payload the check above already accepted, and +// where the last kept unit is a HIGH SURROGATE whose low half did not fit, +// that unit is dropped with it. So a clamp can never invent an unpaired +// surrogate, exactly as kind 12's clamp can never invent a broken sequence. +inline int64_t TableUtf16Clamp( const uint8_t * bytes, int64_t units, int64_t bound ) +{ + if ( units <= bound ) { return units; } + int64_t cut = bound; + if ( cut > 0 ) + { + const uint16_t last = TableUtf16Unit( bytes, cut - 1 ); + if ( last >= 0xD800 && last <= 0xDBFF ) { cut--; } + } + return cut; +} + +// The RESERVED node-table id, the one id the language holds back +// (docs/SPEC-TABLES.md §3.1, §5). It rides in every unit, pointered or not, +// because every body has to know that a NESTED body claiming one is damaged. +static const uint64_t kTableNodeTableFieldId = 0xFFFFFFFFFFFFFFFFull; + +// TableWireForm is the FORM BYTE, and it is the whole header +// (docs/SPEC-TABLES.md §3). A reader that meets a byte it does not know +// refuses the wire by name and never reports damage. +const uint8_t kTableWireForm = 1; + +// TableOpen reads the form byte and the trailer, in that order, and hands back +// the ROOT BODY. It answers one of three verdicts, because five zero counters +// and a false flag are what a clean read prints too: +// +// TableOpenOk the form is known and the table read whole +// TableOpenRefused a FORM BYTE this reader does not carry: nothing is +// decoded, nothing is counted, and no damage is reported +// TableOpenDamaged a table that cannot be read whole — fewer than eight +// bytes, a count whose entries run past the front of the +// file, a count that leaves no room for the form byte, or +// ONE ID IN TWO ENTRIES. The whole wire is malformed, +// nothing is decoded, and one event is counted. +// TableOpenBodyStopped the form and the table were good and the ROOT BODY +// could not be walked to its own terminator. What it +// decoded before that is kept, as everywhere on this wire. +enum TableOpenVerdict { TableOpenOk, TableOpenRefused, TableOpenDamaged, TableOpenBodyStopped }; + +inline TableOpenVerdict TableOpen( const uint8_t * buffer, int64_t bytes, TableIdTable & table, int64_t & body_bytes ) +{ + if ( bytes < 1 ) { return TableOpenDamaged; } + if ( buffer[0] != kTableWireForm ) { return TableOpenRefused; } + if ( bytes < 9 ) { return TableOpenDamaged; } + const uint8_t * tail = buffer + bytes - 8; + uint64_t lo = uint64_t( tail[0] ) | uint64_t( tail[1] ) << 8 | uint64_t( tail[2] ) << 16 | uint64_t( tail[3] ) << 24; + uint64_t hi = uint64_t( tail[4] ) | uint64_t( tail[5] ) << 8 | uint64_t( tail[6] ) << 16 | uint64_t( tail[7] ) << 24; + uint64_t count = lo | ( hi << 32 ); + if ( count > (uint64_t) ( bytes / 8 ) ) { return TableOpenDamaged; } + const int64_t span = (int64_t) count * 8 + 8; + if ( span + 1 > bytes ) { return TableOpenDamaged; } + table.entries = buffer + bytes - span; + table.count = (int64_t) count; + // THE ENTRIES ARE DISTINCT: a table that carries one id twice is malformed + // for the whole wire, because no wire this schema writes carries a repeat + // and it would leave one more shape of table for a hostile writer to aim + // at (docs/SPEC-TABLES.md §3). + for ( int64_t i = 1; i < table.count; i++ ) + { + const uint64_t id = table.at( uint64_t( i ) + 1 ); + for ( int64_t j = 0; j < i; j++ ) + { + if ( table.at( uint64_t( j ) + 1 ) == id ) { return TableOpenDamaged; } + } + } + body_bytes = bytes - span - 1; + return TableOpenOk; +} + +// TableBodyExtent walks a body's framing to the zero reference that ends it, +// so a reader can tell a body that ENDED EARLY — leaving bytes no field claims +// — from one that is merely damaged. ANY BYTE BETWEEN THE ROOT'S TERMINATOR +// AND THE TABLE'S FIRST ENTRY IS MALFORMED, because no field claims it and the +// two ends of the file have met (docs/SPEC-TABLES.md §3). +inline bool TableBodyEndsEarly( const uint8_t * body, int64_t bytes, const TableIdTable & table ) +{ + TableReport ignored; + TableReader r( body, bytes, &ignored, &table ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.getleb( ref ) ) { return false; } + if ( ref == 0 ) { return r.offset != bytes; } + if ( ref > (uint64_t) table.count ) { return false; } + if ( !r.has( 1 ) ) { return false; } + if ( !r.skip( r.get8() ) ) { return false; } + } +} + +// THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3): a batch of BITPACKED bodies +// under one announced vocabulary. +// +// A form 2 wire is THREE PARTS: the form byte, the body count, and the bodies +// as one continuous bit stream, zero-padded to the next byte at the end and +// nowhere else. A body is a sequence of fields, each a REFERENCE followed by a +// PAYLOAD and nothing else: no kind byte and no length, because the +// announcement carries the kind and the shape of every entry. +const uint8_t kTableWireMessageForm = 2; + +// THE COUNT IS A RANGED INTEGER OVER [1, 256], eight bits carrying M - 1. 256 +// is a WIRE CONSTANT of this form rather than a receiver's policy, because the +// count's WIDTH depends on it and two peers that disagreed on the width would +// not be reading the same wire. A batch of zero is not spellable. +static const int64_t kTableMessageBatchMax = 256; + +// The RESERVED ids of the announcement's own two fields (§5, §11), beside the +// node table's. They are the announcement's transport, they never appear in a +// body, and they take no slot in the vocabulary. +static const uint64_t kTableBuildVersionFieldId = 0xFFFFFFFFFFFFFFFEull; +static const uint64_t kTableMessageVocabularyFieldId = 0xFFFFFFFFFFFFFFFDull; + +// THE WIDEST COUNT THIS FORM SPELLS, which is the count an UNBOUNDED array +// announces (§2.9): an unbounded array states no bound, so the announcement +// states the widest one a batch could carry. It is the ceiling an array's or a +// keyed entry's announced min and max are checked against. +static const uint64_t kTableMessageListMax = 0xFFFFFFFFull; + +// THIS UNIT'S OWN REFERENCE WIDTH: the bits a writer spends on every reference +// of every body it writes, which is a compile-time constant because the +// vocabulary is. A READER spends the width the SENDER's vocabulary settles. +static const int64_t kTableMessageRefBitsHere = 7; + +// THIS UNIT'S OWN ENTRY COUNT, which is the CAPACITY a receiver declares for +// its resolved vocabulary when it talks only to peers of this schema (§3.3). +// The vocabulary is a pure function of the build version, so a peer at this +// build announces exactly this many entries; a receiver that means to meet +// OTHER builds declares more, and an announcement above whatever it declared +// is refused as vocabulary_too_large. +static const int64_t kTableMessageEntriesHere = 75; + +// The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A +// pointered body names the node table through it, and the node table is the +// ROOT body's FIRST field because a pointer index's width is settled by the +// node count it carries. +static const uint64_t kTableNodeTableFieldSlot = 54; + +// THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own +// layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, +// so a value written here and a value written by a generated packet writer are +// the same bits in the same places. + +// EIGHT BYTES OF THE STREAM AS ONE WORD, and the word is LITTLE-END-FIRST +// whatever order this host is in, because the stream's own definition puts +// bit i in byte i/8: byte 0 of the run holds the word's low eight bits. That +// is what lets one value of any width move in one unaligned load or store +// instead of one touch a byte, and the BITS ON THE WIRE do not move. +inline uint64_t table_message_byteswap64( uint64_t v ) +{ + return ( v >> 56 ) | ( ( v >> 40 ) & 0xff00ull ) | ( ( v >> 24 ) & 0xff0000ull ) | ( ( v >> 8 ) & 0xff000000ull ) + | ( ( v << 8 ) & 0xff00000000ull ) | ( ( v << 24 ) & 0xff0000000000ull ) | ( ( v << 40 ) & 0xff000000000000ull ) + | ( v << 56 ); +} + +inline uint64_t table_message_load64( const uint8_t * p ) +{ + uint64_t v = 0; + memcpy( &v, p, 8 ); +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + v = table_message_byteswap64( v ); +#endif + return v; +} + +inline void table_message_store64( uint8_t * p, uint64_t v ) +{ +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + v = table_message_byteswap64( v ); +#endif + memcpy( p, &v, 8 ); +} + +struct TableBitWriter +{ + uint8_t * buffer; + int64_t capacity; // bytes + int64_t bits; + bool overflow; + + TableBitWriter() : buffer( NULL ), capacity( 0 ), bits( 0 ), overflow( false ) {} + TableBitWriter( uint8_t * to_buffer, int64_t to_capacity ) : buffer( to_buffer ), capacity( to_capacity ), bits( 0 ), overflow( false ) {} + + // ONE WORD AT A TIME, never one bit and never one byte of arithmetic: the + // value is shifted into place in a REGISTER once, and the bytes it + // occupies are stored from that register with no read back. A sixty-four + // bit field costs one shift rather than nine masked read-modify-writes. + // The word is assembled little-end-first, so the BITS ON THE WIRE are the + // same bits in the same places, bit i in byte i/8 at position i%8 with the + // low bit first, which is what the pinned goldens hold. IT WRITES EXACTLY + // THE BYTES THE VALUE OCCUPIES and never one past them, so a caller's + // buffer beyond the batch is its own. + void put( uint64_t value, int64_t n ) + { + if ( n <= 0 ) { return; } + if ( ( bits + n + 7 ) / 8 > capacity ) { overflow = true; bits += n; return; } + if ( n < 64 ) { value &= ( uint64_t( 1 ) << n ) - 1; } // a caller's high bits never leak + const int64_t index = bits >> 3; + const int64_t bit = bits & 7; + // the byte the write STARTS in keeps the bits already written to it, + // and every byte after it is this value's own + const uint64_t head = bit != 0 ? ( uint64_t( buffer[index] ) & ( ( uint64_t( 1 ) << bit ) - 1 ) ) : 0; + const uint64_t word = head | ( value << bit ); + const int64_t need = ( bit + n + 7 ) >> 3; // 1 to 9 bytes + if ( need >= 8 ) + { + table_message_store64( buffer + index, word ); + if ( need > 8 ) { buffer[index + 8] = uint8_t( value >> ( 64 - bit ) ); } + } + else + { + for ( int64_t i = 0; i < need; i++ ) { buffer[index + i] = uint8_t( word >> ( 8 * i ) ); } + } + bits += n; + } + + // THE ALIGN IS WHAT BUYS THIS (docs/SPEC-TABLES.md §3.3): a string(N), a + // bytes(N) and a blob record align before their bytes precisely so the + // largest payload on the wire moves as ONE memcpy. Off a boundary there is + // nothing to memcpy and the bytes go through put. + void putbytes( const uint8_t * data, int64_t n ) + { + if ( n <= 0 ) { return; } + if ( ( bits & 7 ) == 0 ) + { + if ( ( bits >> 3 ) + n > capacity ) { overflow = true; bits += n * 8; return; } + memcpy( buffer + ( bits >> 3 ), data, (size_t) n ); + bits += n * 8; + return; + } + for ( int64_t i = 0; i < n; i++ ) { put( (uint64_t) data[i], 8 ); } + } + + // a string's or a bytes' payload ALIGNS before its bytes, and a batch + // aligns once at its end. Both are zero fill, spent in one call. + void align() { put( 0, ( 8 - ( bits & 7 ) ) & 7 ); } +}; + +// TableAlignBits is what an align costs from a bit position, which a measure +// spends exactly where a save does. +inline int64_t TableAlignBits( int64_t bits ) { return ( 8 - ( bits % 8 ) ) % 8; } + +struct TableBitReader +{ + const uint8_t * buffer; + int64_t bits; // the stream's extent, in bits + int64_t offset; // bits consumed + + TableBitReader() : buffer( NULL ), bits( 0 ), offset( 0 ) {} + TableBitReader( const uint8_t * from_buffer, int64_t from_bytes ) : buffer( from_buffer ), bits( from_bytes * 8 ), offset( 0 ) {} + + bool has( int64_t n ) const { return n >= 0 && offset + n <= bits; } + + // the primitive is sixty-four bits, and a width above it is refused + // here as well as at the announcement: no field on any body can ask this + // reader to move more bits than it holds + // ONE WORD OF THE BUFFER AT A TIME, the mirror of the writer's put: the + // eight bytes the value starts in load as one little-end-first word and a + // ninth byte carries the spill a value that straddles the word needs. + // Within nine bytes of the stream's end there is no room for a word load + // and the bytes come one at a time, by the same arithmetic. + bool get( uint64_t & value, int64_t n ) + { + if ( n > 64 || !has( n ) ) { return false; } + value = 0; + if ( n == 0 ) { return true; } + const int64_t index = offset >> 3; + const int64_t bit = offset & 7; + const int64_t bytes = ( bits + 7 ) >> 3; + if ( index + 9 <= bytes ) + { + uint64_t v = table_message_load64( buffer + index ) >> bit; + if ( bit != 0 && bit + n > 64 ) { v |= uint64_t( buffer[index + 8] ) << ( 64 - bit ); } + value = n == 64 ? v : ( v & ( ( uint64_t( 1 ) << n ) - 1 ) ); + offset += n; + return true; + } + int64_t got = 0; + while ( got < n ) + { + const int64_t byte = offset >> 3; + const int64_t off = offset & 7; + const int64_t room = 8 - off; + const int64_t take = ( n - got ) < room ? ( n - got ) : room; + const uint64_t chunk = ( uint64_t( buffer[byte] ) >> off ) & ( ( uint64_t( 1 ) << take ) - 1 ); + value |= chunk << got; + offset += take; + got += take; + } + return true; + } + + // the bytes of an ALIGNED payload, which is the read side of the memcpy + // the align buys (docs/SPEC-TABLES.md §3.3) + bool getbytes( uint8_t * out, int64_t n ) + { + if ( n < 0 || !has( n * 8 ) ) { return false; } + if ( ( offset & 7 ) == 0 ) + { + memcpy( out, buffer + ( offset >> 3 ), (size_t) n ); + offset += n * 8; + return true; + } + for ( int64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !get( by, 8 ) ) { return false; } + out[i] = (uint8_t) by; + } + return true; + } + + bool skip( int64_t n ) { if ( !has( n ) ) { return false; } offset += n; return true; } + + // the pad to the next byte boundary is VERIFIED ZERO, which is the packet + // wire's rule for the same reason (SPEC.md §4.3) + bool align() + { + const int64_t pad = ( 8 - ( offset & 7 ) ) & 7; + if ( pad == 0 ) { return true; } + uint64_t bits_read = 0; + return get( bits_read, pad ) && bits_read == 0; + } +}; + +// TableBitsRequired is bits_required( min, max ): the bit length of max - min, +// and zero where the two are equal, which is a value that spends no bit at all. +inline int64_t TableBitsRequired( int64_t min, int64_t max ) +{ + if ( max <= min ) { return 0; } + uint64_t span = (uint64_t) ( max - min ); + int64_t n = 0; + while ( span > 0 ) { n++; span >>= 1; } + return n; +} + +// THE ANNOUNCED ENTRY (§3.3): an id, a kind, and a SHAPE, which is the width +// and range facts a reader needs to SKIP a field exactly and to DECODE one +// whose own declaration has moved. One name may take TWO entries, at two kinds or two +// shapes, and a body names the one it means. +// +// The ELEMENT's own facts ride beside the field's because an array's element +// is the one nesting this wire has: an array of arrays is not a table-wire +// construct, so one level is every level. +// +// IT IS THE RESOLVED ENTRY AND THE CALLER SIZES AN ARRAY OF THEM, so it +// carries what a DECODE takes and nothing a decode does not: qmin, qdelta and +// qcount are what SPEC.md §4.3's rule leaves behind, and the qmax and qres +// that rule CONSUMES are locals of the parse. The widths are int16 because a +// width is bounded by the kind it came under and no kind holds more than 128 +// bits. +struct TableMessageEntry +{ + uint64_t id = 0; + int64_t min = 0; // an array's minimum count + int64_t max = 0; // an array's maximum count, a string's capacity, a keyed array's slots + int64_t base_lo = 0; // the ranged base, low half: a signed kind's sign-extends, an unsigned kind's is whole + int64_t base_hi = 0; // its high half, for a 128-bit kind + int64_t elem_max = 0; + int64_t elem_base_lo = 0; + int64_t elem_base_hi = 0; + // what SPEC.md §4.3's derivation leaves: the base, the step and the count + float qmin = 0.0f; + float qdelta = 0.0f; + uint32_t qcount = 0; + float elem_qmin = 0.0f; + float elem_qdelta = 0.0f; + uint32_t elem_qcount = 0; + // THE PAYLOAD'S WIDTH, RESOLVED: what the kind, the packing and the + // announced bits together say, computed once at AnnounceRead, and -1 + // where the payload is not a fixed-width value at all + int16_t value_bits = -1; + int16_t elem_value_bits = -1; + uint8_t kind = 0; + uint8_t packing = 0; + uint8_t elem_kind = 0; + uint8_t elem_packing = 0; +}; + +// TableMessageEntrySame reports whether two RESOLVED entries carry the same +// shape, which is every fact of the entry but its id and its kind. It is what +// the announcement's duplicate rule is asked in: two entries that agree on all +// three parts are malformed (§3.3). +inline bool TableMessageEntrySame( const TableMessageEntry & a, const TableMessageEntry & b ) +{ + return a.min == b.min && a.max == b.max && a.base_lo == b.base_lo && a.base_hi == b.base_hi + && a.elem_max == b.elem_max && a.elem_base_lo == b.elem_base_lo && a.elem_base_hi == b.elem_base_hi + && a.qmin == b.qmin && a.qdelta == b.qdelta && a.qcount == b.qcount + && a.elem_qmin == b.elem_qmin && a.elem_qdelta == b.elem_qdelta && a.elem_qcount == b.elem_qcount + && a.value_bits == b.value_bits && a.elem_value_bits == b.elem_value_bits + && a.packing == b.packing && a.elem_kind == b.elem_kind && a.elem_packing == b.elem_packing; +} + +// TableMessageKindBits is the widest RANGED value a kind can carry, its own +// storage width: a width above it is a hostile width on the announcement. +inline int64_t TableMessageKindBits( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 6: case 20: case 25: return 8; + case 3: case 7: case 21: case 26: return 16; + case 4: case 8: case 22: case 27: return 32; + case 5: case 9: case 23: case 28: return 64; + default: return 128; + } +} + +// TableMessageQuantization is SPEC.md §4.3's derivation over an announced +// triple, in float32 and by nothing else: delta, the step count and the +// width. False is a triple SPEC.md calls non-conforming, which on the +// announcement is a hostile width like any other (§3.3). +inline bool TableMessageQuantization( float qmin, float qmax, float qres, float & delta, uint32_t & count, int64_t & bits ) +{ + if ( !( qmin < qmax ) || !( qres > 0.0f ) ) { return false; } + delta = qmax - qmin; + float values = delta / qres; + if ( !( delta - delta == 0.0f ) || !( values - values == 0.0f ) ) { return false; } // Inf - Inf is NaN + if ( !( values >= 1.0f ) ) { values = 1.0f; } + else if ( values > 4294967040.0f ) { values = 4294967040.0f; } // the largest float below 2^32 + count = (uint32_t) values; + if ( (float) count < values ) { count++; } // ceil, on a value the cast holds exactly + bits = TableBitsRequired( 0, (int64_t) count ); + return true; +} + +// The two roundings on each side of the rule (SPEC.md §7.2): the product +// rounds to float32 BEFORE the add, which a compiler permitted to contract +// would otherwise fuse into one rounding and move the wire. +#if ( defined( __GNUC__ ) || defined( __clang__ ) ) && ( defined( __aarch64__ ) || defined( _M_ARM64 ) ) +#define TABLE_FLOAT_FORCE_ROUND( x ) __asm__ ( "" : "+w" ( x ) ) +#elif ( defined( __GNUC__ ) || defined( __clang__ ) ) && ( defined( __x86_64__ ) || defined( __i386__ ) ) +#define TABLE_FLOAT_FORCE_ROUND( x ) __asm__ ( "" : "+x" ( x ) ) +#else +#define TABLE_FLOAT_FORCE_ROUND( x ) do { volatile float table_float_force_round_slot = ( x ); ( x ) = table_float_force_round_slot; } while ( 0 ) +#endif + +// TableMessageQuantize is the writer's half: the index a value takes. +inline uint32_t TableMessageQuantize( float value, float qmin, float delta, uint32_t count ) +{ + float normalized = ( value - qmin ) / delta; + if ( !( normalized >= 0.0f ) ) { normalized = 0.0f; } + else if ( !( normalized <= 1.0f ) ) { normalized = 1.0f; } + float scaled = normalized * (float) count; + TABLE_FLOAT_FORCE_ROUND( scaled ); + uint32_t index = (uint32_t) ( scaled + 0.5f ); // floor of a non-negative value + if ( index > count ) { index = count; } + return index; +} + +// TableMessageDequantize is the reader's half: the float an index names. +inline float TableMessageDequantize( uint32_t index, float qmin, float delta, uint32_t count ) +{ + if ( index > count ) { index = count; } + const float normalized = index / (float) count; + float scaled = normalized * delta; + TABLE_FLOAT_FORCE_ROUND( scaled ); + return scaled + qmin; +} + +inline bool TableMessageIntegerKind( uint8_t kind ) +{ + return ( kind >= 2 && kind <= 9 ) || kind == 18 || kind == 19; +} + +inline bool TableMessageFixedKind( uint8_t kind ) { return kind >= 20 && kind <= 29; } + +inline bool TableMessageKnownKind( uint8_t kind ) +{ + return kind == 0 || ( kind >= 1 && kind <= 17 ) || ( kind >= 18 && kind <= 29 ) || ( kind >= 30 && kind <= 33 ); +} + +// A CANONICAL LEB128, which is the announcement's own integer: the +// announcement is a form 1 FILE and takes §3's rule. +inline bool TableMessageLeb( const uint8_t * in, int64_t size, int64_t & at, uint64_t & value ) +{ + value = 0; + for ( int64_t shift = 0; at < size; shift += 7 ) + { + if ( shift >= 64 ) { return false; } + const uint8_t by = in[ at++ ]; + value |= uint64_t( by & 0x7F ) << shift; + if ( ( by & 0x80 ) == 0 ) { return !( shift > 0 && by == 0 ); } + } + return false; +} + +// TableMessageShapeFacts is where one shape's facts land: the field's own, +// or its element's, which is the one nesting this wire has. +struct TableMessageShapeFacts +{ + uint8_t & packing; int64_t & value_bits; int64_t & base_lo; int64_t & base_hi; + float & qmin; float & qmax; float & qres; float & qdelta; uint32_t & qcount; + int64_t & min; int64_t & max; uint8_t & elem_kind; +}; + +inline bool TableMessageShapeRead( const uint8_t * in, int64_t size, int64_t & at, uint8_t kind, TableMessageShapeFacts f ); +inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t value_bits ); + +// TableMessageEntryRead parses ONE entry, and answers false for a HOSTILE +// SHAPE: bits above the kind's own domain, an array whose min exceeds its +// max, an element kind outside the closed set, a quantized triple SPEC.md +// calls non-conforming, or a shape running past the vocabulary's own bytes. +inline bool TableMessageEntryRead( const uint8_t * in, int64_t size, int64_t & at, TableMessageEntry & entry ) +{ + if ( at + 9 > size ) { return false; } + entry = TableMessageEntry(); + for ( int i = 0; i < 8; i++ ) { entry.id |= uint64_t( in[ at + i ] ) << ( 8 * i ); } + entry.kind = in[ at + 8 ]; + at += 9; + if ( !TableMessageKnownKind( entry.kind ) ) { return false; } + // The parse lands in LOCALS and the entry keeps what a decode reads: the + // quantized max and res are the derivation's inputs and never a field's. + uint8_t packing = 0, elem_kind = 0; + int64_t bits = 0, base_lo = 0, base_hi = 0, min = 0, max = 0; + float qmin = 0.0f, qmax = 0.0f, qres = 0.0f, qdelta = 0.0f; + uint32_t qcount = 0; + TableMessageShapeFacts own = { packing, bits, base_lo, base_hi, + qmin, qmax, qres, qdelta, qcount, + min, max, elem_kind }; + if ( !TableMessageShapeRead( in, size, at, entry.kind, own ) ) { return false; } + entry.packing = packing; + entry.value_bits = (int16_t) TableMessageValueBits( entry.kind, packing, bits ); + entry.base_lo = base_lo; + entry.base_hi = base_hi; + entry.qmin = qmin; + entry.qdelta = qdelta; + entry.qcount = qcount; + entry.min = min; + entry.max = max; + entry.elem_kind = elem_kind; + if ( entry.kind == 14 || entry.kind == 16 ) + { + uint8_t elem_packing = 0, inner_kind = 0; + int64_t elem_bits = 0, elem_base_lo = 0, elem_base_hi = 0, elem_min = 0, elem_max = 0; + float elem_qmin = 0.0f, elem_qmax = 0.0f, elem_qres = 0.0f, elem_qdelta = 0.0f; + uint32_t elem_qcount = 0; + TableMessageShapeFacts elem = { elem_packing, elem_bits, elem_base_lo, elem_base_hi, + elem_qmin, elem_qmax, elem_qres, elem_qdelta, elem_qcount, + elem_min, elem_max, inner_kind }; + if ( !TableMessageShapeRead( in, size, at, entry.elem_kind, elem ) ) { return false; } + entry.elem_packing = elem_packing; + entry.elem_value_bits = (int16_t) TableMessageValueBits( entry.elem_kind, elem_packing, elem_bits ); + entry.elem_base_lo = elem_base_lo; + entry.elem_base_hi = elem_base_hi; + entry.elem_qmin = elem_qmin; + entry.elem_qdelta = elem_qdelta; + entry.elem_qcount = elem_qcount; + entry.elem_max = elem_max; + } + return true; +} + +// TableMessageShapeRead is one shape, by the kind that names it (§3.3's shape +// table). Every number in it is a canonical LEB128 except where the row says +// otherwise: a RANGED BASE IS ENCODED BY ITS KIND'S SIGNEDNESS, zigzag for the +// signed kinds, unsigned for the unsigned kinds and sixteen bytes for the +// 128-bit and fixed-point kinds, and a QUANTIZED f32 carries min, max and res +// as float32, from which the step count and the width derive by SPEC.md +// §4.3's rule and by nothing else. +inline bool TableMessageShapeRead( const uint8_t * in, int64_t size, int64_t & at, uint8_t kind, TableMessageShapeFacts f ) +{ + uint64_t v = 0; + if ( TableMessageIntegerKind( kind ) || TableMessageFixedKind( kind ) || kind == 10 ) + { + if ( at >= size ) { return false; } + f.packing = in[ at++ ]; + if ( f.packing == 0 ) { return true; } + if ( f.packing == 1 && kind != 10 ) + { + if ( !TableMessageLeb( in, size, at, v ) || (int64_t) v > TableMessageKindBits( kind ) ) { return false; } + f.value_bits = (int64_t) v; + if ( kind == 18 || kind == 19 || TableMessageFixedKind( kind ) ) + { + if ( at + 16 > size ) { return false; } + uint64_t lo = 0, hi = 0; + for ( int i = 0; i < 8; i++ ) { lo |= uint64_t( in[ at + i ] ) << ( 8 * i ); } + for ( int i = 0; i < 8; i++ ) { hi |= uint64_t( in[ at + 8 + i ] ) << ( 8 * i ); } + f.base_lo = (int64_t) lo; f.base_hi = (int64_t) hi; + at += 16; + return true; + } + if ( !TableMessageLeb( in, size, at, v ) ) { return false; } + if ( kind >= 2 && kind <= 5 ) { f.base_lo = (int64_t) ( v >> 1 ) ^ -(int64_t) ( v & 1 ); } // zigzag + else { f.base_lo = (int64_t) v; } // the unsigned domain, whole + return true; + } + if ( f.packing == 2 && kind == 10 ) + { + if ( at + 12 > size ) { return false; } + uint32_t raw[3] = { 0, 0, 0 }; + for ( int k = 0; k < 3; k++ ) { for ( int i = 0; i < 4; i++ ) { raw[k] |= uint32_t( in[ at + 4 * k + i ] ) << ( 8 * i ); } } + at += 12; + memcpy( &f.qmin, &raw[0], 4 ); + memcpy( &f.qmax, &raw[1], 4 ); + memcpy( &f.qres, &raw[2], 4 ); + return TableMessageQuantization( f.qmin, f.qmax, f.qres, f.qdelta, f.qcount, f.value_bits ); + } + return false; // a packing outside the closed set + } + // A MAX ABOVE WHAT THE KIND CAN HOLD IS A HOSTILE WIDTH (§3.3). A string + // and a wide string are bounded by the int32 storage cap the checker + // applies to every N (SPEC §4.3, §6.1), and an array and a keyed entry by + // the 32-bit count an unbounded array announces (§2.9), which is the + // widest count this form spells. A larger bound is a shape no conforming + // declaration can produce, and a reader that carried it would do its + // length arithmetic in a range that overflows. + if ( kind == 12 || kind == 33 ) + { + if ( !TableMessageLeb( in, size, at, v ) || v > (uint64_t) INT32_MAX ) { return false; } + f.max = (int64_t) v; + return true; + } + if ( kind == 14 || kind == 16 ) + { + if ( kind == 14 ) + { + if ( !TableMessageLeb( in, size, at, v ) || v > kTableMessageListMax ) { return false; } + f.min = (int64_t) v; + } + if ( !TableMessageLeb( in, size, at, v ) || v > kTableMessageListMax ) { return false; } + if ( (int64_t) v < f.min ) { return false; } + f.max = (int64_t) v; + if ( at >= size ) { return false; } + f.elem_kind = in[ at++ ]; + if ( !TableMessageKnownKind( f.elem_kind ) ) { return false; } + // AND AN ELEMENT KIND OF 12 OR 33 IS REFUSED HERE, at the + // announcement, rather than at the skip that would meet it (§3.3): no + // declaration this language accepts is an array of string(N) or of + // wstring(N), so a shape announcing one is one rule's business and not + // two. + if ( f.elem_kind == 12 || f.elem_kind == 33 ) { return false; } + return true; + } + return true; +} + +// TableMessageValueBits is one value's width under a shape, and -1 where the +// kind's payload is not a fixed-width value at all. +inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t value_bits ) +{ + if ( kind == 1 ) { return 1; } + if ( kind == 11 ) { return 64; } + if ( kind == 10 ) { return packing == 2 ? value_bits : 32; } + if ( TableMessageIntegerKind( kind ) || TableMessageFixedKind( kind ) ) + { + if ( packing == 1 ) { return value_bits; } + switch ( kind ) + { + case 2: case 6: case 20: case 25: return 8; + case 3: case 7: case 21: case 26: return 16; + case 4: case 8: case 22: case 27: return 32; + case 5: case 9: case 23: case 28: return 64; + default: return 128; + } + } + return -1; +} + +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an +// ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under +// the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 +// over element kind 6, and a trailer of those two reserved ids. +// +// THE VOCABULARY IS A FIELD AND NOT THE TRAILER, and that buys three things: +// §3's writer rule that an id no body references is never written is restored +// unbroken, an entry can carry a KIND and a SHAPE which a trailer of bare ids +// cannot, and one NAME can appear at two shapes. +// +// The order is the COOK PROJECTION's (§20.2): each record in the order the +// projection renders it and each record's fields in the order the projection +// renders them, then each enum's variants and each union's arms. Then comes +// the tail the projection does not name: the reserved node-table id, the three +// blob type ids as bytes, string and wstring, and every table's own name id in +// the projection's sorted record order. The tail is UNCONDITIONAL, so an +// ordinary edit only ever grows it at its end and never moves a slot a +// generated field header carries as a literal. +static const int64_t kTableAnnounceBytes = 901; +static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, + 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, + 0xe4, 0x7c, 0x11, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, + 0x20, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0d, 0xec, 0x10, + 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, + 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, + 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, + 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, + 0x07, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x21, 0x06, + 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0c, 0x10, 0x38, 0x81, + 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, + 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, +}; + +// TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries +// an announcement carried, RESOLVED ONCE, under one numbering. +// +// THE RECEIVER RESOLVES ONCE (§3.3), so this holds the entries themselves and +// not the announcement's bytes: every entry is parsed at AnnounceRead, and +// every body after it dispatches through ONE ARRAY INDEX with nothing to +// re-read and nothing to decide. The announcement is free the moment +// AnnounceRead returns. +// +// THE STORAGE IS THE CALLER'S and this library never allocates. The caller +// declares an array of entries wherever it wants it, static, on a heap, in an +// arena or beside its connection, and hands it here with its CAPACITY. The +// announcement holds for the life of the connection (§3.3), so the array does +// too, and a peer holds TWO for a connection, the one it writes with and the +// one it reads with. A restart opens a fresh connection with an empty +// vocabulary and nothing is cached across connections. +// +// kTableMessageEntriesHere is the capacity a receiver that talks only to peers +// of THIS schema declares, and a receiver meeting other builds declares more. +struct TableVocabulary +{ + // THE CONFORMING DEFAULT BYTE BOUND (§3.3). The ENTRY bound has no default + // because it IS the caller's capacity: an announcement naming more entries + // than the caller made room for is refused as vocabulary_too_large before + // an entry is touched, and the byte bound is read off the vocabulary + // field's own length before that. + static const int64_t kDefaultMaxBytes = 64 * 1024; + + TableVocabulary( TableMessageEntry * storage, int64_t capacity ) + : entries( storage ), max_entries( capacity ) {} + + TableMessageEntry * entries; // THE CALLER'S, capacity max_entries + int64_t max_entries; + int64_t count = 0; + int64_t ref_bits = 0; + uint64_t build_version = 0; + bool announced = false; + // REFUSAL IS TERMINAL (§3.3): a connection whose first announcement was + // refused, for any reason, carries no vocabulary for its life, and every + // announcement after it is refused as second_announcement + bool refused = false; + int64_t max_bytes = kDefaultMaxBytes; +}; + +// TableVocabularyEntryAt is the entry a reference names, counted from 1: ONE +// ARRAY INDEX into the caller's resolved storage, no parse and no branch. +inline const TableMessageEntry & TableVocabularyEntryAt( const TableVocabulary & vocabulary, uint64_t slot ) +{ + return vocabulary.entries[ slot - 1 ]; +} + +// AnnounceRead reads an announcement into one direction's vocabulary (§3.3). +// +// The announcement IS a file, so every malformed rule of §3 already covers it. +// Over its body there are EXACTLY TWO STRICT CHECKS: the BUILD VERSION +// present, exactly once, under kind 9, eight bytes wide, and the VOCABULARY +// present, exactly once, under kind 14 over element kind 6. Everything else is +// ordinary and tolerant, so an unknown field is skipped and counted and the +// announcement can GAIN a field in a later minor without a lockstep redeploy. +// +// The FIRST announcement sets the vocabulary and it is the only one that can. +// A SECOND is refused by name: it does not replace it, does not amend it and +// changes nothing. A refused announcement sets NO VOCABULARY, and the refusal +// is TERMINAL: every announcement after it, whether or not the first set +// anything, is second_announcement, so a peer holds no retry on the +// connection and cannot buy a second resolve by having its first refused. +inline bool AnnounceReadOnce( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * to ); + +inline bool AnnounceRead( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * to = report != NULL ? report : &ignored; + if ( vocabulary.announced || vocabulary.refused ) + { + to->refused = true; + to->reason = second_announcement; + return false; + } + const bool set = AnnounceReadOnce( vocabulary, buffer, bytes, to ); + if ( !set ) { vocabulary.refused = true; } + return set; +} + +inline bool AnnounceReadOnce( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * to ) +{ + if ( bytes < 1 ) { to->malformed = true; return false; } + if ( buffer[0] != kTableWireForm ) + { + to->refused = true; + to->reason = buffer[0] == kTableWireMessageForm ? message_form_as_file : newer_form; + return false; + } + if ( bytes < 9 ) { to->malformed = true; return false; } + TableIdTable table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( buffer, bytes, table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { to->malformed = true; } + else { to->refused = true; to->reason = newer_form; } + return false; + } + if ( TableBodyEndsEarly( buffer + 1, body_bytes, table ) ) { to->malformed = true; return false; } + TableReader r( buffer + 1, body_bytes, to, &table ); + uint64_t version = 0; + const uint8_t * words = NULL; + int64_t words_bytes = 0; + int32_t seen_version = 0, seen_vocabulary = 0; + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.getleb( ref ) ) { to->malformed = true; return false; } + if ( ref == 0 ) { break; } + if ( ref > (uint64_t) table.count || !r.has( 1 ) ) { to->malformed = true; return false; } + const uint64_t id = table.at( ref ); + const uint8_t kind = r.get8(); + if ( id == kTableBuildVersionFieldId ) + { + if ( kind != 9 || !r.has( 8 ) ) { to->malformed = true; return false; } + version = r.get64(); + // THE BUILD VERSION IS KEPT THE MOMENT IT IS READ, refusal or not, so + // that a refusal on this connection NAMES IT (§3.3). It is not the + // vocabulary, and a refused announcement still sets none. + vocabulary.build_version = version; + seen_version++; + continue; + } + if ( id == kTableMessageVocabularyFieldId ) + { + // kind 14 over element kind 6, which is §3's spelling for an + // opaque run of bytes + uint64_t framed = 0; + if ( kind != 14 || !r.getleb( framed ) || !r.has( (int64_t) framed ) ) { to->malformed = true; return false; } + const int64_t begin = r.offset, end = r.offset + (int64_t) framed; + r.offset = end; + if ( begin >= end || r.buffer[ begin ] != 6 ) { to->malformed = true; return false; } + int64_t at = begin + 1; + uint64_t length = 0; + if ( !TableMessageLeb( r.buffer, end, at, length ) || at + (int64_t) length != end ) { to->malformed = true; return false; } + if ( (int64_t) length > vocabulary.max_bytes ) { to->refused = true; to->reason = vocabulary_too_large; return false; } + words = r.buffer + at; + words_bytes = (int64_t) length; + seen_vocabulary++; + continue; + } + to->unknown++; + if ( !r.skip( kind ) ) { to->malformed = true; return false; } + } + if ( seen_version != 1 || seen_vocabulary != 1 ) { to->malformed = true; return false; } + + // THE ENTRIES, RESOLVED ONCE into the caller's storage (§3.3): every width + // is checked here and never again, and no body after this parses a byte of + // an announcement. An entry count above the caller's CAPACITY is refused + // by name before the entry is touched. + int64_t at = 0, count = 0, node_table_slots = 0; + while ( at < words_bytes ) + { + if ( count >= vocabulary.max_entries ) { to->refused = true; to->reason = vocabulary_too_large; return false; } + TableMessageEntry & parsed = vocabulary.entries[ count ]; + if ( !TableMessageEntryRead( words, words_bytes, at, parsed ) ) { to->malformed = true; return false; } + // THE RESERVED IDS WHERE THEY DO NOT BELONG (§3.3): the announcement's + // own two never take a slot, and the node-table id takes exactly one, + // so a vocabulary carrying either of the first or a SECOND node-table + // id is malformed whole and sets nothing + if ( parsed.id == kTableBuildVersionFieldId || parsed.id == kTableMessageVocabularyFieldId ) { to->malformed = true; return false; } + if ( parsed.id == kTableNodeTableFieldId ) { if ( node_table_slots++ > 0 ) { to->malformed = true; return false; } } + // A TRIPLE ALREADY PLACED IS NEVER PLACED TWICE, so two entries that + // agree on the id, the kind and every fact of the shape are malformed + // (§3.3): no writer this wire has produces one, and a reader that took + // it would carry two slots naming one thing. The scan is quadratic in + // the entry count, and the entry count is bounded above at 4096, so it + // is at most eight million compares on a path that runs ONCE a + // connection and never again. + for ( int64_t seen = 0; seen < count; seen++ ) + { + const TableMessageEntry & other = vocabulary.entries[ seen ]; + if ( other.id == parsed.id && other.kind == parsed.kind && TableMessageEntrySame( other, parsed ) ) { to->malformed = true; return false; } + } + count++; + } + vocabulary.count = count; + vocabulary.ref_bits = TableBitsRequired( 0, count ); + vocabulary.build_version = version; + vocabulary.announced = true; + return true; +} + +// TableMessageReserved is one of the three ids the language holds back (§3.1, +// §3.3, §5): each is malformed anywhere but its own transport, and the rule +// OUTRANKS the wrong-sort rule below. +inline bool TableMessageReserved( uint64_t id ) +{ + // THE THREE ARE THE TOP THREE VALUES a uint64 holds, so the test is ONE + // comparison: 0xFFFFFFFFFFFFFFFD, FE and FF and nothing else is at or + // above the vocabulary's own id, and a declaration hashing to any of them + // is refused by name (§11) + return id >= kTableMessageVocabularyFieldId; +} + +// TableMessageNameEntry resolves a reference used as a VALUE, which is an +// enum's variant, a keyed array's slot key or a node record's type id, and +// which must name a kind-0 entry (§3.3). A reference of 0 where an entry is required, one +// above E, one naming a reserved id and one naming an entry that carries a +// payload are each damage: the reader RESOLVED the entry and it contradicts +// the position it was used in, so the next bit's meaning is what is in doubt. +inline bool TableMessageNameEntry( const TableVocabulary & vocabulary, uint64_t ref, TableMessageEntry & entry ) +{ + if ( ref == 0 || ref > (uint64_t) vocabulary.count ) { return false; } + entry = TableVocabularyEntryAt( vocabulary, ref ); + return !TableMessageReserved( entry.id ) && entry.kind == 0; +} + +// TableMessageArmEntry resolves a UNION's arm reference, which must name an +// entry carrying the arm's own kind and shape: a kind-0 entry frames nothing, +// and a reserved id belongs to no arm (§3.3). +inline bool TableMessageArmEntry( const TableVocabulary & vocabulary, uint64_t ref, TableMessageEntry & entry ) +{ + if ( ref == 0 || ref > (uint64_t) vocabulary.count ) { return false; } + entry = TableVocabularyEntryAt( vocabulary, ref ); + return !TableMessageReserved( entry.id ) && entry.kind != 0; +} + +// TableMessageSkipVariant steps over an ENUM's variant reference on a SKIP +// path and RESOLVES it while it is there: 0 is None and the whole payload, and +// every other reference must name a kind-0 entry, because every reference +// above E is damage and one naming an entry that carries a payload +// contradicts the position it was used in, whether or not this reader was +// going to keep the value (§3.3). +inline bool TableMessageSkipVariant( TableBitReader & r, const TableVocabulary & vocabulary ) +{ + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + TableMessageEntry named; + return TableMessageNameEntry( vocabulary, ref, named ); +} +// TableMessageSkip steps over one field's payload without decoding it, using +// the announced ENTRY alone (§3.3). It is what makes an unknown entry +// skippable on a body with no kind byte, and it is ONE function over every +// table, because a shape says everything a skipper needs. +inline bool TableMessageSkipBody( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits ); +inline bool TableMessageSkip( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ); + +// TableMessageSkipElement steps over ONE element of an array or keyed entry +// by the element's own announced shape: a nested body to its zero reference, +// a variant or a node index at its reference width, a union arm by its own +// entry, and a fixed-width value at its bits. +inline bool TableMessageSkipElement( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ) +{ + switch ( entry.elem_kind ) + { + case 13: return TableMessageSkipBody( r, vocabulary, index_bits ); + case 30: return TableMessageSkipVariant( r, vocabulary ); + case 17: return index_bits > 0 && r.skip( index_bits ); + case 15: + { + TableMessageEntry inner; + inner.kind = 15; + return TableMessageSkip( r, vocabulary, index_bits, inner ); + } + default: + { + const int64_t elem = entry.elem_value_bits; + return elem >= 0 && r.skip( elem ); + } + } +} + +// TableMessageElementRunBits is the bits ONE element of an array or a keyed +// entry occupies on the SKIP path, where nothing is resolved and a run of them +// is one multiplication, and -1 where the element's width is its own +// content's. A ZERO is a real answer, and it is why this exists: a ranged +// element whose min equals its max rides no bits at all (§3.3). +inline int64_t TableMessageElementRunBits( const TableVocabulary & vocabulary, const TableMessageEntry & entry ) +{ + int64_t elem = 0; + switch ( entry.elem_kind ) + { + // a nested body, a union arm, an enum's variant and a node index each + // RESOLVE something, and a resolve that contradicts its position is + // damage this reader must still find, so they are walked + case 13: case 15: case 30: case 17: return -1; + default: elem = entry.elem_value_bits; break; + } + if ( elem < 0 ) { return -1; } + if ( entry.kind == 16 ) { elem += vocabulary.ref_bits; } // a keyed slot's own key reference + return elem; +} + +// TableMessageSkipRun steps over n elements of one fixed width in a single +// arithmetic step. A FIXED-WIDTH ELEMENT IS ARITHMETIC (§3.3), and a loop here +// would be the one superlinear thing in this form: a zero-width element under +// a count of 2^31 is six bytes of wire. +inline bool TableMessageSkipRun( TableBitReader & r, uint64_t n, int64_t width ) +{ + if ( width < 0 ) { return false; } + if ( width == 0 ) { return true; } + if ( n > (uint64_t) ( INT64_MAX / width ) ) { return false; } + return r.skip( (int64_t) ( n * (uint64_t) width ) ); +} + +inline bool TableMessageSkip( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ) +{ + switch ( entry.kind ) + { + case 0: case 32: return true; // a name, and a payload-free arm + case 30: return TableMessageSkipVariant( r, vocabulary ); + case 13: return TableMessageSkipBody( r, vocabulary, index_bits ); + case 17: return index_bits > 0 && r.skip( index_bits ); // a node index, at the width the body's node count settled + case 15: + { + uint64_t arm = 0; + if ( !r.get( arm, vocabulary.ref_bits ) ) { return false; } + if ( arm == 0 ) { return true; } + TableMessageEntry arm_entry; + if ( !TableMessageArmEntry( vocabulary, arm, arm_entry ) ) { return false; } + return TableMessageSkip( r, vocabulary, index_bits, arm_entry ); + } + case 12: + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() ) { return false; } + return r.skip( (int64_t) n * 8 ); + } + case 33: + { + // the length, NO align, then SIXTEEN bits a code unit (§3.3) + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + return r.skip( (int64_t) n * 16 ); + } + case 31: + { + // THE ESCAPE: align, a thirty-two bit L, then L bytes, opaque. It is + // the one path a later-major writer has on this form (§3.3) + uint64_t n = 0; + if ( !r.align() || !r.get( n, 32 ) ) { return false; } + return r.skip( (int64_t) n * 8 ); + } + case 14: case 16: + { + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, width ) ) { return false; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !r.align() ) { return false; } + // A RUN OF FIXED-WIDTH ELEMENTS IS ONE MULTIPLICATION (§3.3), and + // only an element whose width is its own content's is walked + const int64_t run = TableMessageElementRunBits( vocabulary, entry ); + if ( run >= 0 ) { return TableMessageSkipRun( r, n, run ); } + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 && !r.skip( vocabulary.ref_bits ) ) { return false; } + if ( !TableMessageSkipElement( r, vocabulary, index_bits, entry ) ) { return false; } + } + return true; + } + } + const int64_t width = entry.value_bits; + return width >= 0 && r.skip( width ); +} + +inline bool TableMessageSkipBody( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + if ( !TableMessageSkip( r, vocabulary, index_bits, TableVocabularyEntryAt( vocabulary, ref ) ) ) { return false; } + } +} + +// TableMessageNodeTableOpen reads the node table's opening when a body has +// one: the reserved id's reference and the count at thirty-two raw bits. A +// body whose first reference is anything else has no node table, and the +// reader is left where it was. False is damage: a reference past E, or bits +// that run out. +inline bool TableMessageNodeTableOpen( TableBitReader & r, const TableVocabulary & vocabulary, int64_t & count ) +{ + count = 0; + const int64_t at = r.offset; + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { r.offset = at; return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + if ( TableVocabularyEntryAt( vocabulary, ref ).id != kTableNodeTableFieldId ) { r.offset = at; return true; } + uint64_t n = 0; + if ( !r.get( n, 32 ) ) { return false; } + count = (int64_t) n; + return true; +} +// AnnounceMeasure is the announcement's byte count, which is a constant of the +// unit and not a walk. +inline int64_t AnnounceMeasure() { return kTableAnnounceBytes; } + +// Announce writes the announcement into the caller's buffer and answers the +// bytes written, which is exactly AnnounceMeasure's answer, or -1 when the +// buffer is too small. It allocates nothing and walks nothing. +inline int64_t Announce( uint8_t * buffer, int64_t capacity ) +{ + if ( buffer == NULL || capacity < kTableAnnounceBytes ) { return -1; } + memcpy( buffer, kTableAnnounce, (size_t) kTableAnnounceBytes ); + return kTableAnnounceBytes; +} + +// THE PRIMITIVE IS A BATCH (§3.3): a number of bodies of ONE ROOT in one +// buffer, one count and one continuous bit stream with no alignment between +// them. A single message is the batch of one. +// +// The count rides ahead of the bodies, so a writer declares it at Begin and +// End refuses a batch that wrote a different number: a count the bodies do not +// match is not a wire this writer will hand anyone. +struct TableMessageBatch +{ + TableBitWriter w; + int64_t declared = 0; + int64_t written = 0; +}; + +inline bool TableMessageBatchBegin( TableMessageBatch & batch, uint8_t * buffer, int64_t capacity, int64_t bodies ) +{ + if ( buffer == NULL || capacity < 1 || bodies < 1 || bodies > kTableMessageBatchMax ) { return false; } + buffer[0] = kTableWireMessageForm; // the FORM BYTE is read first, always + batch.w = TableBitWriter( buffer + 1, capacity - 1 ); + batch.declared = bodies; + batch.written = 0; + batch.w.put( (uint64_t) ( bodies - 1 ), 8 ); // a ranged integer over [1, 256] + return true; +} + +// TableMessageBatchEnd zero-fills to the next byte, the one alignment a batch +// spends at its end, and answers the whole batch's byte count, or -1. +inline int64_t TableMessageBatchEnd( TableMessageBatch & batch ) +{ + if ( batch.written != batch.declared || batch.w.overflow ) { return -1; } + batch.w.align(); + if ( batch.w.overflow ) { return -1; } + return 1 + batch.w.bits / 8; +} + +// TableMessageBatchBytes is a batch's byte count from its bodies' BIT count, +// which is what every MeasureMessages answers. +inline int64_t TableMessageBatchBytes( int64_t body_bits ) +{ + if ( body_bits < 0 ) { return -1; } + return 1 + ( 8 + body_bits + 7 ) / 8; +} + +// The reading half. A batch is opened once and its bodies are then read in +// order into the storage the caller sized for them: which root a batch carries +// is the APPLICATION's and never this wire's. +struct TableMessageBatchReader +{ + TableBitReader r; + const TableVocabulary * vocabulary = NULL; + TableReport * report = NULL; + int64_t remaining = 0; + // THE SINK A CALLER THAT PASSED NO REPORT WRITES INTO IS THE READER'S OWN, + // not a static: a static is shared mutable state, and two threads reading + // two batches without reports would be writing one object. LoadMessages + // already keeps its sink locally, for the same reason. + TableReport ignored; +}; + +// TableMessageBatchOpen answers the batch's body count, or -1 with the refusal +// on the report: a form byte this reader does not carry, or a body from a peer +// that never announced. +inline int64_t TableMessageBatchOpen( TableMessageBatchReader & br, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + br.report = report != NULL ? report : &br.ignored; + br.vocabulary = &vocabulary; + if ( bytes < 1 ) { br.report->malformed = true; return -1; } + if ( buffer[0] != kTableWireMessageForm ) { br.report->refused = true; br.report->reason = newer_form; return -1; } + if ( !vocabulary.announced ) { br.report->refused = true; br.report->reason = no_vocabulary; return -1; } + br.r = TableBitReader( buffer + 1, bytes - 1 ); + uint64_t count = 0; + if ( !br.r.get( count, 8 ) ) { br.report->malformed = true; return -1; } + br.remaining = (int64_t) count + 1; + return br.remaining; +} + +// TableMessageRefuseBatch is the batch's own refusal (§3.3): M above 256 on the +// write side, or above the caller's capacity on the read side. Nothing is +// written or decoded, no counter moves, and the reason names it. +inline void TableMessageRefuseBatch( TableReport * report ) +{ + if ( report == NULL ) { return; } + report->refused = true; + report->reason = batch_too_large; +} + +// TableMessageBatchClose verifies the trailing pad, and that NOTHING FOLLOWS +// IT: the batch ends at the pad to the byte boundary, and a buffer with bytes +// left over describes no batch this reader can name (§3.3). +inline bool TableMessageBatchClose( TableMessageBatchReader & br ) +{ + if ( br.remaining != 0 || !br.r.align() || br.r.offset != br.r.bits ) { br.report->malformed = true; return false; } + return true; +} + + +// An ENUM-KEYED array's storage: E.Max slots, ONE PER NAMED VARIANT, with the +// key k at index k-1 — the storage SHIFTS LEFT and nothing is stored for None. +// +// NOTHING OUTSIDE THE ARRAY NAMES ITS SIZE: the extent is derived from E::Max +// here and nowhere else, so there is no size parameter to spell and no count a +// consumer could put one out of step with. +// +// NONE IS THE NULL KEY: it names no slot, it never rides on the wire, a stored +// key of 0 is malformed, and INDEXING BY IT IS A PROGRAM ERROR IN EVERY +// CONFIGURATION — caught by operator[], which cannot see a runtime key any +// earlier, and REFUSED UNCONDITIONALLY. A KEY PAST Max IS THE SAME ERROR for +// the same reason — it names a variant this enum does not have — so the +// accessor refuses BOTH ENDS. NDEBUG does not remove the compare: +// there is NO UB PATH here in any build. ITERATION is still the surface a +// consumer of the whole array wants: begin()/end() walk every stored slot and +// yield the KEY, 1..E.Max, so a call site writes no bound, no cast, no shift +// and no None question. +template +struct TableKeyed +{ + // the extent is the enum's, derived here and named nowhere else + static constexpr int32_t kSlots = (int32_t) E::Max; + + T slots[kSlots] = {}; + + T & operator[]( E key ) + { + RefuseKey( key ); + return slots[ (int32_t) key - 1 ]; + } + const T & operator[]( E key ) const + { + RefuseKey( key ); + return slots[ (int32_t) key - 1 ]; + } + + // THE REFUSAL, and it stands in EVERY BUILD, AT BOTH ENDS. The storage + // holds one slot per NAMED variant: nothing for None below it and nothing + // above Max, so a build that skipped this compare would index one element + // BEFORE the array or past its end — undefined behavior in the + // configuration a game ships. Either key is a program error, so the + // accessor ends the program rather than reading something. The assert + // carries the message where a debugger can read it and NDEBUG removes + // that; the fatal is what stands after it. BOTH GO THROUGH THE HOOKS — + // define schema_assert and schema_fatal and this refusal lands in your + // own handler. + // + // ONE UNSIGNED COMPARE COVERS BOTH ENDS: the storage index is key - 1, and + // None's is -1, which wraps above kSlots unsigned. The cost is one + // perfectly-predicted compare, on a path that reads config. + static void RefuseKey( E key ) + { + if ( (uint32_t) ( (int32_t) key - 1 ) >= (uint32_t) kSlots ) + { + schema_assert( false && "an enum-keyed array holds one slot per named variant: None keys none, and neither does a key past Max" ); + schema_fatal(); + } + } + + // ---- iteration: keys 1..E.Max over storage 0..E.Max-1, key beside element ---- + // + // The entry is a key and a REFERENCE, handed out BY VALUE the way any + // proxy is: for ( auto [ key, element ] : keyed ) binds element to the + // reference member, so iterating fills the array as well as reads it. + // auto & [ key, element ] does NOT compile, and that is by design — a + // non-const lvalue reference cannot bind to the proxy. Write + // auto [ ... ], or auto && [ ... ] if you prefer the reference form. + // + // THE ITERATORS CARRY NO iterator_traits TYPEDEFS. They bought std::distance + // and the forward-pass algorithms for an audience that does not call them, + // and the they need is the single most expensive include the + // generated corpus had: 536 headers and 986 KB, in a header whose whole + // remaining set is 123. begin(), end() and size() need none of it. + + struct Entry { E key; T & element; }; + struct ConstEntry { E key; const T & element; }; + + struct Iterator + { + T * slots; + int32_t index; // the STORAGE index; the key it holds is index + 1 + Entry operator*() const { return Entry{ (E) ( index + 1 ), slots[index] }; } + Iterator & operator++() { index++; return *this; } + bool operator==( const Iterator & other ) const { return index == other.index; } + bool operator!=( const Iterator & other ) const { return index != other.index; } + }; + + struct ConstIterator + { + const T * slots; + int32_t index; // the STORAGE index; the key it holds is index + 1 + ConstEntry operator*() const { return ConstEntry{ (E) ( index + 1 ), slots[index] }; } + ConstIterator & operator++() { index++; return *this; } + bool operator==( const ConstIterator & other ) const { return index == other.index; } + bool operator!=( const ConstIterator & other ) const { return index != other.index; } + }; + + Iterator begin() { return Iterator{ slots, 0 }; } + Iterator end() { return Iterator{ slots, kSlots }; } + ConstIterator begin() const { return ConstIterator{ slots, 0 }; } + ConstIterator end() const { return ConstIterator{ slots, kSlots }; } +}; + +inline float table_bits_to_float( uint32_t bits ) { float f; memcpy( &f, &bits, 4 ); return f; } +inline uint32_t table_float_to_bits( float f ) { uint32_t b; memcpy( &b, &f, 4 ); return b; } +inline double table_bits_to_double( uint64_t bits ) { double d; memcpy( &d, &bits, 8 ); return d; } +inline uint64_t table_double_to_bits( double d ) { uint64_t b; memcpy( &b, &d, 8 ); return b; } + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_PRIMITIVES + +#ifndef MAPDEMO_SCHEMA_TABLE_ARENA +#define MAPDEMO_SCHEMA_TABLE_ARENA + +namespace mapdemo { + +// ---- variable-length tables: tuning constants (docs/SPEC-TABLES.md) ---- +// +// The segment size and the count multiply to exactly 2^32: the u32 reference +// is the arena's hard ceiling, and these constants saturate it rather than +// leaving address space unreachable. Slab handout costs one atomic per slab, +// so per-node allocation costs no synchronization at all. + +static const uint32_t kTableSegmentBits = 22; // 4 MiB segments +static const uint32_t kTableSegmentSize = 1u << kTableSegmentBits; +static const uint32_t kTableSegmentMask = kTableSegmentSize - 1u; +static const uint32_t kTableMaxSegments = 1u << ( 32 - kTableSegmentBits ); // 1024 -> 4 GiB +static const uint32_t kTableSlabBytes = 64u * 1024u; // one atomic per slab +static const uint32_t kTableAlign = 8; // every node starts 8-aligned +static const uint32_t kTableAllocFailed = 0xFFFFFFFFu; + +// ---- THE CALLER'S ALLOCATOR (docs/SPEC-TABLES.md §6.5) ---- +// +// Every allocation the variable-length runtime makes goes through one of +// these — the arena's segments, the pack walk's identity map, the numbering's +// entry array, the packed region, and the tool path's node directory. There is +// no other call to the C library on this path, so a counting allocator sees +// every byte and a game's own heap can own all of it. +// +// It is the shape TableBlockAllocator already has (§19.1): two function +// pointers and a context the caller carries. What it adds is a CONTRACT ON +// alloc — the bytes come back ZEROED. Lock copies whole nodes, PADDING +// INCLUDED, so anything left uninitialized reaches a packed region; the default +// pair reaches that through calloc, which costs nothing measurable because a +// fresh segment is untouched pages either way. +struct TableAllocator +{ + void * ( *alloc )( void * context, int64_t bytes ); // ZEROED bytes, NULL on failure + void ( *free )( void * context, void * pointer ); + void * context; +}; + +// The default pair, and it is the one every entry point takes when the caller +// names none. It calls schema_allocate / schema_release, so a program with its +// own C-library replacement can move the floor without writing a struct at all. +inline void * table_default_alloc( void * context, int64_t bytes ) { (void) context; return schema_allocate( bytes ); } +inline void table_default_free( void * context, void * pointer ) { (void) context; schema_release( pointer ); } + +inline TableAllocator TableDefaultAllocator() +{ + TableAllocator allocator; + allocator.alloc = table_default_alloc; + allocator.free = table_default_free; + allocator.context = NULL; + return allocator; +} + +// ---- TableRef: a relocatable reference (never a machine pointer) ---- +// +// Two encodings, one slot, and the FORM says which is in force: +// +// in the arena — the node's arena offset (segment index in the high bits) +// in a region — the SELF-RELATIVE byte delta from this slot's own address, +// so a deref is one add, needs no base pointer, and a whole +// region relocates by memcpy with zero fix-up +// +// 0 is null in both, and a slot can never name the node that contains it, so +// zero names nothing real in either form. +// +// A REGION DELTA HAS NO REQUIRED SIGN (§6.3). A region is packed depth-first, +// so a node's FIRST reference points forward; every LATER reference to that +// same node points BACK at the one body it already has, which is exactly what +// makes one node one node in a region. Sharing and a back-reference are the +// same fact, and nothing validates a reference by its sign. +// +// IT IS EIGHT BYTES, SIGNED, so ONE REGION REACHES EVERYTHING (§6.3, §7): a +// four-byte slot bounded a region at 2 GiB, and the scale a cook exists for is +// *"100mbs or many gigabytes of data in Assets.bin"*. +struct TableRef +{ + int64_t value = 0; + bool null() const { return value == 0; } +}; + +// TableSlot is what Alloc hands back: usable as the node pointer (write +// fields through it) AND as the reference to store in a pointer field. +template struct TableSlot +{ + T * ptr = NULL; + TableRef ref; + T * operator->() const { return ptr; } + T & operator*() const { return *ptr; } + operator T *() const { return ptr; } + operator TableRef() const { return ref; } + bool null() const { return ptr == NULL; } +}; + +inline uint32_t TableAlignUp( uint32_t bytes ) { return ( bytes + kTableAlign - 1 ) & ~( kTableAlign - 1 ); } +inline int64_t TableAlignUp64( int64_t bytes ) { return ( bytes + kTableAlign - 1 ) & ~( int64_t( kTableAlign ) - 1 ); } + +// ---- a BYTE BUFFER's node (docs/SPEC-TABLES.md §2.5, §6.3) ---- +// +// A *bytes or *string slot is a TableRef like every pointer slot, and it names +// a BLOB NODE: this eight-byte header and then the bytes, at offset eight so +// the data is eight-aligned. A *string blob carries one more zero byte after +// its data, so a region hands back a C string with no copy. The node's extent +// is the header plus its bytes, rounded to the arena's alignment like every +// node's; on the wire it is a record whose body is the bytes (§3.1). +struct TableBlob +{ + uint32_t length; + uint32_t zero; +}; + +static const int64_t kTableBlobHeader = 8; // length (u32), then four zero bytes +static const int64_t kTableBlobMaxLength = 0xFFFFFFFF; // a record's length is a u32 (§3.1) + +// the node's storage: the header, the bytes, a string's terminator, rounded +// to the arena's alignment like every node +inline int64_t TableBlobStorage( int64_t length, bool terminated ) +{ + return TableAlignUp64( kTableBlobHeader + length + ( terminated ? 1 : 0 ) ); +} + +// What a read answers: a pointer INTO the region and the length, NULL and +// zero for a null slot. Off a locked region, a loaded one or an opened cook +// the pointer is one add from the slot, and nothing is copied. +struct TableBytesView +{ + const uint8_t * data; + int64_t length; +}; + +struct TableStringView +{ + const char * data; // zero-terminated + int64_t length; +}; + +// What AllocBytes and AllocString hand back: the bytes to write through, the +// length asked for, and the reference to store in the slot — the three +// answers TableSlot gives for a table node. +struct TableBytesSlot +{ + uint8_t * data = NULL; + int64_t length = 0; + TableRef ref; + bool null() const { return data == NULL; } + operator TableRef() const { return ref; } +}; + +struct TableStringSlot +{ + char * data = NULL; // room for length bytes and the terminator, already zero + int64_t length = 0; + TableRef ref; + bool null() const { return data == NULL; } + operator TableRef() const { return ref; } +}; + +// ---- the arena: segmented, slab-handed, lock-free by ownership ---- +// +// Allocation is thread-local inside a worker's slab — no atomics on the node +// path. A worker takes its next slab with ONE compare-exchange, and a new +// segment is published with one more. Nothing ever moves: a segment, once +// allocated, lives untouched until the arena is torn down, so a T* obtained +// from Alloc stays valid while other workers allocate, and an offset stays +// correct while the arena grows. +// +// The model this DELIBERATELY refuses: one buffer under a lock, grown by +// realloc. A realloc moves the buffer under workers mid-write; offsets fix +// identity but not the raw references already resolved from them, and the +// resulting corruption is invisible until much later. Segments never move, so +// that bug class cannot be written here. +// +// Slack: at most one slab tail per worker plus one slab per segment (a slab +// that will not fit is skipped rather than split), i.e. under 2% of a segment +// plus threads x 64 KiB. That is the price of never synchronizing per node. +struct TableArena +{ + std::atomic segments[ kTableMaxSegments ]; + std::atomic cursor; // (segment << kTableSegmentBits) | bytes handed out + bool locked = false; // MONOTONIC: Lock() is one-way, there is no unlock + // THE ARENA CARRIES ITS OWN, so everything downstream of a builder — + // segments, pack map, numbering, region, node directory — allocates through + // the one pair the caller named, with nothing to thread by hand. + TableAllocator allocator; +}; + +inline void TableArenaInit( TableArena & arena, TableAllocator allocator ) +{ + for ( uint32_t i = 0; i < kTableMaxSegments; i++ ) + { + arena.segments[i].store( NULL, std::memory_order_relaxed ); + } + arena.cursor.store( 0, std::memory_order_relaxed ); + arena.locked = false; + arena.allocator = allocator; +} + +inline void TableArenaShutdown( TableArena & arena ) +{ + for ( uint32_t i = 0; i < kTableMaxSegments; i++ ) + { + uint8_t * segment = arena.segments[i].exchange( NULL, std::memory_order_acq_rel ); + if ( segment != NULL ) { arena.allocator.free( arena.allocator.context, segment ); } + } + arena.cursor.store( 0, std::memory_order_relaxed ); +} + +// one L1 load plus an add: the segment table is 8 KiB and stays hot +inline uint8_t * TableArenaAt( const TableArena & arena, uint32_t offset ) +{ + return arena.segments[ offset >> kTableSegmentBits ].load( std::memory_order_relaxed ) + ( offset & kTableSegmentMask ); +} + +// TableArenaGrabSlab hands one worker its next private slab. Returns +// kTableAllocFailed when the arena's address space or the allocator is +// exhausted — a loud refusal, never a silent smaller slab. +inline uint32_t TableArenaGrabSlab( TableArena & arena ) +{ + for ( ;; ) + { + uint32_t cursor = arena.cursor.load( std::memory_order_acquire ); + uint32_t segment = cursor >> kTableSegmentBits; + uint32_t used = cursor & kTableSegmentMask; + // strictly less: a slab is never split across segments, and the tail + // is the documented slack + if ( used + kTableSlabBytes < kTableSegmentSize ) + { + if ( arena.segments[segment].load( std::memory_order_acquire ) == NULL ) + { + // THE SEGMENT COMES BACK ZEROED, which is the allocator's + // contract and not an extra pass here: Lock copies whole nodes, + // PADDING INCLUDED, so anything uninitialized reaches a packed + // region. Value-initializing a node with placement new zeroes + // its MEMBERS and not its padding, so the zeroing has to happen + // at the segment or not at all. It costs nothing measurable: a + // fresh segment is untouched pages either way, and the default + // pair's calloc has the kernel hand them over zeroed. + uint8_t * memory = (uint8_t *) arena.allocator.alloc( arena.allocator.context, (int64_t) kTableSegmentSize ); + if ( memory == NULL ) { return kTableAllocFailed; } + uint8_t * expected = NULL; + if ( !arena.segments[segment].compare_exchange_strong( expected, memory, std::memory_order_acq_rel ) ) + { + // another worker published this segment first + arena.allocator.free( arena.allocator.context, memory ); + } + } + if ( arena.cursor.compare_exchange_weak( cursor, cursor + kTableSlabBytes, std::memory_order_acq_rel ) ) + { + return ( segment << kTableSegmentBits ) | used; + } + continue; + } + uint32_t next_segment = segment + 1; + if ( next_segment >= kTableMaxSegments ) { return kTableAllocFailed; } // 4 GiB: the u32 reference's ceiling + arena.cursor.compare_exchange_weak( cursor, next_segment << kTableSegmentBits, std::memory_order_acq_rel ); + } +} + +// TableArenaGrabSpan reserves a SPAN of the arena's address space for one node +// larger than a slab — a BYTE BUFFER of any size (docs/SPEC-TABLES.md §2.5) — +// and allocates it as one contiguous block. It takes whole segment indices +// from the cursor, starting at the index after the cursor's so nothing else +// is ever handed out inside the span, and publishes the block under the first +// of them; the indices the span covers past that one stay NULL, which is +// enough, because only a node's START is ever resolved through the segment +// table and a blob's bytes follow its header inside the one allocation. The +// unused tail of the segment the cursor was in is slack, like a slab tail. +// Returns kTableAllocFailed when the address space or the allocator is +// exhausted — a loud refusal, never a smaller blob. +inline uint32_t TableArenaGrabSpan( TableArena & arena, int64_t bytes ) +{ + if ( bytes <= 0 || bytes > ( (int64_t) kTableMaxSegments - 2 ) * (int64_t) kTableSegmentSize ) { return kTableAllocFailed; } + const uint32_t spanned = (uint32_t) ( ( bytes + kTableSegmentSize - 1 ) >> kTableSegmentBits ); + for ( ;; ) + { + uint32_t cursor = arena.cursor.load( std::memory_order_acquire ); + uint32_t start = ( cursor >> kTableSegmentBits ) + 1; + if ( start + spanned >= kTableMaxSegments ) { return kTableAllocFailed; } // 4 GiB: the u32 reference's ceiling + uint32_t next = ( start + spanned ) << kTableSegmentBits; + if ( !arena.cursor.compare_exchange_weak( cursor, next, std::memory_order_acq_rel ) ) { continue; } + // the span is this worker's now: nothing else can publish under its + // first index, so a plain store suffices, and the block comes back + // ZEROED like every segment — the blob's bytes and its tail are zeros + // until written + uint8_t * memory = (uint8_t *) arena.allocator.alloc( arena.allocator.context, bytes ); + if ( memory == NULL ) { return kTableAllocFailed; } + arena.segments[start].store( memory, std::memory_order_release ); + return start << kTableSegmentBits; + } +} + +// ---- TableWorker: one thread's allocation front ---- +// +// The threading contract, stated plainly: +// * Alloc on YOUR OWN worker is safe concurrently with any other worker's. +// No locks, no atomics per node. +// * Writing fields of a node ANOTHER worker allocated is your own +// synchronization problem — this runtime does not arbitrate it. +// * Lock and Save are single-threaded: call them after the workers have +// joined. +struct TableWorker +{ + TableArena * arena = NULL; + uint32_t next = 0; + uint32_t end = 0; + + template TableSlot Alloc() + { + static_assert( alignof( T ) <= kTableAlign, "a table node's alignment must fit the arena's" ); + TableSlot slot; + if ( arena == NULL || arena->locked ) { return slot; } + uint32_t bytes = TableAlignUp( (uint32_t) sizeof( T ) ); + if ( bytes > kTableSlabBytes ) { return slot; } // a node larger than a slab: refused, never split + if ( end == 0 || next + bytes > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return slot; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + uint32_t at = next; + next += bytes; + // A NODE IS BORN IN TWO HALVES: start its lifetime in the raw + // storage, then write the declared defaults ONE MEMBER AT A TIME. + // + // It is "T", not "T{}". Value-initialising the whole aggregate says + // the same thing and costs cl O(BYTES) TO COMPILE — it expands element + // by element in its front end — while both halves here cost + // O(declarations). The slab cap below refuses a large node at RUN + // TIME and bounds nothing at compile time: the cost is paid by + // whatever T a caller instantiates this with. + // Padding is not the difference: value-initialisation zeroes MEMBERS + // and not padding either way, which is why the segment is calloc'd. + // + // TableReset is an OVERLOAD SET, one per closure member, reached from + // this template by argument-dependent lookup on T's own namespace — + // Alloc is a template and cannot spell Reset. + // + // The reset is here because ONE DEFINITION SAYS WHAT THE DECLARED + // DEFAULTS ARE, and it is Reset. Default-initialisation lands on + // the same values today, because a member with a non-zero default + // carries a member initializer that says so — but that is the class + // definition agreeing with Reset, not the arena reading it, and #320's + // fix was itself a pass that MOVED initialisation between the two. + // The arena reads the definition. + slot.ptr = new ( TableArenaAt( *arena, at ) ) T; + TableReset( *slot.ptr ); + slot.ref.value = at; + return slot; + } + + // Alloc a BYTE BUFFER's node of exactly length bytes (docs/SPEC-TABLES.md + // §2.5): the blob header and its bytes, zeroed, in this thread's slab when + // it fits and in a span of the arena's own when it does not. NULL is the + // arena locked, a length below zero or past a record's u32, or the + // allocator refusing. The offset comes back for the reference. + TableBlob * AllocBlob( int64_t length, bool terminated, uint32_t & at ) + { + at = 0; + if ( arena == NULL || arena->locked ) { return NULL; } + if ( length < 0 || length > kTableBlobMaxLength ) { return NULL; } + const int64_t bytes = TableBlobStorage( length, terminated ); + if ( bytes > (int64_t) kTableSlabBytes ) + { + at = TableArenaGrabSpan( *arena, bytes ); + if ( at == kTableAllocFailed ) { at = 0; return NULL; } + } + else + { + if ( end == 0 || next + (uint32_t) bytes > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return NULL; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + at = next; + next += (uint32_t) bytes; + } + TableBlob * blob = (TableBlob *) TableArenaAt( *arena, at ); + blob->length = (uint32_t) length; // the bytes after it are the segment's zeros + blob->zero = 0; + return blob; + } + + // RAW, ZEROED storage of the bytes asked for, at the alignment asked for: a MAP's or a LIST's builder + // head and its segments (docs/SPEC-TABLES.md §2.8, §2.9). It is not a node: it carries + // no type id, takes no index and has no Reset, so it goes through the same + // slab and span the blob path uses rather than through Alloc. + uint8_t * AllocRaw( int64_t bytes, int64_t align, uint32_t & at ) + { + at = 0; + if ( arena == NULL || arena->locked ) { return NULL; } + if ( bytes <= 0 || align > (int64_t) kTableAlign ) { return NULL; } + const int64_t rounded = TableAlignUp64( bytes ); + if ( rounded > (int64_t) kTableSlabBytes ) + { + at = TableArenaGrabSpan( *arena, rounded ); + if ( at == kTableAllocFailed ) { at = 0; return NULL; } + return TableArenaAt( *arena, at ); + } + if ( end == 0 || next + (uint32_t) rounded > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return NULL; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + at = next; + next += (uint32_t) rounded; + return TableArenaAt( *arena, at ); // the segment came back zeroed + } + // a *bytes node: the bytes to write through, and the reference to store + TableBytesSlot AllocBytes( int64_t length ) + { + TableBytesSlot slot; + uint32_t at = 0; + TableBlob * blob = AllocBlob( length, false, at ); + if ( blob == NULL ) { return slot; } + slot.data = (uint8_t *) ( blob + 1 ); + slot.length = length; + slot.ref.value = at; + return slot; + } + + // a *string node: room for length bytes and the zero byte after them + TableStringSlot AllocString( int64_t length ) + { + TableStringSlot slot; + uint32_t at = 0; + TableBlob * blob = AllocBlob( length, true, at ); + if ( blob == NULL ) { return slot; } + slot.data = (char *) ( blob + 1 ); + slot.length = length; + slot.ref.value = at; + return slot; + } +}; + +// ---- TablePackMap: the pack walk's identity map (docs/SPEC-TABLES.md §3.1, §6.2) ---- +// +// ONE ENTRY PER REACHABLE NODE, and that map IS identity: a node must know +// where it landed to be named a second time, so Lock packs a shared node ONCE +// and every later reference resolves to the one body it already has. That is +// the same first-visit numbering the wire uses, so the pack order and the node +// order are one order. +// +// COLOURING AN ENTRY WHILE ITS DESCENT IS OPEN COSTS ONE BIT, and it is what +// makes a data cycle free to refuse: a reference to an entry still open is a +// cycle, and Lock returns failure rather than recursing away. The ROOT's entry +// is open for the whole walk. +// +// The map is proportional to NODES, never to bytes, and it lives on the +// AUTHORING side, where §6.5 licenses allocation. Nothing on the reading path +// ever builds one. +struct TablePackEntry +{ + const void * key; // the node's address in the graph being packed + int64_t offset; // where that node landed in the region + uint8_t open; // its descent is still open: a reference here is a cycle +}; + +struct TablePackMap +{ + TablePackEntry * entries = NULL; + int64_t capacity = 0; // a power of two, or zero while empty + int64_t count = 0; + TableAllocator allocator; // the caller's, carried from the walk that built it +}; + +inline void TablePackMapInit( TablePackMap & map, TableAllocator allocator ) +{ + map.entries = NULL; + map.capacity = 0; + map.count = 0; + map.allocator = allocator; +} + +inline void TablePackMapShutdown( TablePackMap & map ) +{ + map.allocator.free( map.allocator.context, map.entries ); + TablePackMapInit( map, map.allocator ); +} + +// The two walks behind Lock re-derive the SAME map from the same graph — the +// numbering is never carried between them (§3.1) — so the second starts from +// an empty map and keeps the capacity the first paid for. +inline void TablePackMapReset( TablePackMap & map ) +{ + if ( map.entries != NULL ) { memset( map.entries, 0, (size_t) map.capacity * sizeof( TablePackEntry ) ); } + map.count = 0; +} + +// open addressing, linear probing, a multiply-shift hash over the address: a +// node key is a pointer and its low bits are alignment, so the low bits alone +// would collide on every node of one type +inline int64_t TablePackMapSlot( const TablePackMap & map, const void * key ) +{ + uint64_t hash = (uint64_t) (uintptr_t) key; + hash *= 0x9E3779B97F4A7C15ull; + hash ^= hash >> 29; + int64_t mask = map.capacity - 1; + int64_t at = (int64_t) ( hash & (uint64_t) mask ); + while ( map.entries[at].key != NULL && map.entries[at].key != key ) + { + at = ( at + 1 ) & mask; + } + return at; +} + +inline TablePackEntry * TablePackMapFind( TablePackMap & map, const void * key ) +{ + if ( map.capacity == 0 ) { return NULL; } + TablePackEntry * entry = &map.entries[ TablePackMapSlot( map, key ) ]; + return entry->key == key ? entry : NULL; +} + +// QUADRUPLING, not doubling, and the reason is measured: growth rehashes every +// entry, and on a graph of 131,071 nodes the doubling schedule spent 45% of +// Lock in rehashing alone. Quadrupling from 1024 buys 1.35x on that graph and +// keeps the map NODE-proportional (§6.2) — under 128 bytes a node at its +// worst, right after a grow, and about 64 on average. +inline bool TablePackMapGrow( TablePackMap & map ) +{ + TablePackMap grown; + grown.allocator = map.allocator; + grown.capacity = map.capacity != 0 ? map.capacity * 4 : 1024; + grown.entries = (TablePackEntry *) map.allocator.alloc( map.allocator.context, grown.capacity * (int64_t) sizeof( TablePackEntry ) ); + if ( grown.entries == NULL ) { return false; } + for ( int64_t i = 0; i < map.capacity; i++ ) + { + if ( map.entries[i].key == NULL ) { continue; } + grown.entries[ TablePackMapSlot( grown, map.entries[i].key ) ] = map.entries[i]; + grown.count++; + } + map.allocator.free( map.allocator.context, map.entries ); + map = grown; + return true; +} + +// REACH a node: one probe answers both questions the walk has. A true "taken" +// says this is a FIRST visit, and the entry is now the node's, coloured open +// at "offset"; otherwise the entry is the one the node already has, and its +// open bit says cycle or sharing. NULL is an allocation failure, and it is a +// refusal like any other: Lock fails rather than packing a graph it cannot +// track. +// +// It is one call and not a find followed by an insert because the walk asks +// this question twice per node — once to measure, once to pack — and every +// probe is a miss into a table larger than L2. +inline TablePackEntry * TablePackMapReach( TablePackMap & map, const void * key, int64_t offset, bool & taken, int64_t & slot ) +{ + if ( ( map.count + 1 ) * 4 >= map.capacity * 3 ) // keep the load factor under three quarters + { + if ( !TablePackMapGrow( map ) ) { return NULL; } + } + slot = TablePackMapSlot( map, key ); + TablePackEntry * entry = &map.entries[slot]; + taken = entry->key != key; // an empty slot is a first visit; the key is never NULL + if ( taken ) + { + entry->key = key; + entry->offset = offset; + entry->open = 1; + map.count++; + } + return entry; +} + +// The descent finished: the node keeps its entry — identity outlives the +// descent — and stops being a cycle. The "hint" is the slot Reach returned, and it +// is checked against the key rather than trusted, so a rehash between the two +// costs a second probe instead of correctness. +inline void TablePackMapClose( TablePackMap & map, const void * key, int64_t hint ) +{ + if ( hint >= 0 && hint < map.capacity && map.entries[hint].key == key ) + { + map.entries[hint].open = 0; + return; + } + TablePackEntry * entry = TablePackMapFind( map, key ); + if ( entry != NULL ) { entry->open = 0; } +} + +// ---- resolution contexts: which encoding a walk is reading ---- + +struct TableArenaCtx { const TableArena * arena; }; +struct TableRegionCtx {}; + +// ---- a BYTE BUFFER's resolution (docs/SPEC-TABLES.md §2.5, §6.3) ---- +// +// The same two encodings a table pointer has, resolved the same way: a +// self-relative delta in a region — one add, no base — and an arena offset +// while the builder is mutable. The blob is reached through its header, and a +// view is the header plus eight and the header's first word. Nothing here +// allocates and nothing copies: off a locked region, a loaded one or an +// opened cook the view points INTO the region. +inline const TableBlob * TableBlobAt( const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) ( (const uint8_t *) &ref + ref.value ) : NULL; +} +inline const TableBlob * TableBlobAt( const TableRegionCtx &, const TableRef & ref ) { return TableBlobAt( ref ); } +inline const TableBlob * TableBlobAt( const TableArenaCtx & ctx, const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) TableArenaAt( *ctx.arena, (uint32_t) ref.value ) : NULL; +} +inline const TableBlob * TableBlobAt( const TableArena & arena, const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) TableArenaAt( arena, (uint32_t) ref.value ) : NULL; +} + +inline TableBytesView TableBytesViewOf( const TableBlob * blob ) +{ + TableBytesView view = { NULL, 0 }; + if ( blob != NULL ) { view.data = (const uint8_t *) ( blob + 1 ); view.length = (int64_t) blob->length; } + return view; +} +inline TableStringView TableStringViewOf( const TableBlob * blob ) +{ + TableStringView view = { NULL, 0 }; + if ( blob != NULL ) { view.data = (const char *) ( blob + 1 ); view.length = (int64_t) blob->length; } + return view; +} + +// the const form's hot path: one add, no base +inline TableBytesView TableBytesAt( const TableRef & ref ) { return TableBytesViewOf( TableBlobAt( ref ) ); } +inline TableStringView TableStringAt( const TableRef & ref ) { return TableStringViewOf( TableBlobAt( ref ) ); } +// and the context forms a walk uses: a region context, an arena context, or +// the arena itself while the builder is mutable +template inline TableBytesView TableBytesAt( const Ctx & ctx, const TableRef & ref ) { return TableBytesViewOf( TableBlobAt( ctx, ref ) ); } +template inline TableStringView TableStringAt( const Ctx & ctx, const TableRef & ref ) { return TableStringViewOf( TableBlobAt( ctx, ref ) ); } + +// allocate a blob in the arena and point the slot at it; the slot holds the +// arena offset, as every slot does while the builder is mutable +inline uint8_t * TableBytesEmplace( TableWorker & worker, TableRef & slot, int64_t length ) +{ + TableBytesSlot allocated = worker.AllocBytes( length ); + slot = allocated.ref; + return allocated.data; +} +// the text is copied in when one is given; a NULL text leaves the zeros for +// the caller to fill +inline char * TableStringEmplace( TableWorker & worker, TableRef & slot, const char * text, int64_t length ) +{ + TableStringSlot allocated = worker.AllocString( length ); + slot = allocated.ref; + if ( allocated.data != NULL && text != NULL && length > 0 ) { memcpy( allocated.data, text, (size_t) length ); } + return allocated.data; +} + +// ---- the FLAT NODE TABLE (docs/SPEC-TABLES.md §3.1) ---- +// +// A pointered save writes every reachable node ONCE, into a node table, and a +// pointer field rides as an INDEX into it under kind 17. The encoding is +// flat: no pointer edge is a nesting level, so a chain's length is not a depth, +// and two references to one node are one node. +// +// THE FIELD RIDES ONCE: an L with sixty-four bits of capability frames a +// numbering of any size, so the whole numbering is one contiguous payload and a +// save's node bodies have no aggregate ceiling. + +static const uint64_t kTableNodeIndexNull = 0; // absence and null are one value +static const uint64_t kTableNodeIndexRoot = 1; // the body that hosts the table + +// The not-materialized sentinel (§6.3): a record whose type id this build could +// not name. Distinct from every real offset including the root's 0, so an index +// resolving through it yields NULL and can never fabricate the root. +static const uint64_t kTableNodeAbsent = 0xFFFFFFFFFFFFFFFFull; + +// What a node's storage answers when the FRAMING ITSELF is refused rather than +// merely unnameable: a count its L cannot carry, one above the int32 cap, or a +// blob past the size cap (docs/SPEC-TABLES.md §3.1, §6.5). An unnameable type +// id commands no storage and keeps its index. This one makes the whole measure +// answer -1 with its reason. +static const int64_t kTableNodeRefused = -2; + +// ---- the numbering, on the SAVE side ---- +// +// One entry per reachable node in FIRST-VISIT order, so entry k is node index +// k + 2. The two thunks are what let one loop write a table of mixed types: the +// numbering walk knows each target's type STATICALLY at the site it numbers it, +// so it stores the instantiation there and the loop never asks what a node is. +struct TableNumbering; + +struct TableNodeEntry +{ + const void * node; + uint64_t type_id; + // the type id's MESSAGE-FORM SLOT (docs/SPEC-TABLES.md §3.3), stored where + // the numbering walk stores the id itself and for the same reason: the + // target's type is known STATICALLY at the site that numbers it, so a + // form 2 save reads the slot out of the entry instead of looking an id up. + // Every pointer target's type id is an entry of the announcement, which is + // what makes the slot a compile-time fact of a POINTERED message too. + uint64_t type_slot; + int64_t ( * measure )( const void * ctx, const TableNumbering & numbering, TableIds & ids, const void * node ); + bool ( * save )( const void * ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const void * node ); + // the same two over the MESSAGE FORM (docs/SPEC-TABLES.md §3.3): a bitpacked + // body at a bit position, its pointer indices at the width the node count + // settled + int64_t ( * message_measure )( const void * ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const void * node ); + bool ( * message_save )( const void * ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const void * node ); +}; + +struct TableNumbering +{ + TablePackMap seen; // node -> index; the ROOT is index 1, open for the whole walk + TableNodeEntry * entries = NULL; + int64_t count = 0; + int64_t capacity = 0; +}; + +// The numbering allocates through the map's pair rather than carrying a second +// copy of it: one numbering is one walk, and a walk has one allocator. +inline void TableNumberingInit( TableNumbering & n, TableAllocator allocator ) +{ + TablePackMapInit( n.seen, allocator ); + n.entries = NULL; + n.count = 0; + n.capacity = 0; +} + +inline void TableNumberingShutdown( TableNumbering & n ) +{ + TableAllocator allocator = n.seen.allocator; + TablePackMapShutdown( n.seen ); + allocator.free( allocator.context, n.entries ); + n.entries = NULL; + n.count = 0; + n.capacity = 0; +} + +// The index a numbered node was given, for the save that writes it into a +// pointer slot. False means the two walks disagree about the graph, which is a +// refusal and never a guess. +inline bool TableNumberingIndex( const TableNumbering & n, const void * node, uint64_t & index ) +{ + if ( n.seen.capacity == 0 ) { return false; } + const TablePackEntry & entry = n.seen.entries[ TablePackMapSlot( n.seen, node ) ]; + if ( entry.key != node ) { return false; } + index = (uint64_t) entry.offset; + return true; +} + +inline bool TableNumberingAppend( TableNumbering & n, const TableNodeEntry & entry ) +{ + if ( n.count == n.capacity ) + { + // GROW BY COPY, never by realloc: the allocator hook is a PAIR, and a + // game's heap is not required to have a resize primitive at all. The + // schedule quadruples, so the copying is amortized to a constant per + // entry and the growth is the same growth it always was. + int64_t capacity = n.capacity != 0 ? n.capacity * 4 : 256; + TableAllocator allocator = n.seen.allocator; + TableNodeEntry * grown = (TableNodeEntry *) allocator.alloc( allocator.context, capacity * (int64_t) sizeof( TableNodeEntry ) ); + if ( grown == NULL ) { return false; } + if ( n.entries != NULL ) + { + memcpy( grown, n.entries, (size_t) n.count * sizeof( TableNodeEntry ) ); + allocator.free( allocator.context, n.entries ); + } + n.entries = grown; + n.capacity = capacity; + } + n.entries[n.count++] = entry; + return true; +} + +// The thunks the numbering stores. Each resolves to the closure member's own +// MeasureBody / SaveBodyFields through an overload set in the member's DECLARING +// file, reached by argument-dependent lookup at instantiation — the same bridge +// the arena's TableReset uses, and the reason a numbering may span the files of +// one unit without any file naming another's members. +template +inline int64_t TableNodeMeasureThunk( const void * ctx, const TableNumbering & numbering, TableIds & ids, const void * node ) +{ + return TableNodeMeasure( *(const Ctx *) ctx, numbering, ids, *(const T *) node ); +} + +template +inline bool TableNodeSaveThunk( const void * ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const void * node ) +{ + return TableNodeSave( *(const Ctx *) ctx, numbering, w, ids, *(const T *) node ); +} + +template +inline int64_t TableNodeMessageMeasureThunk( const void * ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const void * node ) +{ + return TableNodeMessageMeasure( *(const Ctx *) ctx, numbering, index_bits, at, *(const T *) node ); +} + +template +inline bool TableNodeMessageSaveThunk( const void * ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const void * node ) +{ + return TableNodeMessageSave( *(const Ctx *) ctx, numbering, index_bits, w, *(const T *) node ); +} +// ---- a BYTE BUFFER's record (docs/SPEC-TABLES.md §2.5, §3.1) ---- +// +// A blob rides as a node record under one of two RESERVED type ids — the fold +// a table's name takes, over the keywords "bytes" and "string", which no table +// can be named — with the bytes as its body and nothing framed inside. These +// two thunks are what the numbering stores for a blob, as it stores a +// member's codec for a table: the length, and the bytes verbatim. +static const uint64_t kTableBytesTypeId = 0x2f2ec0474f1c4fe4ull; // fnv1a64( "bytes" ) +static const uint64_t kTableStringTypeId = 0x704be0d8faaffc58ull; // fnv1a64( "string" ) + +template +inline int64_t TableBlobMeasureThunk( const void *, const TableNumbering &, TableIds &, const void * node ) +{ + return (int64_t) ( (const TableBlob *) node )->length; +} + +template +inline bool TableBlobSaveThunk( const void *, const TableNumbering &, TableWriter & w, TableIds &, const void * node ) +{ + const TableBlob * blob = (const TableBlob *) node; + w.raw( (const void *) ( blob + 1 ), (int64_t) blob->length ); + return true; +} + +// and the same two on the MESSAGE FORM (§3.3): a blob record is its length at +// thirty-two raw bits, an ALIGN, then the bytes verbatim +template +inline int64_t TableBlobMessageMeasureThunk( const void *, const TableNumbering &, int64_t, int64_t at, const void * node ) +{ + const int64_t length = (int64_t) ( (const TableBlob *) node )->length; + return 32 + TableAlignBits( at + 32 ) + length * 8; +} + +template +inline bool TableBlobMessageSaveThunk( const void *, const TableNumbering &, int64_t, TableBitWriter & w, const void * node ) +{ + const TableBlob * blob = (const TableBlob *) node; + w.put( (uint64_t) blob->length, 32 ); + w.align(); + w.putbytes( (const uint8_t *) ( blob + 1 ), (int64_t) blob->length ); + return !w.overflow; +} +// TableNodeTableMeasure and TableNodeTableSave are the framing, and they are +// ONE fill rule written twice — measure derives it from the graph and save +// derives the same one, which is what makes measure == save hold across a +// pointer graph (§3.1). +// +// The field rides ONCE, under the reserved id, kind 12: the payload opens with +// the count and then carries the records back to back, each a type id +// REFERENCE, a length and a body. The reserved id is interned BEFORE the +// records, and a record's type id before its body, which is the first-use order +// the trailer is written in (§3). +template +inline int64_t TableNodeTablePayload( const Ctx & ctx, TableIds & ids, const TableNumbering & n ) +{ + int64_t payload = TableLebBytes( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + payload += TableLebBytes( ids.ref( n.entries[k].type_id ) ); + const int64_t body = n.entries[k].measure( (const void *) &ctx, n, ids, n.entries[k].node ); + if ( body < 0 ) { return -1; } + payload += TableLebBytes( (uint64_t) body ) + body; + } + return payload; +} + +template +inline int64_t TableNodeTableMeasure( const Ctx & ctx, TableIds & ids, const TableNumbering & n ) +{ + if ( n.count == 0 ) { return 0; } // a root that reaches no nodes writes none of them + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayload( ctx, ids, n ); + if ( payload < 0 ) { return -1; } + return TableLebBytes( ref ) + 1 + TableLebBytes( (uint64_t) payload ) + payload; +} + +template +inline bool TableNodeTableSave( const Ctx & ctx, TableWriter & w, TableIds & ids, const TableNumbering & n ) +{ + if ( n.count == 0 ) { return true; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayload( ctx, ids, n ); + if ( payload < 0 ) { return false; } + w.putleb( ref ); + w.put8( 12 ); // kind 12 is the opaque byte payload: a reader that cannot name the id skips by L + w.putleb( (uint64_t) payload ); + w.putleb( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.putleb( ids.ref( n.entries[k].type_id ) ); + const int64_t body = n.entries[k].measure( (const void *) &ctx, n, ids, n.entries[k].node ); + if ( body < 0 ) { return false; } + w.putleb( (uint64_t) body ); + if ( !n.entries[k].save( (const void *) &ctx, n, w, ids, n.entries[k].node ) ) { return false; } + } + return true; +} + +// ---- the numbering, on the LOAD side: a region's NODE DIRECTORY (§6.3) ---- +// +// The wire's numbering made resident: one entry per numbered node, in index +// order, position i describing node index i + 1 — so position 0 is the ROOT at +// offset 0. It is ATTRIBUTION, and attribution is separable: nothing that reads +// a structure touches it, a deref is one add on a self-relative offset, and a +// caller may release it once Load returns. +struct TableNodeDirEntry +{ + uint64_t offset; + uint64_t type_id; +}; + +// the node's extent cursor, defined with the extent runtime (docs/SPEC-TABLES.md +// §2.8, §2.9); the node map names it only through a pointer. +struct TableExtentCarve; + +// TableNodeMap is what a pointer slot resolves through while a body decodes. +struct TableNodeMap +{ + uint8_t * base = NULL; + const TableNodeDirEntry * entries = NULL; + int64_t count = 0; // the ROOT's entry included, so it is records + 1 + bool good = false; // the node table read whole; a numbering that failed resolves nothing + // WHERE THE NODES LIVE, and therefore what a resolved slot holds: a region + // takes the SELF-RELATIVE delta so a deref is one add, and the tool's + // builder path takes the node's ARENA OFFSET (§6.3). + bool arena = false; + // WHERE A MAP'S ENTRIES AND A LIST'S ELEMENTS LAND while this node's body + // decodes (docs/SPEC-TABLES.md §2.8, §2.9): the node's own extent on the + // region path and the builder's arena on the tool's. It is MUTABLE + // because the cursor belongs to ONE node's decode and the dispatch that + // owns that node holds the map by const reference, exactly as it did + // before either construct existed. The decoder's signature does not + // move for a construct it may not carry. + mutable TableExtentCarve * carve = NULL; + // and the TOOL's path's allocation front, set once: there the arrays + // are the builder's arena's rather than a node's extent. + TableWorker * worker = NULL; + // THE TOOL PATH'S REFUSAL (docs/SPEC-TABLES.md §2.9): a count above the + // int32 cap met while a body decoded. LoadBuilder answers NULL for it + // and moves no counter; mutable for the reason the cursor is. + mutable bool refused = false; +}; + +// TableNodeResolve places one node index in a pointer slot, and every failure +// is one of §4's events with the pointer left null. The declared TARGET type id +// is checked at every index, the root's included: the root carries no record +// and therefore no wire type id, so the READER'S OWN root type is what the +// claim is checked against. +inline void TableNodeResolve( const TableNodeMap & map, TableRef & slot, uint64_t index, uint64_t target, TableReport * report ) +{ + slot.value = 0; + if ( index == kTableNodeIndexNull || !map.good ) { return; } + if ( index - 1 >= (uint64_t) map.count ) + { + report->malformed = true; // an index above node_count + 1 + return; + } + const TableNodeDirEntry & entry = map.entries[index - 1]; + if ( entry.offset == kTableNodeAbsent ) + { + // a node whose type id this build could not name KEEPS ITS INDEX, and + // every pointer naming it reads null. The unknown was counted once, at + // the node, not once per pointer. + return; + } + if ( entry.type_id != target ) + { + report->kind_mismatch++; + return; + } + slot.value = map.arena ? (int64_t) entry.offset + : (int64_t) ( ( map.base + entry.offset ) - (const uint8_t *) &slot ); +} + +// ---- the record SCAN, and it is the whole of load's bound (§3.1) ---- +// +// Reading follows no reference. The scan walks the root body's top-level fields, +// finds the ONE under the reserved id, and reads records out of its payload in +// order — the field rides once, so nothing is copied to make a body contiguous +// and the generated body decoder never learns the transport exists. +struct TableNodeScan +{ + TableReader fields; // over the ROOT body, skipping past everything else + const uint8_t * payload; // the node-table field's payload + int64_t payload_size; + int64_t payload_offset; + bool opened; // the root body has been walked for the field + uint64_t declared; + int64_t records; + bool present; // the root body carries a node table at all + bool malformed; + const TableIdTable * ids; +}; + +inline TableNodeScan TableNodeScanBegin( const uint8_t * body, int64_t size, TableReport * report, const TableIdTable * ids ) +{ + TableNodeScan s = { TableReader( body, size, report, ids ), NULL, 0, 0, false, 0, 0, false, false, ids }; + return s; +} + +// find the node-table field, or answer false when the root body has none. A +// body carrying an id more than once is legal input and THE LAST OCCURRENCE +// WINS (docs/SPEC-TABLES.md §3), so the walk runs to the terminator and keeps +// the last rather than stopping at the first. +inline bool TableNodeScanOpen( TableNodeScan & s ) +{ + if ( s.opened ) { return false; } + s.opened = true; + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.fields.getleb( ref ) ) { break; } + if ( ref == 0 ) { break; } // the terminator + if ( s.ids == NULL || ref > (uint64_t) s.ids->count ) { break; } + const uint64_t id = s.ids->at( ref ); + if ( !s.fields.has( 1 ) ) { break; } + const uint8_t kind = s.fields.get8(); + if ( id == kTableNodeTableFieldId ) + { + s.present = true; + if ( kind != 12 ) { s.malformed = true; return false; } + uint64_t length = 0; + if ( !s.fields.getleb( length ) || !s.fields.room( length ) ) { s.malformed = true; return false; } + s.payload = s.fields.buffer + s.fields.offset; + s.payload_size = (int64_t) length; + s.fields.offset += (int64_t) length; + continue; + } + if ( !s.fields.skip( kind ) ) { break; } + } + if ( s.payload == NULL ) { return false; } + TableReader head( s.payload, s.payload_size, s.fields.report, s.ids ); + if ( !head.getleb( s.declared ) ) { s.malformed = true; return false; } + s.payload_offset = head.offset; + return true; +} + +// the next record, or false at the end of the table — s.malformed says whether +// the end was the end or the framing giving out +inline bool TableNodeScanNext( TableNodeScan & s, uint64_t & type_id, const uint8_t * & body, int64_t & length ) +{ + if ( !s.opened && !TableNodeScanOpen( s ) ) { return false; } + if ( s.payload == NULL || s.payload_offset >= s.payload_size ) { return false; } + TableReader rec( s.payload, s.payload_size, s.fields.report, s.ids ); + rec.offset = s.payload_offset; + uint64_t ref = 0; + if ( !rec.getleb( ref ) || ref == 0 || s.ids == NULL || ref > (uint64_t) s.ids->count ) + { + s.malformed = true; // a type id reference of 0, or one past the table + return false; + } + type_id = s.ids->at( ref ); + uint64_t declared_length = 0; + if ( !rec.getleb( declared_length ) ) + { + s.malformed = true; // a record whose length is damaged + return false; + } + if ( declared_length > (uint64_t) ( s.payload_size - rec.offset ) ) + { + s.malformed = true; // a record whose length runs past its field + return false; + } + body = s.payload + rec.offset; + length = (int64_t) declared_length; + s.payload_offset = rec.offset + length; + s.records++; + return true; +} + +// The record scan is AUTHORITATIVE: node_count is data from the wire, and a +// count that disagrees with the scan is malformed. Nothing is sized from it +// before the scan has confirmed it. +inline bool TableNodeScanWhole( TableNodeScan & s ) +{ + if ( s.malformed ) { return false; } + if ( !s.present ) { return true; } // no node table at all is not a broken one + return s.declared == (uint64_t) s.records; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_ARENA + +#ifndef MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES +#define MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES + +namespace mapdemo { + +// ---- the NODE TABLE on the message wire (docs/SPEC-TABLES.md §3.1, §3.3) ---- +// +// THE NODE TABLE, WHEN A BODY HAS ONE, IS THE FIRST FIELD OF THE ROOT BODY: the +// reserved id as a reference, the node count at THIRTY-TWO RAW BITS, then the +// records back to back, each a type id reference and a body: a table's fields +// end at their own zero reference, and a blob's body is a length, an align and +// its bytes. A root +// that reaches no node elides the field, like every other empty thing. +// +// Measure derives the numbering from the graph and save derives the same one, +// and the two thunks stored at numbering time are what let one loop write a +// table of mixed types. +template +inline int64_t TableMessageNodeTableMeasure( const Ctx & ctx, const TableNumbering & n, int64_t index_bits, int64_t at ) +{ + if ( n.count == 0 ) { return 0; } // a root that reaches no nodes writes none of them + int64_t bits = kTableMessageRefBitsHere + 32; + for ( int64_t k = 0; k < n.count; k++ ) + { + bits += kTableMessageRefBitsHere; + const int64_t body = n.entries[k].message_measure( (const void *) &ctx, n, index_bits, at + bits, n.entries[k].node ); + if ( body < 0 ) { return -1; } + bits += body; + } + return bits; +} + +template +inline bool TableMessageNodeTableSave( const Ctx & ctx, const TableNumbering & n, int64_t index_bits, TableBitWriter & w ) +{ + if ( n.count == 0 ) { return true; } + w.put( kTableNodeTableFieldSlot, kTableMessageRefBitsHere ); + w.put( (uint64_t) n.count, 32 ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.put( n.entries[k].type_slot, kTableMessageRefBitsHere ); + if ( !n.entries[k].message_save( (const void *) &ctx, n, index_bits, w, n.entries[k].node ) ) { return false; } + } + return !w.overflow; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES + +#ifndef MAPDEMO_SCHEMA_TABLE_RETAIN +#define MAPDEMO_SCHEMA_TABLE_RETAIN + +namespace mapdemo { + +// ---- RETAIN-UNKNOWN (docs/SPEC-TABLES.md §6.6) ---- +// +// A REGION ROUND TRIP AND ONLY THAT: LoadRetain is Load's path into a region +// and SaveRetain saves from that same region. The builder path carries no +// retention, because a builder has no node directory to anchor a record on and +// re-derives its numbering from the reader's declaration order. +// +// Nothing here allocates. The record bytes and the retained-id list are the +// caller's storage, declared with their capacities, and a record that does not +// fit whole is dropped with one retain_lost. + +// THE PATH NAMES THE BODY, and it is the REGION's own address (§6.6). Step one +// is the node's index in the region's node directory, 1 for the root body and +// k for the node at directory position k - 1. Every further step is the PAIR: +// the field ordinal in the body the step descends from, in the READER's own +// declaration order, and the element index inside that field: zero for a +// scalar body, the element's index for an array of any of the four kinds, the +// ARM's OWN ORDINAL for a union, and the key's slot for a map. +static const int32_t kTableRetainDepthMax = 5; + +struct TableRetainStep +{ + uint32_t ordinal; + uint32_t index; +}; + +// at is the node's own address, which is what the SAVE side matches on: a +// record carries the directory INDEX and the directory answers the address in +// one add, so neither side ever searches a numbering. +struct TableRetainPath +{ + const void * at; + uint32_t node; + int32_t depth; + TableRetainStep steps[ kTableRetainDepthMax ]; +}; + +inline TableRetainPath TableRetainPathRoot( const void * at, uint32_t node ) +{ + TableRetainPath path; + path.at = at; + path.node = node; + path.depth = 0; + return path; +} + +// A STEP IS COMPUTED LOCALLY, at the moment the walk descends (§6.6), and it +// is taken by VALUE so that a descent is an expression: both sides walk the +// same declaration order, so neither numbers a tree and neither pops. +inline TableRetainPath TableRetainStepInto( const TableRetainPath & path, uint32_t ordinal, uint32_t index ) +{ + TableRetainPath out = path; + if ( out.depth < kTableRetainDepthMax ) + { + out.steps[ out.depth ].ordinal = ordinal; + out.steps[ out.depth ].index = index; + } + out.depth++; + return out; +} + +// THE CALLER'S TWO STORES (§6.6): the record bytes and the retained ids, each +// a pointer, a capacity and what has been used of it. A retention buffer +// belongs to ONE loaded region, and the next LoadRetain into it resets both. +struct TableRetain +{ + // AN ENTRY IS THE ID AND ITS SLOT IN THE TRAILER BEING WRITTEN. The two + // stores are numbered into ONE trailer in merged first-use order, so an + // index into this list is not the number a second reference wants and the + // slot rides beside the id. The layout is this port's own. + struct Id + { + uint64_t id; + int32_t slot; + }; + + uint8_t * bytes = NULL; + int64_t capacity = 0; + int64_t used = 0; + Id * ids = NULL; + int32_t id_capacity = 0; + int32_t id_used = 0; + int32_t count = 0; // records held + + // the REGION this buffer belongs to: a record carries a directory index + // and the save resolves it here, so nothing searches and nothing allocates + const uint8_t * base = NULL; + const TableNodeDirEntry * directory = NULL; + int64_t directory_count = 0; +}; + +// A RETAINED RECORD IS READER-PRIVATE (§6.6). It is not a wire form: no form +// byte, no version, no declared byte order, and nothing ever writes one to +// disk or hands one to another process. What it must CARRY is the body it +// belongs to, and the field's identity and bytes with every reference +// resolved. This layout is one sound way to carry them and nothing compares +// two ports' buffers. +// +// u32 record bytes, this header included +// u32 node the path's first step +// u32 depth the step pairs that follow +// u32 payload bytes +// u64 field id +// u8 kind +// u8 placed the save's own mark, cleared before every save +// depth x { u32 ordinal, u32 index } +// payload the field's payload with every reference resolved +static const int64_t kTableRetainRecordHeader = 26; + +inline uint32_t TableRetainRead32( const uint8_t * p ) +{ + return uint32_t( p[0] ) | uint32_t( p[1] ) << 8 | uint32_t( p[2] ) << 16 | uint32_t( p[3] ) << 24; +} + +inline uint64_t TableRetainRead64( const uint8_t * p ) +{ + return uint64_t( TableRetainRead32( p ) ) | ( uint64_t( TableRetainRead32( p + 4 ) ) << 32 ); +} + +inline void TableRetainWrite32( uint8_t * p, uint32_t v ) +{ + p[0] = uint8_t( v ); p[1] = uint8_t( v >> 8 ); p[2] = uint8_t( v >> 16 ); p[3] = uint8_t( v >> 24 ); +} + +inline void TableRetainWrite64( uint8_t * p, uint64_t v ) +{ + TableRetainWrite32( p, uint32_t( v ) ); + TableRetainWrite32( p + 4, uint32_t( v >> 32 ) ); +} + +inline int64_t TableRetainRecordBytes( const uint8_t * record ) { return (int64_t) TableRetainRead32( record ); } +inline uint32_t TableRetainRecordNode( const uint8_t * record ) { return TableRetainRead32( record + 4 ); } +inline int32_t TableRetainRecordDepth( const uint8_t * record ) { return (int32_t) TableRetainRead32( record + 8 ); } +inline int64_t TableRetainRecordPayloadBytes( const uint8_t * record ) { return (int64_t) TableRetainRead32( record + 12 ); } +inline uint64_t TableRetainRecordId( const uint8_t * record ) { return TableRetainRead64( record + 16 ); } +inline uint8_t TableRetainRecordKind( const uint8_t * record ) { return record[24]; } +inline bool TableRetainRecordPlaced( const uint8_t * record ) { return record[25] != 0; } +inline const uint8_t * TableRetainRecordSteps( const uint8_t * record ) { return record + kTableRetainRecordHeader; } +inline const uint8_t * TableRetainRecordPayload( const uint8_t * record ) +{ + return record + kTableRetainRecordHeader + 8 * (int64_t) TableRetainRecordDepth( record ); +} +inline uint8_t * TableRetainRecordPayload( uint8_t * record ) +{ + return record + kTableRetainRecordHeader + 8 * (int64_t) TableRetainRecordDepth( record ); +} + +// THE RECORD'S OWN BODY, resolved through the directory the buffer holds. A +// node index names one node for the life of the region, so this is one add. +inline const void * TableRetainRecordAt( const TableRetain & retain, const uint8_t * record ) +{ + const uint32_t node = TableRetainRecordNode( record ); + if ( retain.directory == NULL || node == 0 || (int64_t) node > retain.directory_count ) { return NULL; } + return (const void *) ( retain.base + retain.directory[ node - 1 ].offset ); +} + +// Does this record belong to the body the walk is standing in? The node first, +// which rejects almost everything in one compare, then the step pairs. +inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * record, const TableRetainPath & path ) +{ + if ( TableRetainRecordDepth( record ) != path.depth ) { return false; } + if ( TableRetainRecordAt( retain, record ) != path.at ) { return false; } + const uint8_t * steps = TableRetainRecordSteps( record ); + for ( int32_t i = 0; i < path.depth; i++ ) + { + if ( TableRetainRead32( steps + 8 * i ) != path.steps[i].ordinal ) { return false; } + if ( TableRetainRead32( steps + 8 * i + 4 ) != path.steps[i].index ) { return false; } + } + return true; +} + +// EVERY ID THIS BUILD CAN NAME, ascending: the set TableIds's capacity is +// derived from. An id inside a retained record takes its trailer entry from +// the GENERATED table when it is here and from the CALLER's list otherwise, so +// no retained id ever enters the generated table and no id is written twice. +static const int32_t kTableRetainKnownIds = 76; +static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, + 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, + 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, + 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, + 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, +}; + +inline bool TableRetainNameable( uint64_t id ) +{ + int32_t low = 0, high = kTableRetainKnownIds - 1; + while ( low <= high ) + { + const int32_t mid = low + ( high - low ) / 2; + if ( kTableRetainKnown[mid] == id ) { return true; } + if ( kTableRetainKnown[mid] < id ) { low = mid + 1; } else { high = mid - 1; } + } + return false; +} + +// THE TWO STORES, NUMBERED INTO ONE TRAILER in merged first-use order (§6.6). +// It answers the surface TableIds answers, ref, count, truncate and +// overflow, so the retain family's codec is the plain one with its names +// changed, and +// the GENERATED TABLE IS UNTOUCHED: its capacity, its overflow rule and its +// -1 stand exactly as they are for every save. +struct TableRetainIds +{ + TableIds known; + int32_t known_slot[ TableIds::kCapacity ]; + TableRetain * retain; + int32_t count; + bool overflow; + bool lost; // a retained id past the caller's capacity: the record is dropped + + TableRetainIds( TableRetain * to_retain ) : retain( to_retain ), count( 0 ), overflow( false ), lost( false ) {} + + // an id this build CAN name, which is every id the generated codec writes + uint64_t ref( uint64_t id ) + { + const int32_t before = known.count; + const uint64_t k = known.ref( id ); + if ( known.overflow ) { overflow = true; return 1; } + if ( known.count != before ) { known_slot[ (int32_t) k - 1 ] = ++count; } + return (uint64_t) known_slot[ (int32_t) k - 1 ]; + } + + // an id from INSIDE a retained record. A retained id takes its entry from + // the caller's list, and one past the capacity sets lost: the record is + // dropped, nothing else about the save changes, and the save is never + // refused (§6.6). + uint64_t record_ref( uint64_t id ) + { + if ( TableRetainNameable( id ) ) { return ref( id ); } + if ( retain == NULL ) { lost = true; return 0; } + for ( int32_t i = 0; i < retain->id_used; i++ ) + { + if ( retain->ids[i].id == id ) { return (uint64_t) retain->ids[i].slot; } + } + if ( retain->id_used >= retain->id_capacity ) { lost = true; return 0; } + retain->ids[ retain->id_used ].id = id; + retain->ids[ retain->id_used ].slot = ++count; + retain->id_used++; + return (uint64_t) count; + } + + // undo every entry taken since mark, in either store. Both are appended in + // slot order, so an entry removed is the last one of its store. + void truncate( int32_t mark ) + { + while ( known.count > 0 && known_slot[ known.count - 1 ] > mark ) { known.truncate( known.count - 1 ); } + while ( retain != NULL && retain->id_used > 0 && retain->ids[ retain->id_used - 1 ].slot > mark ) { retain->id_used--; } + count = mark; + } +}; + +// THE FILE STILL CARRIES ONE ID TABLE (§3): the split is the writer's storage +// rather than the wire's, and the trailer is one merge of two slot-ordered +// stores. +inline int64_t TableRetainIdsBytes( const TableRetainIds & ids ) { return int64_t( ids.count ) * 8 + 8; } + +// A TWO-WAY MERGE over the stores' own slot order, and not a scan for each +// slot. Every entry either store holds took its slot from the same counter, so +// the two runs interleave to exactly the slots 1 to count and the merge has no +// case for a slot neither store took. +inline void TableRetainIdsWrite( TableWriter & w, const TableRetainIds & ids ) +{ + const int32_t retained = ids.retain != NULL ? ids.retain->id_used : 0; + int32_t i = 0, j = 0; + while ( i < ids.known.count || j < retained ) + { + if ( j >= retained || ( i < ids.known.count && ids.known_slot[i] < ids.retain->ids[j].slot ) ) + { + w.put64( ids.known.ids[i] ); + i++; + continue; + } + w.put64( ids.retain->ids[j].id ); + j++; + } + w.put64( uint64_t( ids.count ) ); +} + +// ---- THE RESOLVING WALK (§6.6) ---- +// +// A reference names a SLOT of the file's id table, so a verbatim copy +// re-emitted into a file whose table is ordered differently would point at +// other names in silence. A retained record therefore holds the field with +// every reference replaced by the sixty-four-bit id it names, and every length +// that frames a rewritten reference recomputed. +// +// THE WALK IS AN INTERPRETATION, AND ITS VERDICT IS STATED: it reads kind +// bytes, lengths and references and nothing else. No value is decoded, no +// bound is checked, no branch is taken on a payload byte, and anything it +// cannot frame DROPS THE RECORD, counts one retain_lost, and never raises +// malformed on the plain read. +// +// THE WALK IS ONE PASS EACH WAY, and its cost is linear in the record's own +// bytes. Every length that frames a content in the resolved form is a fixed +// slot rather than a canonical LEB128, so the capture reserves it, writes the +// content, and fills the slot in behind it. A spelling that had to know the +// resolved size before writing it would have to walk each content twice, once +// at every level, and the file chooses the nesting. +// +// A retained record's inner nesting is the WRITER's and not this build's, so +// it is the one depth on this path a file can drive. The cap counts NESTED +// BODIES, and a record past it is dropped on the same rule as any other shape +// the walk cannot take. Time no longer rests on it: it is a small stated +// constant and nothing more. +static const int32_t kTableRetainWalkDepthMax = 64; + +// the three RESERVED ids (§3.1, §3.3). One inside a retained record's payload +// would be re-emitted into a nested body, where it is malformed, so meeting +// one drops the record. +inline bool TableRetainReservedId( uint64_t id ) +{ + return id == kTableNodeTableFieldId || id == kTableBuildVersionFieldId || id == kTableMessageVocabularyFieldId; +} + +struct TableRetainIn +{ + const uint8_t * in; + int64_t size; + int64_t at; + const TableIdTable * ids; + uint8_t * out; // NULL: measuring, and nothing is written + int64_t out_at; +}; + +inline void TableRetainInRaw( TableRetainIn & s, const uint8_t * from, int64_t bytes ) +{ + if ( s.out != NULL ) { memcpy( s.out + s.out_at, from, (size_t) bytes ); } + s.out_at += bytes; +} + +inline void TableRetainInLeb( TableRetainIn & s, uint64_t v ) +{ + uint8_t b[10]; + int64_t n = 0; + while ( v >= 0x80 ) { b[n++] = uint8_t( v ) | 0x80; v >>= 7; } + b[n++] = uint8_t( v ); + TableRetainInRaw( s, b, n ); +} + +inline void TableRetainInId( TableRetainIn & s, uint64_t id ) +{ + uint8_t b[8]; + TableRetainWrite64( b, id ); + TableRetainInRaw( s, b, 8 ); +} + +inline bool TableRetainInLebRead( TableRetainIn & s, uint64_t & value ) +{ + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( s.at >= s.size ) { return false; } + const uint8_t b = s.in[ s.at++ ]; + if ( i == 9 && b > 1 ) { return false; } + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) { return i == 0 || b != 0; } + shift += 7; + } + return false; +} + +// one REFERENCE resolved to the id it names. A zero reference is the wire's +// own "no id", the enum's None and the union's empty arm, and rides as the +// id zero. A reference above the entry count, a reference at an id-table entry +// of zero, and a reference at a reserved id are each damage the plain read +// never looked at, and each drops the record. +inline bool TableRetainInRef( TableRetainIn & s, bool zero_allowed ) +{ + uint64_t ref = 0; + if ( !TableRetainInLebRead( s, ref ) ) { return false; } + if ( ref == 0 ) + { + if ( !zero_allowed ) { return false; } + TableRetainInId( s, 0 ); + return true; + } + if ( s.ids == NULL || ref > (uint64_t) s.ids->count ) { return false; } + const uint64_t id = s.ids->at( ref ); + if ( id == 0 || TableRetainReservedId( id ) ) { return false; } + TableRetainInId( s, id ); + return true; +} + +inline int64_t TableRetainInPayload( TableRetainIn & s, uint8_t kind, int32_t depth ); +inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ); + +// ONE FRAMED LENGTH in the resolved form: a pair of fixed u32. The first is +// the RESOLVED byte count of the content it frames, reserved here and written +// once the content is out. The second is the SAVE's scratch, left zero by the +// capture and filled by the walk that emits. +// +// The record is the reader's own storage and nothing outside this family ever +// reads it, so a length may be written after the bytes it measures. That is +// the whole of what makes the walk one pass. +static const int64_t kTableRetainSlotBytes = 8; + +inline int64_t TableRetainInSlot( TableRetainIn & s ) +{ + const int64_t at = s.out_at; + const uint8_t zero[ kTableRetainSlotBytes ] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + TableRetainInRaw( s, zero, kTableRetainSlotBytes ); + return at; +} + +inline void TableRetainInPatch( TableRetainIn & s, int64_t slot, int64_t resolved ) +{ + if ( s.out != NULL ) { TableRetainWrite32( s.out + slot, (uint32_t) resolved ); } +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it. +// The measuring pass takes the same path and reserves the same fixed width, so +// the size it answers is the size the writing pass lays down. +inline int64_t TableRetainInFramed( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s ); + const int64_t resolved = TableRetainInContent( s, kind, length, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s, slot, resolved ); + return resolved; +} + +inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + if ( length < 0 || s.at + length > s.size ) { return -1; } + const int64_t end = s.at + length; + const int64_t began = s.out_at; + switch ( kind ) + { + case 13: // a table BODY: fields, then the zero reference + { + for ( ;; ) + { + uint64_t ref = 0; + const int64_t mark = s.at; + if ( !TableRetainInLebRead( s, ref ) ) { return -1; } + // THE TERMINATOR IS A REFERENCE, and a reference in the + // resolved form is a fixed eight-byte id: the zero that ends a + // body rides at the width every other one does. + if ( ref == 0 ) { TableRetainInId( s, 0 ); break; } + s.at = mark; + if ( !TableRetainInRef( s, false ) ) { return -1; } + if ( s.at >= end ) { return -1; } + const uint8_t field_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &field_kind, 1 ); + if ( TableRetainInPayload( s, field_kind, depth ) < 0 ) { return -1; } + if ( s.at > end ) { return -1; } + } + break; + } + case 14: // an ARRAY body: the element kind, the count, then the elements + { + if ( s.at >= end ) { return -1; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainInLebRead( s, n ) ) { return -1; } + TableRetainInLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( TableRetainInPayload( s, elem_kind, depth ) < 0 ) { return -1; } + if ( s.at > end ) { return -1; } + } + break; + } + case 16: // an ENUM-KEYED body: N triples of a KEY REFERENCE, an L and the element + { + if ( s.at >= end ) { return -1; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainInLebRead( s, n ) ) { return -1; } + TableRetainInLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + // A KEYED BODY'S KEYS RESOLVE AT EVERY ELEMENT KIND (§6.6, §3.2) + if ( !TableRetainInRef( s, false ) ) { return -1; } + uint64_t slot_bytes = 0; + if ( !TableRetainInLebRead( s, slot_bytes ) ) { return -1; } + if ( slot_bytes > (uint64_t) ( end - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, elem_kind, (int64_t) slot_bytes, depth + 1 ) < 0 ) { return -1; } + } + break; + } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + default: + // every other content is bytes: a string, wide text, an escape, a + // payload-free kind, a scalar under a keyed slot's own length + TableRetainInRaw( s, s.in + s.at, length ); + s.at += length; + break; + } + if ( s.at != end ) { return -1; } + return s.out_at - began; +} + +// the depth a payload carries is its enclosing body's: only a framed CONTENT +// is a level, and TableRetainInContent is the one place the cap is read. +inline int64_t TableRetainInPayload( TableRetainIn & s, uint8_t kind, int32_t depth ) +{ + const int64_t began = s.out_at; + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: // the fixed-width kinds, by width + case 3: case 7: case 21: case 26: + case 4: case 8: case 10: case 22: case 27: + case 5: case 9: case 11: case 23: case 28: + case 18: case 19: case 24: case 29: + { + int64_t width = 1; + switch ( kind ) + { + case 3: case 7: case 21: case 26: width = 2; break; + case 4: case 8: case 10: case 22: case 27: width = 4; break; + case 5: case 9: case 11: case 23: case 28: width = 8; break; + case 18: case 19: case 24: case 29: width = 16; break; + default: width = 1; break; + } + if ( s.at + width > s.size ) { return -1; } + TableRetainInRaw( s, s.in + s.at, width ); + s.at += width; + break; + } + case 12: case 31: case 32: case 33: // L, then L bytes, nothing framed inside + { + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + TableRetainInLeb( s, length ); + TableRetainInRaw( s, s.in + s.at, (int64_t) length ); + s.at += (int64_t) length; + break; + } + case 13: case 14: case 16: // L, then a body the walk resolves + { + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, kind, (int64_t) length, depth + 1 ) < 0 ) { return -1; } + break; + } + case 15: // a UNION: the arm id reference, and when it is not zero its kind, L and payload + { + const int64_t mark = s.at; + uint64_t arm = 0; + if ( !TableRetainInLebRead( s, arm ) ) { return -1; } + s.at = mark; + if ( !TableRetainInRef( s, true ) ) { return -1; } + if ( arm == 0 ) { break; } + if ( s.at >= s.size ) { return -1; } + const uint8_t arm_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &arm_kind, 1 ); + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, arm_kind, (int64_t) length, depth + 1 ) < 0 ) { return -1; } + break; + } + case 30: // an ENUM's variant reference, zero for None + { + if ( !TableRetainInRef( s, true ) ) { return -1; } + break; + } + case 17: return -1; // A NODE INDEX (§3.1): the whole record goes with it + default: return -1; // a kind this walk cannot frame + } + return s.out_at - began; +} + +// ---- CAPTURE: the load side (§6.6) ---- +// +// The field is skipped by its framing exactly as it always was and counted +// unknown exactly as it always was, so a full buffer degrades to the default +// behavior one field at a time. False is what r.skip( kind ) answers false +// for, and nothing else: retention can lose a field, it can never turn a good +// read into a bad one. +inline bool TableRetainCapture( TableRetain * retain, TableReader & r, const TableRetainPath & path, + uint64_t field_id, uint8_t kind ) +{ + const int64_t start = r.offset; + if ( !r.skip( kind ) ) { return false; } + if ( retain == NULL ) { return true; } + const int64_t wire_bytes = r.offset - start; + + TableRetainIn probe; + probe.in = r.buffer + start; + probe.size = wire_bytes; + probe.at = 0; + probe.ids = r.ids; + probe.out = NULL; + probe.out_at = 0; + const int64_t payload = TableRetainInPayload( probe, kind, 0 ); + if ( payload < 0 || probe.at != wire_bytes ) { r.report->retain_lost++; return true; } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL: the buffer never holds a + // truncated field, and the read continues (§6.6) + r.report->retain_lost++; + return true; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, field_id ); + record[24] = kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableRetainIn write; + write.in = r.buffer + start; + write.size = wire_bytes; + write.at = 0; + write.ids = r.ids; + write.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out_at = 0; + if ( TableRetainInPayload( write, kind, 0 ) < 0 ) { r.report->retain_lost++; return true; } + retain->used += need; + retain->count++; + r.report->retained++; + return true; +} + +// LoadRetain RESETS BOTH STORES and writes into neither list (§6.6): a +// retained record carries its field's identity in the record itself, with +// every reference resolved. +inline void TableRetainReset( TableRetain * retain, const TableNodeMap & nodes, const uint8_t * region ) +{ + if ( retain == NULL ) { return; } + retain->used = 0; + retain->id_used = 0; + retain->count = 0; + retain->base = region; + retain->directory = nodes.entries; + retain->directory_count = nodes.count; +} + +// ---- RECORD LIFETIME (docs/SPEC-TABLES.md §6.6) ---- +// +// A RETAINED RECORD BELONGS TO THE BODY OCCURRENCE THAT CARRIED IT, AND DIES +// WITH IT. Legal input can carry a known child body twice, and the later +// occurrence resets the child and wins whole (§3, §4): the records retained +// under the earlier occurrence go with the values it held. The discard moves +// NEITHER counter. The writer superseded the data, so nothing was lost that +// the load could have kept, and retained counted the record when its bytes +// were kept and does not fall when they are let go. +// +// The occurrences are four, and each is a body the wire lets a writer put down +// again: a repeated TABLE field, by value or under ?, a UNION whose arm is +// written again, a MAP's duplicate key, and a KEYED-ARRAY slot written again. +// The FIELD form covers the three where the field itself is read again, arm +// switches and shrinking arrays included; the BODY form covers a duplicate key +// inside one occurrence of a map, where the field is read once and the entry +// twice. +inline bool TableRetainUnder( const uint8_t * record, const void * at, const TableRetain & retain, + const TableRetainPath & path, bool field, uint32_t ordinal ) +{ + if ( TableRetainRecordAt( retain, record ) != at ) { return false; } + const int32_t depth = TableRetainRecordDepth( record ); + if ( field ) + { + if ( depth <= path.depth ) { return false; } + } + else if ( depth < path.depth ) { return false; } + const uint8_t * steps = TableRetainRecordSteps( record ); + for ( int32_t i = 0; i < path.depth; i++ ) + { + if ( TableRetainRead32( steps + 8 * i ) != path.steps[i].ordinal ) { return false; } + if ( TableRetainRead32( steps + 8 * i + 4 ) != path.steps[i].index ) { return false; } + } + if ( field && TableRetainRead32( steps + 8 * path.depth ) != ordinal ) { return false; } + return true; +} + +inline void TableRetainDiscard( TableRetain * retain, const TableRetainPath & path, bool field, uint32_t ordinal ) +{ + if ( retain == NULL || retain->count == 0 ) { return; } + int64_t read = 0, write = 0; + int32_t kept = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + const int64_t bytes = TableRetainRecordBytes( retain->bytes + read ); + if ( !TableRetainUnder( retain->bytes + read, path.at, *retain, path, field, ordinal ) ) + { + if ( write != read ) { memmove( retain->bytes + write, retain->bytes + read, (size_t) bytes ); } + write += bytes; + kept++; + } + read += bytes; + } + retain->used = write; + retain->count = kept; +} + +inline void TableRetainDiscardBody( TableRetain * retain, const TableRetainPath & path ) +{ + TableRetainDiscard( retain, path, false, 0 ); +} + +inline void TableRetainDiscardField( TableRetain * retain, const TableRetainPath & path, uint32_t ordinal ) +{ + TableRetainDiscard( retain, path, true, ordinal ); +} + +// ---- EMIT: the save side (§6.6) ---- +// +// The record read back the other way: every resolved id becomes the reference +// the trailer being written gives it, and every length is recomputed against +// the references' new widths. The walk is the capture's mirror and the same +// damage rules apply, except that damage cannot be met: these bytes are the +// reader's own. +// +// A WIRE LENGTH IS CANONICAL LEB128 AND RIDES BEFORE ITS CONTENT, so this side +// cannot fill a slot in behind the bytes the way the capture does. It takes +// one POST-ORDER pass instead: measuring computes each content's wire size and +// leaves it in that content's own scratch slot, and the emit reads the size +// there rather than walking for it. Measuring runs immediately before the +// emit, on the same record and the same id table, which is what makes the two +// readings one walk. +struct TableRetainOut +{ + uint8_t * in; // the record: only a framed length's scratch half is written + int64_t size; + int64_t at; + TableRetainIds * ids; + TableWriter * w; // NULL: measuring, and the scratch slots are being filled + int64_t bytes; +}; + +inline void TableRetainOutRaw( TableRetainOut & s, const uint8_t * from, int64_t bytes ) +{ + if ( s.w != NULL ) { s.w->raw( from, bytes ); } + s.bytes += bytes; +} + +inline void TableRetainOutLeb( TableRetainOut & s, uint64_t v ) +{ + if ( s.w != NULL ) { s.w->putleb( v ); } + s.bytes += TableLebBytes( v ); +} + +inline bool TableRetainOutLebRead( TableRetainOut & s, uint64_t & value ) +{ + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( s.at >= s.size ) { return false; } + const uint8_t b = s.in[ s.at++ ]; + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) { return true; } + shift += 7; + } + return false; +} + +inline bool TableRetainOutRef( TableRetainOut & s ) +{ + if ( s.at + 8 > s.size ) { return false; } + const uint64_t id = TableRetainRead64( s.in + s.at ); + s.at += 8; + if ( id == 0 ) { TableRetainOutLeb( s, 0 ); return true; } // the wire's own no-id + const uint64_t ref = s.ids->record_ref( id ); + if ( s.ids->lost || s.ids->overflow ) { return false; } + TableRetainOutLeb( s, ref ); + return true; +} + +inline bool TableRetainOutPayload( TableRetainOut & s, uint8_t kind, int32_t depth ); +inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t length, int32_t depth ); + +// ONE FRAMED CONTENT, read out of the record's fixed slot and written with the +// canonical LEB128 length this wire wants. The ids it names are interned on +// the way past, which is what makes measure and save one walk in two readings, +// exactly as every other body on this wire is. +// +// MEASURING walks the content, then leaves the wire size it found in the +// scratch half of the slot. EMITTING reads that size, writes it, and CHECKS +// the content against it: a size no measure of this save left there is a +// record refused rather than a length that does not frame what follows. +inline bool TableRetainOutFramed( TableRetainOut & s, uint8_t kind, int32_t depth ) +{ + if ( s.at + kTableRetainSlotBytes > s.size ) { return false; } + uint8_t * const slot = s.in + s.at; + const int64_t resolved = (int64_t) TableRetainRead32( slot ); + s.at += kTableRetainSlotBytes; + if ( s.w == NULL ) + { + const int64_t began = s.bytes; + if ( !TableRetainOutContent( s, kind, resolved, depth ) ) { return false; } + const int64_t wire = s.bytes - began; + if ( wire > 0xFFFFFFFFll ) { return false; } + TableRetainWrite32( slot + 4, (uint32_t) wire ); + s.bytes += TableLebBytes( (uint64_t) wire ); + return true; + } + const int64_t wire = (int64_t) TableRetainRead32( slot + 4 ); + TableRetainOutLeb( s, (uint64_t) wire ); + const int64_t began = s.bytes; + if ( !TableRetainOutContent( s, kind, resolved, depth ) ) { return false; } + return s.bytes - began == wire; +} + +inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t length, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return false; } + if ( length < 0 || s.at + length > s.size ) { return false; } + const int64_t end = s.at + length; + switch ( kind ) + { + case 13: + { + for ( ;; ) + { + if ( s.at + 8 > end ) { return false; } + const uint64_t id = TableRetainRead64( s.in + s.at ); + if ( id == 0 ) { s.at += 8; TableRetainOutLeb( s, 0 ); break; } + if ( !TableRetainOutRef( s ) ) { return false; } + if ( s.at >= end ) { return false; } + const uint8_t field_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &field_kind, 1 ); + if ( !TableRetainOutPayload( s, field_kind, depth ) ) { return false; } + } + break; + } + case 14: + { + if ( s.at >= end ) { return false; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainOutLebRead( s, n ) ) { return false; } + TableRetainOutLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( !TableRetainOutPayload( s, elem_kind, depth ) ) { return false; } + } + break; + } + case 16: + { + if ( s.at >= end ) { return false; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainOutLebRead( s, n ) ) { return false; } + TableRetainOutLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( !TableRetainOutRef( s ) ) { return false; } + if ( !TableRetainOutFramed( s, elem_kind, depth + 1 ) ) { return false; } + } + break; + } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; + default: + TableRetainOutRaw( s, s.in + s.at, length ); + s.at += length; + break; + } + return s.at == end; +} + +// the depth a payload carries is its enclosing body's, exactly as on the +// capture side: only a framed CONTENT is a level. +inline bool TableRetainOutPayload( TableRetainOut & s, uint8_t kind, int32_t depth ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: + case 3: case 7: case 21: case 26: + case 4: case 8: case 10: case 22: case 27: + case 5: case 9: case 11: case 23: case 28: + case 18: case 19: case 24: case 29: + { + int64_t width = 1; + switch ( kind ) + { + case 3: case 7: case 21: case 26: width = 2; break; + case 4: case 8: case 10: case 22: case 27: width = 4; break; + case 5: case 9: case 11: case 23: case 28: width = 8; break; + case 18: case 19: case 24: case 29: width = 16; break; + default: width = 1; break; + } + if ( s.at + width > s.size ) { return false; } + TableRetainOutRaw( s, s.in + s.at, width ); + s.at += width; + break; + } + case 12: case 31: case 32: case 33: + { + uint64_t length = 0; + if ( !TableRetainOutLebRead( s, length ) ) { return false; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return false; } + TableRetainOutLeb( s, length ); + TableRetainOutRaw( s, s.in + s.at, (int64_t) length ); + s.at += (int64_t) length; + break; + } + case 13: case 14: case 16: + { + if ( !TableRetainOutFramed( s, kind, depth + 1 ) ) { return false; } + break; + } + case 15: + { + if ( s.at + 8 > s.size ) { return false; } + const uint64_t arm = TableRetainRead64( s.in + s.at ); + if ( !TableRetainOutRef( s ) ) { return false; } + if ( arm == 0 ) { break; } + if ( s.at >= s.size ) { return false; } + const uint8_t arm_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &arm_kind, 1 ); + if ( !TableRetainOutFramed( s, arm_kind, depth + 1 ) ) { return false; } + break; + } + case 30: + { + if ( !TableRetainOutRef( s ) ) { return false; } + break; + } + default: return false; + } + return true; +} + +// ---- THE RETAINED TAIL: where the records go back (§6.6) ---- +// +// AT THE END OF THEIR OWN BODY, IN THE ORDER RETAINED. Position carries +// nothing on this wire, so appending is chosen for three properties: it is a +// write with no splice, the retained order is preserved, and the result is +// IDEMPOTENT after the first save. +// +// A RETAINED ID PAST THE CAPACITY COUNTS ONE retain_lost AND ITS RECORD IS +// DROPPED, and the save is never refused. MeasureRetain and SaveRetain drop +// the same records under the same walk, so the measure sees the same overflow +// and its answer is the size the save writes. + +// one record's WIRE bytes under the trailer being written, and -1 for a record +// this save cannot place: an id the caller's list had no room for, or a +// resolved form the walk cannot read back. The ids it names are interned on +// the way past, which is what makes measure and save one rule read twice. +// +// THIS IS THE MEASURING PASS, and it leaves every framed content's wire size +// in that content's own scratch slot. The record is the caller's buffer and +// the pass writes nothing else into it. +inline int64_t TableRetainRecordWire( uint8_t * record, TableRetainIds & ids, uint64_t & ref ) +{ + const int32_t mark = ids.count; + ids.lost = false; + ref = ids.record_ref( TableRetainRecordId( record ) ); + if ( !ids.lost && !ids.overflow ) + { + TableRetainOut s; + s.in = TableRetainRecordPayload( record ); + s.size = TableRetainRecordPayloadBytes( record ); + s.at = 0; + s.ids = &ids; + s.w = NULL; + s.bytes = 0; + if ( TableRetainOutPayload( s, TableRetainRecordKind( record ), 0 ) && s.at == s.size ) + { + return TableLebBytes( ref ) + 1 + s.bytes; + } + } + // the record is not written at all, and nothing else about the save + // changes: a full id list degrades to the default behavior one record at a + // time, and the entries this attempt took are given back + ids.truncate( mark ); + ids.lost = false; + return -1; +} + +inline int64_t TableRetainTailMeasure( TableRetain * retain, TableRetainIds & ids, const TableRetainPath & path ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return 0; } + int64_t bytes = 0; + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordHere( *retain, record, path ) ) { continue; } + uint64_t ref = 0; + const int64_t wire = TableRetainRecordWire( record, ids, ref ); + if ( wire < 0 ) { continue; } + bytes += wire; + } + return bytes; +} + +inline bool TableRetainTailSave( TableRetain * retain, TableRetainIds & ids, TableWriter & w, const TableRetainPath & path ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return true; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordHere( *retain, record, path ) ) { continue; } + uint64_t ref = 0; + if ( TableRetainRecordWire( record, ids, ref ) < 0 ) { continue; } + w.putleb( ref ); + w.put8( TableRetainRecordKind( record ) ); + TableRetainOut s; + s.in = TableRetainRecordPayload( record ); + s.size = TableRetainRecordPayloadBytes( record ); + s.at = 0; + s.ids = &ids; + s.w = &w; + s.bytes = 0; + if ( !TableRetainOutPayload( s, TableRetainRecordKind( record ), 0 ) ) { return false; } + record[25] = 1; // PLACED: the one mark the save leaves on the buffer + } + return !w.overflow; +} + +// THE SAVE'S OWN SHARE OF retain_lost, counted ONCE and read after the save +// (§6.6): every record the walk did not place. A record whose path no longer +// names a body, one the caller's id list had no room for, and one the walk +// could not read back are one number here, because the check a caller reads is +// one number. A record is marked as it is written, so this cannot double-count +// a body measured twice. +inline void TableRetainClearPlaced( TableRetain * retain ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + retain->bytes[ at + 25 ] = 0; + at += TableRetainRecordBytes( retain->bytes + at ); + } +} + +inline void TableRetainCountLost( const TableRetain * retain, TableReport * report ) +{ + if ( retain == NULL || retain->bytes == NULL || report == NULL ) { return; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + const uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordPlaced( record ) ) { report->retain_lost++; } + } +} + +// THE NODE TABLE under retention (§3.1, §6.6): the same fill rule the plain +// pair derives, with the retain family's ids and each record's own body +// reached through a dispatch the CALL supplies rather than a second pair of +// thunks on the numbering. A store per node on the PLAIN save path would be a +// cost this feature is not allowed to have. +template +inline int64_t TableNodeTablePayloadRetain( const Ctx & ctx, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure ) +{ + int64_t payload = TableLebBytes( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + payload += TableLebBytes( ids.ref( n.entries[k].type_id ) ); + const int64_t body = measure( ctx, n, ids, n.entries[k].type_id, n.entries[k].node, retain ); + if ( body < 0 ) { return -1; } + payload += TableLebBytes( (uint64_t) body ) + body; + } + return payload; +} + +template +inline int64_t TableNodeTableMeasureRetain( const Ctx & ctx, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure ) +{ + if ( n.count == 0 ) { return 0; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayloadRetain( ctx, ids, n, retain, measure ); + if ( payload < 0 ) { return -1; } + return TableLebBytes( ref ) + 1 + TableLebBytes( (uint64_t) payload ) + payload; +} + +template +inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure, Save save ) +{ + if ( n.count == 0 ) { return true; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayloadRetain( ctx, ids, n, retain, measure ); + if ( payload < 0 ) { return false; } + w.putleb( ref ); + w.put8( 12 ); // kind 12 is the opaque byte payload, exactly as the plain save writes it + w.putleb( (uint64_t) payload ); + w.putleb( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.putleb( ids.ref( n.entries[k].type_id ) ); + const int64_t body = measure( ctx, n, ids, n.entries[k].type_id, n.entries[k].node, retain ); + if ( body < 0 ) { return false; } + w.putleb( (uint64_t) body ); + if ( !save( ctx, n, w, ids, n.entries[k].type_id, n.entries[k].node, retain ) ) { return false; } + } + return true; +} +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_RETAIN + +#ifndef MAPDEMO_SCHEMA_TABLE_EXTENT +#define MAPDEMO_SCHEMA_TABLE_EXTENT + +namespace mapdemo { + +// ---- the NODE EXTENT: where a map's entries and a list's elements live (§2.8, §2.9) ---- + +// TableExtentCarve is a node's extent cursor, PRE-ORDER: a container's whole +// array first, then, element by element in the container's own order, the +// arrays of any list or map an element holds by value. The cursor is the node +// map's, because the generated decoder is threaded with that and not with a +// region. +struct TableExtentCarve +{ + uint8_t * at = NULL; // the region path: the node's extent, unspent + int64_t left = 0; + TableWorker * worker = NULL; // the TOOL's path: the arrays come from the arena +}; + +// AN UNREACHED SLOT MUST HOLD NO LIST OR MAP WITH ELEMENTS IN IT (§2.8, §2.9, +// §7.6). An empty one takes no bytes, so a record whose extent measures ZERO is +// a record whose every by-value list and map is empty. A measure that REFUSED +// answers non-zero here too, and refusing on it is the same answer one level up. +inline bool TableExtentUnreachedEmpty( int64_t extent ) { return extent == 0; } + +// ---- LoadMeasure's framing walk (§6.5) ---- +// +// The measure reads no field value: it walks each record's field headers, +// skipping every payload by its framing, to reach each N at every depth. A +// false is a REFUSAL, and it carries its reason (§6.5). +typedef bool ( * TableWireExtentFn )( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ); + +// the framing walk over an ARRAY OF TABLES held by value: its elements' own +// lists and maps are part of this node's extent too +inline bool TableWireExtentElements( const uint8_t * body, int64_t length, int64_t & at, TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } + if ( r.get8() != 13 ) { return true; } + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +// and over an ENUM-KEYED array, whose triples carry a key REFERENCE before each +// length-prefixed element (docs/SPEC-TABLES.md §3.2) +inline bool TableWireExtentKeyed( const uint8_t * body, int64_t length, int64_t & at, TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } + if ( r.get8() != 13 ) { return true; } + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t key = 0; + if ( !r.getleb( key ) ) { return true; } + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_EXTENT + +#ifndef MAPDEMO_SCHEMA_TABLE_MAP +#define MAPDEMO_SCHEMA_TABLE_MAP + +namespace mapdemo { + +// ---- a MAP: a sorted entry array, and the lookup over it (§2.8) ---- +// +// On the wire, in a region and in a cook a map is an array of one generated +// ENTRY table held in ascending key order. What this adds is Find — a binary +// search over that array where it lies — and a builder that inserts, replaces +// and erases by key. Nothing here is stored: a region and a cook carry the +// array and the count, and not one byte about a hash or a probe. + +// entries carved from ONE call to the allocator pair; a new segment is +// appended when the current one fills, and nothing ever moves (§6.4) +static const int32_t kTableMapSegmentEntries = 32; + +// TableDeclRef names a type in an unevaluated context and is never defined — +// what 's declval is for, without the include the generated corpus +// refuses to pay for (the iterator_traits note, §13.9). +template T & TableDeclRef(); + +// THE ORDER IS TOTAL, AND IT IS THE SAME IN NINE LANGUAGES (§2.8). Integers +// compare by VALUE, signed for the signed kinds and unsigned for the unsigned. +// Strings compare by BYTES, unsigned, a shorter string that is a prefix of a +// longer one first: memcmp over the common length, then the lengths. Never a +// locale, never a code point, never a case fold. +inline int TableKeyOrder( uint64_t a, uint64_t b ) { return a < b ? -1 : ( a > b ? 1 : 0 ); } +inline int TableKeyOrder( int64_t a, int64_t b ) { return a < b ? -1 : ( a > b ? 1 : 0 ); } +inline int TableKeyOrder( const char * a, int32_t a_length, const char * b, int32_t b_length ) +{ + const int32_t common = a_length < b_length ? a_length : b_length; + if ( common > 0 ) + { + const int order = memcmp( (const void *) a, (const void *) b, (size_t) common ); + if ( order != 0 ) { return order < 0 ? -1 : 1; } + } + return a_length < b_length ? -1 : ( a_length > b_length ? 1 : 0 ); +} + +// the length of a NUL-terminated key at a call site, bounded by the storage it +// has to fit: a key one byte longer than the bound is refused, never truncated +inline int32_t TableKeyLength( const char * key, int32_t bound ) +{ + if ( key == NULL ) { return 0; } + for ( int32_t i = 0; i <= bound; i++ ) { if ( key[i] == 0 ) { return i; } } + return bound + 1; // longer than the bound: the caller refuses it +} + +// A KEY IS DATA AND A LENGTH, and the length is CARRIED, never recomputed +// (§2.8, §3). A string(N) key holds any byte a wire or a text can spell, +// U+0000 included, so a lookup that measures to the first NUL answers that "a" +// and "a", 0, "b" are the same key: the first entry is found, RESET, and +// relabeled with the second key, which deletes an entry the report never +// mentions. Every internal lookup and every insertion takes this pair, and the +// public const char * surface builds one and is a wrapper over it. +struct TableMapKeyRef +{ + const char * data; + int32_t length; +}; + +// ---- the storage: SIXTEEN BYTES in the holder's record (§2.8, §7.2) ---- +// +// An int64 self-relative reference to the entry array and an int32 count, then +// padding to eight. The reference is a TableRef like a pointer's: in the arena +// it names the builder's HEAD, in a region it is the delta from the slot to +// the first entry, and 0 is the empty map in both. +template struct TableMap +{ + TableRef entries; + int32_t count = 0; // the LIVE count, in both forms + int32_t padding = 0; // named, so the record has no unwritten byte in it + + // ---- the CONST form: a locked region, a loaded one, an opened cook ---- + // + // One surface over one encoding (§6.3). A region reference resolves from + // the slot's own address, so every one of these is a member and needs no + // base and no context. + const Entry * Entries() const + { + return entries.value != 0 ? (const Entry *) ( (const uint8_t *) &entries + entries.value ) : NULL; + } + int32_t size() const { return count; } + + // FIND: floor( log2 n ) + 1 key compares, in place, no allocation. NULL + // when absent, and on a map[K]*T the RESOLVED pointer, which is what a + // pointer field's accessor answers. + template const Entry * FindEntry( Key key ) const + { + const Entry * base = Entries(); + int32_t low = 0, high = count; + while ( low < high ) + { + const int32_t mid = low + ( high - low ) / 2; + const int order = TableEntryOrder( base[mid], key ); + if ( order == 0 ) { return base + mid; } + if ( order < 0 ) { low = mid + 1; } else { high = mid; } + } + return NULL; + } + // the return type is DEDUCED, so it is worked out when a call site + // instantiates Find and not when the holder's record declares the slot — + // which is what lets the entry's own overloads be declared after it + template auto Find( Key key ) const + { + return TableEntryFound( FindEntry( key ) ); + } + + // ---- iteration: ASCENDING key order, the key beside the value ---- + // + // A proxy BY VALUE, the keyed array's shape (§2.4): for ( auto [ key, + // value ] : map ). It carries no iterator_traits, for the reason + // TableKeyed's does not (§13.9). + struct ConstEntry + { + decltype( TableEntryKey( TableDeclRef() ) ) key; + decltype( TableEntryFound( (const Entry *) NULL ) ) value; + }; + + struct ConstIterator + { + const Entry * at; + ConstEntry operator*() const { return ConstEntry{ TableEntryKey( *at ), TableEntryFound( at ) }; } + ConstIterator & operator++() { at++; return *this; } + bool operator==( const ConstIterator & other ) const { return at == other.at; } + bool operator!=( const ConstIterator & other ) const { return at != other.at; } + }; + + ConstIterator begin() const { return ConstIterator{ Entries() }; } + ConstIterator end() const { return ConstIterator{ Entries() + count }; } +}; + +// ---- the BUILDER's side: a head, and segments that never move (§2.8, §6.4) ---- +// +// The head is a small node in the arena holding the segment chain, the live +// count and the dead count, allocated when the first entry is inserted. Each +// segment is a fixed number of entries carved from one call to the allocator +// pair. An entry's address is stable for the arena's life, so a value handed +// back by an insert stays valid while other entries arrive. +struct TableMapHead +{ + TableRef first; // the arena offset of the first segment + TableRef last; // and of the one an insert appends into + int32_t live; + int32_t dead; +}; + +template struct TableMapSegment +{ + TableRef next; + int32_t used; // entries carved from this segment + int32_t padding; + uint32_t dead[ ( kTableMapSegmentEntries + 31 ) / 32 ]; // Erase marks one bit, never the entry + Entry entries[ kTableMapSegmentEntries ]; +}; + +inline bool TableMapSegmentDead( const uint32_t * dead, int32_t index ) +{ + return ( dead[ index / 32 ] & ( 1u << ( index % 32 ) ) ) != 0; +} + +// ---- the ORDERED CURSOR the four writing walks read (§2.8) ---- +// +// Measure, Save, Lock and Cook each write a map's entries in ascending key +// order with no key twice, deriving the order from the builder's entries as +// each walk derives the numbering (§3.1). Nothing passes between them, so +// measure == save over a map is a real check on two sorts agreeing. +// +// A REGION is already sorted, so its cursor is the array in place and +// allocates nothing. The BUILDER's is the sort: an array of entry pointers +// allocated through the pair and released before the walk returns, because +// sorting the segments themselves would move entries whose addresses a caller +// holds. +template struct TableMapCursor +{ + const Entry * const * order = NULL; // the builder's form: sorted pointers + const Entry * entries = NULL; // the region's form: the array in place + int32_t count = 0; + TableAllocator allocator; + bool ok = false; + const Entry * operator[]( int32_t index ) const + { + return order != NULL ? order[index] : entries + index; + } +}; + +// heapsort: O( n log n ) once per map, no recursion, no allocation past the +// pointer array the caller already paid for +template inline void TableMapSort( const Entry ** order, int32_t count ) +{ + for ( int32_t start = count / 2 - 1; start >= 0; start-- ) + { + int32_t root = start; + for ( ;; ) + { + int32_t child = 2 * root + 1; + if ( child >= count ) { break; } + if ( child + 1 < count && TableEntryOrder( *order[child], *order[child + 1] ) < 0 ) { child++; } + if ( TableEntryOrder( *order[root], *order[child] ) >= 0 ) { break; } + const Entry * swap = order[root]; order[root] = order[child]; order[child] = swap; + root = child; + } + } + for ( int32_t end = count - 1; end > 0; end-- ) + { + const Entry * swap = order[0]; order[0] = order[end]; order[end] = swap; + int32_t root = 0; + for ( ;; ) + { + int32_t child = 2 * root + 1; + if ( child >= end ) { break; } + if ( child + 1 < end && TableEntryOrder( *order[child], *order[child + 1] ) < 0 ) { child++; } + if ( TableEntryOrder( *order[root], *order[child] ) >= 0 ) { break; } + const Entry * hold = order[root]; order[root] = order[child]; order[child] = hold; + root = child; + } + } +} + +// the REGION form: the array is already sorted, so the cursor is the array +template +inline TableMapCursor TableMapOrder( const TableRegionCtx &, const TableMap & map ) +{ + TableMapCursor cursor; + cursor.entries = map.Entries(); + cursor.count = map.count; + cursor.ok = true; + return cursor; +} + +// the BUILDER's form: gather the LIVE entries out of the segment chain in +// insertion order, then sort. A dead entry costs nothing on any wire (§2.8). +template +inline TableMapCursor TableMapOrder( const TableArena & arena, const TableMap & map ) +{ + TableMapCursor cursor; + cursor.allocator = arena.allocator; + cursor.count = map.count; + if ( map.entries.value == 0 || map.count <= 0 ) { cursor.ok = map.count == 0; cursor.count = 0; return cursor; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + if ( head->live != map.count ) { return cursor; } // the slot and the head disagree: refused, never guessed + const Entry ** order = (const Entry **) arena.allocator.alloc( arena.allocator.context, (int64_t) map.count * (int64_t) sizeof( const Entry * ) ); + if ( order == NULL ) { return cursor; } + int32_t at = 0; + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 && at < map.count ) + { + const TableMapSegment * segment = (const TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used && at < map.count; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + order[at++] = segment->entries + i; + } + segment_ref = segment->next; + } + if ( at != map.count ) + { + arena.allocator.free( arena.allocator.context, order ); + return cursor; + } + TableMapSort( order, map.count ); + cursor.order = order; + cursor.ok = true; + return cursor; +} + +template +inline TableMapCursor TableMapOrder( const TableArenaCtx & ctx, const TableMap & map ) +{ + return TableMapOrder( *ctx.arena, map ); +} + +template inline void TableMapRelease( TableMapCursor & cursor ) +{ + if ( cursor.order != NULL ) { cursor.allocator.free( cursor.allocator.context, (void *) cursor.order ); } + cursor.order = NULL; +} + +// ---- the builder's five (§2.8) ---- +// +// Insert APPENDS after one LINEAR SCAN of the live entries for the key it may +// replace, Find is that same scan, and Erase is the scan and one bit. The +// builder builds NO INDEX, and that is a rule: the sort happens once, at Lock, +// Save or Cook, and every lookup that matters runs over the sorted region. + +// the head, allocated when the first entry is inserted +template +inline TableMapHead * TableMapReach( TableWorker & worker, TableMap & map ) +{ + if ( worker.arena == NULL || worker.arena->locked ) { return NULL; } + if ( map.entries.value != 0 ) { return (TableMapHead *) TableArenaAt( *worker.arena, (uint32_t) map.entries.value ); } + uint32_t at = 0; + TableMapHead * head = (TableMapHead *) worker.AllocRaw( (int64_t) sizeof( TableMapHead ), (int64_t) alignof( TableMapHead ), at ); + if ( head == NULL ) { return NULL; } + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + map.entries.value = (int64_t) at; + return head; +} + +// one entry's storage, appended: the current segment when it has room, a new +// one carved from one call to the pair when it does not +template +inline Entry * TableMapAppend( TableWorker & worker, TableMapHead * head, TableMap & map ) +{ + TableMapSegment * segment = NULL; + if ( head->last.value != 0 ) + { + segment = (TableMapSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + if ( segment->used >= kTableMapSegmentEntries ) { segment = NULL; } + } + if ( segment == NULL ) + { + uint32_t at = 0; + segment = (TableMapSegment *) worker.AllocRaw( (int64_t) sizeof( TableMapSegment ), (int64_t) alignof( TableMapSegment ), at ); + if ( segment == NULL ) { return NULL; } // the arena could not carve another segment + segment->next.value = 0; + segment->used = 0; + segment->padding = 0; + for ( int32_t i = 0; i < (int32_t) ( sizeof( segment->dead ) / sizeof( segment->dead[0] ) ); i++ ) { segment->dead[i] = 0; } + if ( head->last.value != 0 ) + { + TableMapSegment * previous = (TableMapSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + previous->next.value = (int64_t) at; + } + else + { + head->first.value = (int64_t) at; + } + head->last.value = (int64_t) at; + } + Entry * entry = segment->entries + segment->used; + segment->used++; + head->live++; + map.count++; + return entry; +} + +// the LINEAR SCAN: the live entries in insertion order, O( n ) key compares +template +inline Entry * TableMapScan( const TableArena & arena, const TableMap & map, Key key ) +{ + if ( map.entries.value == 0 ) { return NULL; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( TableEntryOrder( segment->entries[i], key ) == 0 ) { return segment->entries + i; } + } + segment_ref = segment->next; + } + return NULL; +} + +// ERASE marks the entry DEAD, one bit in the segment's slot and not in the +// entry table, and decrements the live count. Its storage is reclaimed at +// RESET and never reused mid-build, because reusing a slot would make "an +// entry's address is stable" false for exactly one case. +template +inline bool TableMapErase( TableArena & arena, TableMap & map, Key key ) +{ + if ( map.entries.value == 0 ) { return false; } + TableMapHead * head = (TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( TableEntryOrder( segment->entries[i], key ) != 0 ) { continue; } + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + map.count--; + return true; + } + segment_ref = segment->next; + } + return false; +} + +// ---- iterate on the BUILDER: INSERTION order, live entries only (§2.8) ---- +template struct TableMapEach +{ + const TableArena * arena; + TableRef first; + + struct Iterator + { + const TableArena * arena; + TableMapSegment * segment; + int32_t index; + + void Skip() + { + for ( ;; ) + { + if ( segment == NULL ) { return; } + if ( index >= segment->used ) + { + segment = segment->next.value != 0 ? (TableMapSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + index = 0; + continue; + } + if ( TableMapSegmentDead( segment->dead, index ) ) { index++; continue; } + return; + } + } + auto operator*() const { return TableEntryEach( segment->entries + index ); } + Iterator & operator++() { index++; Skip(); return *this; } + bool operator==( const Iterator & other ) const { return segment == other.segment && index == other.index; } + bool operator!=( const Iterator & other ) const { return !( *this == other ); } + }; + + Iterator begin() const + { + Iterator it = { arena, first.value != 0 ? (TableMapSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL, 0 }; + it.Skip(); + return it; + } + Iterator end() const { Iterator it = { arena, NULL, 0 }; return it; } +}; + +template +inline TableMapEach TableMapEachOf( const TableArena & arena, const TableMap & map ) +{ + TableMapEach each = { &arena, TableRef() }; + if ( map.entries.value != 0 ) + { + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + each.first = head->first; + } + return each; +} + +// ---- the LOAD side: where a decoded entry lands (§2.8) ---- +// +// THE READER TRUSTS NOTHING and spends one compare per entry. Every load path +// applies the same rules and produces one report (§4), so the region load of +// §6.5 and LoadBuilder never disagree about a wire. These two shapes are what +// makes that true with one generated decoder: a REGION carves the entry array +// out of the holder node's own extent, and the TOOL's path appends into the +// builder's arena, and the decoder above them cannot tell which it has. + +// The node's extent cursor is TableExtentCarve, the extent runtime's (§2.8, +// §2.9): a map's whole entry array is carved first, then, entry by entry in +// key order, the arrays of any list or map an entry's value holds by value. + +// TableMapFill is one map field being decoded: where the next entry lands, and +// the entry that last LANDED, which is what the ascending check compares +// against. +template struct TableMapFill +{ + TableMap * map = NULL; + Entry * array = NULL; // the region path: the carved array + int32_t capacity = 0; + TableWorker * worker = NULL; // the TOOL's path + bool ok = false; +}; + +template +inline TableMapFill TableMapFillBegin( const TableNodeMap & nodes, TableMap & map, uint32_t n ) +{ + TableMapFill fill; + fill.map = ↦ + map.entries.value = 0; + map.count = 0; + if ( nodes.carve == NULL ) { return fill; } + if ( nodes.carve->worker != NULL ) + { + fill.worker = nodes.carve->worker; // the tool's path: the arena carves + fill.ok = true; + return fill; + } + const int64_t align = (int64_t) alignof( Entry ); + uint8_t * base = (uint8_t *) ( ( (uintptr_t) nodes.carve->at + (uintptr_t) ( align - 1 ) ) & ~( (uintptr_t) ( align - 1 ) ) ); + const int64_t bytes = (int64_t) n * (int64_t) sizeof( Entry ); + const int64_t pad = (int64_t) ( base - nodes.carve->at ); + if ( pad + bytes > nodes.carve->left ) { return fill; } // the measure and the load disagree: refused + nodes.carve->at = base + bytes; + nodes.carve->left -= pad + bytes; + fill.array = (Entry *) base; + fill.capacity = (int32_t) n; + map.entries.value = (int64_t) ( base - (const uint8_t *) &map.entries ); + fill.ok = true; + return fill; +} + +// the entry that last LANDED — NULL before the first +template inline Entry * TableMapFillLast( TableMapFill & fill ) +{ + if ( fill.map->count <= 0 ) { return NULL; } + if ( fill.array != NULL ) { return fill.array + ( fill.map->count - 1 ); } + return TableMapLive( *fill.worker->arena, *fill.map, fill.map->count - 1 ); +} + +// the next slot, at the entry type's declared defaults +template inline Entry * TableMapFillNext( TableMapFill & fill ) +{ + if ( fill.array != NULL ) + { + if ( fill.map->count >= fill.capacity ) { return NULL; } + Entry * entry = fill.array + fill.map->count; + TableReset( *entry ); + fill.map->count++; + return entry; + } + TableMapHead * head = TableMapReach( *fill.worker, *fill.map ); + if ( head == NULL ) { return NULL; } + Entry * entry = TableMapAppend( *fill.worker, head, *fill.map ); + if ( entry != NULL ) { TableReset( *entry ); } + return entry; +} + +// A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): at the first entry whose key +// kind disagrees with the reader's declaration the map resets to EMPTY, one +// kind_mismatch is counted for the map, and its remaining bytes are skipped. +template inline void TableMapFillReset( TableMapFill & fill ) +{ + if ( fill.array != NULL ) + { + fill.map->entries.value = 0; + fill.map->count = 0; + return; + } + if ( fill.map->entries.value != 0 ) + { + TableMapHead * head = (TableMapHead *) TableArenaAt( *fill.worker->arena, (uint32_t) fill.map->entries.value ); + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + } + fill.map->count = 0; +} + +// an EMPTY map's reference is null in both encodings, so a load that placed +// nothing leaves the slot exactly as a Reset does +template inline void TableMapFillEnd( TableMapFill & fill ) +{ + if ( fill.array != NULL && fill.map->count == 0 ) { fill.map->entries.value = 0; } +} + +// the k-th LIVE entry of a builder map, in insertion order — what the tool +// path's ascending check compares against +template +inline Entry * TableMapLive( const TableArena & arena, const TableMap & map, int32_t index ) +{ + if ( map.entries.value == 0 ) { return NULL; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + int32_t at = 0; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( at == index ) { return segment->entries + i; } + at++; + } + segment_ref = segment->next; + } + return NULL; +} + +// ---- LoadMeasure's term, from the FRAMING alone (§2.8, §6.5) ---- +// +// LoadMeasure's term for a map is N x sizeof( Entry ) rounded to +// alignof( Entry ), AT EVERY DEPTH. N is framing and not a value, so this +// reads no field: it walks the map's own header and, where an entry's value +// holds a map or a list of its own, the entries' headers under it. The caller +// owns the allocation precisely so it can refuse a number it did not expect. +// Every -1 carries its REASON (§6.5): the int32 cap first, because a count +// past it cannot fit any body, and then the body's own L, the one rule a +// list's term answers by. +// A MAP ENTRY'S SMALLEST WIRE FOOTPRINT that commands one storage unit is its +// own L and the body's terminator, and under this form's variable lengths that +// footprint is TWO BYTES (docs/SPEC-TABLES.md §4.2). It is what bounds the N a +// map's L can carry, and therefore what a LoadMeasure may be asked for. +static const int64_t kTableMapEntryFloor = 2; + +inline bool TableMapWireExtent( const uint8_t * body, int64_t length, int64_t & at, + int64_t entry_size, int64_t entry_align, TableWireExtentFn inner, + const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } // no array header: nothing rides + if ( r.get8() != 13 ) { return true; } // not an array of tables: §4's ordinary kind mismatch + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + if ( n > (uint64_t) INT32_MAX ) { reason = count_over_extent_cap; return false; } + const int64_t rest = length - r.offset; + if ( n > (uint64_t) ( rest / kTableMapEntryFloor ) ) { reason = count_over_length; return false; } // an N the map's L cannot carry + at = ( at + entry_align - 1 ) & ~( entry_align - 1 ); + at += (int64_t) n * entry_size; + if ( inner == NULL ) { return true; } // no map below an entry: one depth is the whole term + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } // framing damage: the load reports it + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +// ---- the TEXT form's placement (docs/SPEC-TABLES.md §2.8, §16) ---- +// +// The text is a plain JSON object keyed by the KEY, and the generic walk fills +// it through the ENTRY'S OWN descriptor — so all it needs from here is one +// entry at one key, handed back at its defaults. It is the builder's Insert +// with the ENTRY returned rather than its value, because the walk writes the +// value through a field row and not through a typed pointer. +// +// THE ONE INSERTION PRIMITIVE. Lookup, reset, allocation and the KEY COPY are +// all here, so no caller mutates an entry this did not create and no caller +// relabels one it found. A key is copied only when an entry is created, which +// is what makes a duplicate key leave the identity it matched untouched. NULL +// is one thing and one thing only: the arena refused. +template +inline Entry * TableMapPlace( TableWorker & worker, TableMap & map, Key key ) +{ + if ( worker.arena == NULL ) { return NULL; } + Entry * found = TableMapScan( *worker.arena, map, key ); + if ( found != NULL ) + { + TableResetMapValue( *found ); // a repeated key is LAST-WINS, whole + return found; + } + TableMapHead * head = TableMapReach( worker, map ); + if ( head == NULL ) { return NULL; } + Entry * entry = TableMapAppend( worker, head, map ); + if ( entry == NULL ) { return NULL; } + TableReset( *entry ); + TableEntrySetKey( *entry, key ); + return entry; +} + +// ---- the OPTIONAL RUNTIME INDEX (§2.8) ---- +// +// Open addressing with LINEAR PROBING over the sorted array, built AT LOAD for +// a map large enough that log n compares over a cold array cost more than one +// hash and a probe. IT IS NEVER STORED: the caller measures it, owns its +// storage, builds it in one pass and releases it whenever. +// +// ITS HASH AND ITS LOAD FACTOR ARE NOT A CROSS-PORT CONTRACT, and that is a +// rule. What a port is held to is the CONTRACT of the lookup: the same value +// the sorted array's Find returns for the same key, and no allocation past the +// storage the caller handed in. +struct TableMapIndex +{ + int32_t * slots = NULL; // entry indices, +1; 0 is an empty slot + int32_t capacity = 0; + bool good = false; +}; + +// this runtime's own, and no port reproduces it: fnv1a64 over the key's bytes +inline uint64_t TableMapHash( const void * bytes, int32_t length ) +{ + uint64_t hash = 0xCBF29CE484222325ull; + const uint8_t * at = (const uint8_t *) bytes; + for ( int32_t i = 0; i < length; i++ ) { hash ^= (uint64_t) at[i]; hash *= 0x100000001B3ull; } + return hash; +} +inline uint64_t TableMapHash( uint64_t key ) { return TableMapHash( (const void *) &key, (int32_t) sizeof( key ) ); } + +// this runtime's own load factor, and no port reproduces it either: the next +// power of two at or above twice the count, so a probe run stays short +inline int32_t TableMapIndexSlots( int32_t count ) +{ + int32_t slots = 8; + while ( slots < count * 2 ) { slots *= 2; } + return slots; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_MAP + +#ifndef MAPDEMO_SCHEMA_TABLE_LIST +#define MAPDEMO_SCHEMA_TABLE_LIST + +namespace mapdemo { + +// ---- an UNBOUNDED ARRAY: a counted array whose count the data decides (§2.9) ---- +// +// On the wire, in a region and in a cook a list is the kind 14 body a [..N]T +// writes, its elements by-value records inside the holder's node extent. What +// this adds is the slot, a builder that appends into segments that never +// move, and a const surface that indexes and iterates in place. There is no +// sort, no key and no lookup: the order is INSERTION order, and it is +// identity the way position is identity in a fixed array. + +// elements carved from ONE call to the allocator pair. A new segment is +// appended when the current one fills, and nothing ever moves (§6.4) +static const int32_t kTableListSegmentElements = 32; + +// THE ELEMENT STORAGE: T itself, and a TableRef slot for a []*T, whose +// elements are references exactly as a pointer field's slot is (§2.1) +template struct TableListStorage { typedef T Element; }; +template struct TableListStorage { typedef TableRef Element; }; + +// WHAT THE CONST FORM ANSWERS: the element by reference, and on a []*T the +// RESOLVED pointer, one add on the self-relative delta, NULL for a null slot, +// exactly as At answers it (§6.2, §6.3) +template struct TableListConst +{ + typedef const T & Result; + static Result At( const T * element ) { return *element; } +}; +template struct TableListConst +{ + typedef const T * Result; + static Result At( const TableRef * element ) + { + return element->value != 0 ? (const T *) ( (const uint8_t *) element + element->value ) : NULL; + } +}; + +// ---- the storage: SIXTEEN BYTES in the holder's record (§2.9, §7.2) ---- +// +// An int64 self-relative reference to the element array and an int32 count, +// then padding to eight. The reference is a TableRef like a pointer's: in the +// arena it names the builder's HEAD, in a region it is the delta from the slot +// to the first element, and 0 is the empty list in both. It is the map's slot +// exactly, because it is the same two facts. +template struct TableList +{ + typedef typename TableListStorage::Element Element; + + TableRef elements; + int32_t count = 0; // the LIVE count, in both forms + int32_t padding = 0; // named, so the record has no unwritten byte in it + + // ---- the CONST form: a locked region, a loaded one, an opened cook ---- + // + // One surface over one encoding (§6.3). A region reference resolves from + // the slot's own address, so every one of these is a member and needs no + // base and no context. + const Element * Elements() const + { + return elements.value != 0 ? (const Element *) ( (const uint8_t *) &elements + elements.value ) : NULL; + } + int32_t size() const { return count; } + + // INDEXING IS BOUNDS-CHECKED IN EVERY BUILD (§2.4, §2.9): the extent is a + // number that CAME FROM A FILE, so an index past it is not a mistake a + // release build gets to make cheaply. There is no undefined-behavior path + // here in any configuration. The assert carries the message where a + // debugger can read it and NDEBUG removes that. The fatal is what stands + // after it. Both go through the hooks: define schema_assert and + // schema_fatal and this refusal lands in your own handler. + void RefuseIndex( int32_t index ) const + { + if ( (uint32_t) index >= (uint32_t) count ) + { + schema_assert( false && "an unbounded array is indexed inside its count, which came from a file" ); + schema_fatal(); + } + } + typename TableListConst::Result operator[]( int32_t index ) const + { + RefuseIndex( index ); + return TableListConst::At( Elements() + index ); + } + + // ---- iteration: INDEX order, the element and no key ---- + // + // It carries no iterator_traits, for the reason TableKeyed's does not + // (§13.9). + struct ConstIterator + { + const Element * at; + typename TableListConst::Result operator*() const { return TableListConst::At( at ); } + ConstIterator & operator++() { at++; return *this; } + bool operator==( const ConstIterator & other ) const { return at == other.at; } + bool operator!=( const ConstIterator & other ) const { return at != other.at; } + }; + + ConstIterator begin() const { return ConstIterator{ Elements() }; } + ConstIterator end() const { return ConstIterator{ Elements() + count }; } +}; + +// ---- the BUILDER's side: a head, and segments that never move (§2.9, §6.4) ---- +// +// The head is a small node in the arena holding the segment chain, the live +// count and the dead count, allocated when the first element is added. Each +// segment is a fixed number of elements carved from one call to the allocator +// pair. An element's address is stable for the arena's life, so a T * handed +// back by Add stays valid while other elements arrive. +struct TableListHead +{ + TableRef first; // the arena offset of the first segment + TableRef last; // and of the one an Add appends into + int32_t live; + int32_t dead; +}; + +template struct TableListSegment +{ + TableRef next; + int32_t used; // elements carved from this segment + int32_t padding; + uint32_t dead[ ( kTableListSegmentElements + 31 ) / 32 ]; // Erase marks one bit, never the element + Element elements[ kTableListSegmentElements ]; +}; + +inline bool TableListSegmentDead( const uint32_t * dead, int32_t index ) +{ + return ( dead[ index / 32 ] & ( 1u << ( index % 32 ) ) ) != 0; +} + +// the head, allocated when the first element is added +template +inline TableListHead * TableListReach( TableWorker & worker, TableList & list ) +{ + if ( worker.arena == NULL || worker.arena->locked ) { return NULL; } + if ( list.elements.value != 0 ) { return (TableListHead *) TableArenaAt( *worker.arena, (uint32_t) list.elements.value ); } + uint32_t at = 0; + TableListHead * head = (TableListHead *) worker.AllocRaw( (int64_t) sizeof( TableListHead ), (int64_t) alignof( TableListHead ), at ); + if ( head == NULL ) { return NULL; } + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + list.elements.value = (int64_t) at; + return head; +} + +// one element's storage, appended: the current segment when it has room, a +// new one carved from one call to the pair when it does not. NULL means NOT +// ADDED: an arena that cannot carve another segment, or a count at the int32 +// cap (§2.2, §2.9). +template +inline typename TableList::Element * TableListAppend( TableWorker & worker, TableListHead * head, TableList & list ) +{ + typedef typename TableList::Element Element; + if ( list.count >= INT32_MAX ) { return NULL; } // the int32 storage cap + TableListSegment * segment = NULL; + if ( head->last.value != 0 ) + { + segment = (TableListSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + if ( segment->used >= kTableListSegmentElements ) { segment = NULL; } + } + if ( segment == NULL ) + { + uint32_t at = 0; + segment = (TableListSegment *) worker.AllocRaw( (int64_t) sizeof( TableListSegment ), (int64_t) alignof( TableListSegment ), at ); + if ( segment == NULL ) { return NULL; } // the arena could not carve another segment + segment->next.value = 0; + segment->used = 0; + segment->padding = 0; + for ( int32_t i = 0; i < (int32_t) ( sizeof( segment->dead ) / sizeof( segment->dead[0] ) ); i++ ) { segment->dead[i] = 0; } + if ( head->last.value != 0 ) + { + TableListSegment * previous = (TableListSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + previous->next.value = (int64_t) at; + } + else + { + head->first.value = (int64_t) at; + } + head->last.value = (int64_t) at; + } + Element * element = segment->elements + segment->used; + segment->used++; + head->live++; + list.count++; + return element; +} + +// ADD, whole: the head, the append, and the element at its declared defaults +// (§2.9). The text form's placement is this same call, because a list has no +// key to place under (§16). +template +inline typename TableList::Element * TableListPlace( TableWorker & worker, TableList & list ) +{ + typedef typename TableList::Element Element; + TableListHead * head = TableListReach( worker, list ); + if ( head == NULL ) { return NULL; } + Element * element = TableListAppend( worker, head, list ); + if ( element == NULL ) { return NULL; } + new ( element ) Element(); // value-init: the declared defaults, and null for a slot + return element; +} + +// ERASE, ADDRESSED BY THE POINTER (§2.9): the element Add handed back is the +// handle, because a list has no key and the address is the one thing the +// builder promises never moves (§6.4). It marks the element DEAD, one bit in +// the segment's slot and not in the element storage, and decrements the live +// count. False when the pointer is not this list's. Its storage is reclaimed +// at RESET and never reused mid-build, the map's rule for the map's reason. +template +inline bool TableListErase( TableArena & arena, TableList & list, const typename TableList::Element * element ) +{ + typedef typename TableList::Element Element; + if ( list.elements.value == 0 || element == NULL ) { return false; } + TableListHead * head = (TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableListSegment * segment = (TableListSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + if ( element >= segment->elements && element < segment->elements + segment->used ) + { + const int32_t i = (int32_t) ( element - segment->elements ); + if ( TableListSegmentDead( segment->dead, i ) ) { return false; } // already erased + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + list.count--; + return true; + } + segment_ref = segment->next; + } + return false; +} + +// ---- iterate on the BUILDER: INDEX order, live elements only (§2.9) ---- +template struct TableListEach +{ + typedef typename TableList::Element Element; + const TableArena * arena; + TableRef first; + + struct Iterator + { + const TableArena * arena; + TableListSegment * segment; + int32_t index; + + void Skip() + { + for ( ;; ) + { + if ( segment == NULL ) { return; } + if ( index >= segment->used ) + { + segment = segment->next.value != 0 ? (TableListSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + index = 0; + continue; + } + if ( TableListSegmentDead( segment->dead, index ) ) { index++; continue; } + return; + } + } + Element * operator*() const { return segment->elements + index; } + Iterator & operator++() { index++; Skip(); return *this; } + bool operator==( const Iterator & other ) const { return segment == other.segment && index == other.index; } + bool operator!=( const Iterator & other ) const { return !( *this == other ); } + }; + + Iterator begin() const + { + Iterator it = { arena, first.value != 0 ? (TableListSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL, 0 }; + it.Skip(); + return it; + } + Iterator end() const { Iterator it = { arena, NULL, 0 }; return it; } +}; + +template +inline TableListEach TableListEachOf( const TableArena & arena, const TableList & list ) +{ + TableListEach each = { &arena, TableRef() }; + if ( list.elements.value != 0 ) + { + const TableListHead * head = (const TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + each.first = head->first; + } + return each; +} + +// ---- the INDEX-ORDER CURSOR the four writing walks read (§2.9) ---- +// +// Measure, Save, Lock and Cook each visit a list's live elements in the order +// they were added, and they allocate nothing to do it: a region's cursor is +// the array in place, and the builder's walks the segment chain. Indexing the +// builder's form is SEQUENTIAL by construction, every walk steps i, i + 1, +// i + 2, so the cursor remembers where the last access landed and moves one +// live slot per step. An access behind the memo restarts from the first +// segment, which no walk here does. +template struct TableListCursor +{ + const Element * elements = NULL; // the region's form: the array in place + const TableArena * arena = NULL; // the builder's form: the segments + TableRef first; + int32_t count = 0; + bool ok = false; + // the memo: the segment and slot the last access landed on, and the live + // index that slot holds + mutable const TableListSegment * segment = NULL; + mutable int32_t within = -1; + mutable int32_t logical = -1; + + const Element * At( int32_t index ) const + { + if ( elements != NULL ) { return elements + index; } + if ( segment == NULL || index < logical ) + { + segment = first.value != 0 ? (const TableListSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL; + within = -1; + logical = -1; + } + while ( logical < index ) + { + for ( ;; ) + { + within++; + while ( segment != NULL && within >= segment->used ) + { + segment = segment->next.value != 0 ? (const TableListSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + within = 0; + } + if ( segment == NULL ) { return NULL; } // the slot and the head disagree + if ( !TableListSegmentDead( segment->dead, within ) ) { break; } + } + logical++; + } + return segment->elements + within; + } + const Element & operator[]( int32_t index ) const { return *At( index ); } +}; + +// the REGION form: the array is the cursor +template +inline TableListCursor::Element> TableListElements( const TableRegionCtx &, const TableList & list ) +{ + TableListCursor::Element> cursor; + cursor.elements = list.Elements(); + cursor.count = list.count; + cursor.ok = true; + return cursor; +} + +// the BUILDER's form: the live elements out of the segment chain, in the +// order they were added. A dead element costs nothing on any wire (§2.9). +template +inline TableListCursor::Element> TableListElements( const TableArena & arena, const TableList & list ) +{ + TableListCursor::Element> cursor; + cursor.arena = &arena; + cursor.count = list.count; + if ( list.elements.value == 0 || list.count <= 0 ) { cursor.ok = list.count == 0; cursor.count = 0; return cursor; } + const TableListHead * head = (const TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + if ( head->live != list.count ) { return cursor; } // the slot and the head disagree: refused, never guessed + cursor.first = head->first; + cursor.ok = true; + return cursor; +} + +template +inline TableListCursor::Element> TableListElements( const TableArenaCtx & ctx, const TableList & list ) +{ + return TableListElements( *ctx.arena, list ); +} + +// ---- the LOAD side: where a decoded element lands (§2.9) ---- +// +// The same two shapes the map's fill takes, because the decoder above them +// cannot tell which it has: a REGION carves the element array out of the +// holder node's own extent, PRE-ORDER, and the TOOL's path appends into the +// builder's arena. +template struct TableListFill +{ + typedef typename TableList::Element Element; + TableList * list = NULL; + Element * array = NULL; // the region path: the carved array + int32_t capacity = 0; + TableWorker * worker = NULL; // the TOOL's path + bool ok = false; + bool refused = false; // a count above the int32 cap on the tool's path: LoadBuilder answers NULL +}; + +template +inline TableListFill TableListFillBegin( const TableNodeMap & nodes, TableList & list, uint64_t n ) +{ + typedef typename TableList::Element Element; + TableListFill fill; + fill.list = &list; + list.elements.value = 0; + list.count = 0; + if ( nodes.carve == NULL ) { return fill; } + if ( n > (uint64_t) INT32_MAX ) + { + // A COUNT ABOVE THE int32 STORAGE CAP (§2.2, §2.9): into a region it was + // refused by LoadMeasure before this ran, and into a builder it is the + // refusal LoadBuilder answers NULL for, moving no counter + fill.refused = nodes.carve->worker != NULL; + return fill; + } + if ( nodes.carve->worker != NULL ) + { + fill.worker = nodes.carve->worker; // the tool's path: the arena carves + fill.ok = true; + return fill; + } + const int64_t align = (int64_t) alignof( Element ); + uint8_t * base = (uint8_t *) ( ( (uintptr_t) nodes.carve->at + (uintptr_t) ( align - 1 ) ) & ~( (uintptr_t) ( align - 1 ) ) ); + const int64_t bytes = (int64_t) n * (int64_t) sizeof( Element ); + const int64_t pad = (int64_t) ( base - nodes.carve->at ); + if ( pad + bytes > nodes.carve->left ) { return fill; } // the measure and the load disagree: refused + nodes.carve->at = base + bytes; + nodes.carve->left -= pad + bytes; + fill.array = (Element *) base; + fill.capacity = (int32_t) n; + list.elements.value = (int64_t) ( base - (const uint8_t *) &list.elements ); + fill.ok = true; + return fill; +} + +// the next slot, at the element's declared defaults. NULL when the arena +// could not carve, which the decoder reports as framing damage +template inline typename TableList::Element * TableListFillNext( TableListFill & fill ) +{ + typedef typename TableList::Element Element; + if ( fill.array != NULL ) + { + if ( fill.list->count >= fill.capacity ) { return NULL; } + Element * element = fill.array + fill.list->count; + new ( element ) Element(); + fill.list->count++; + return element; + } + return TableListPlace( *fill.worker, *fill.list ); +} + +// A SLOT WHOSE ELEMENT NEVER LANDED is given back (§2.9, §4): the array keeps +// what it decoded, and an element whose own framing gave out before one byte +// of it decoded was not decoded. The region's form uncounts it, and the builder's +// marks it dead, which is what the storage rule allows mid-build. +template inline void TableListFillDrop( TableListFill & fill ) +{ + typedef typename TableList::Element Element; + if ( fill.array != NULL ) + { + if ( fill.list->count > 0 ) { fill.list->count--; } + return; + } + if ( fill.list->elements.value == 0 ) { return; } + TableListHead * head = (TableListHead *) TableArenaAt( *fill.worker->arena, (uint32_t) fill.list->elements.value ); + if ( head->last.value == 0 ) { return; } + TableListSegment * segment = (TableListSegment *) TableArenaAt( *fill.worker->arena, (uint32_t) head->last.value ); + if ( segment->used <= 0 ) { return; } + const int32_t i = segment->used - 1; + if ( TableListSegmentDead( segment->dead, i ) ) { return; } + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + fill.list->count--; +} + +// an EMPTY list's reference is null in both encodings, so a load that placed +// nothing leaves the slot exactly as a Reset does +template inline void TableListFillEnd( TableListFill & fill ) +{ + if ( fill.array != NULL && fill.list->count == 0 ) { fill.list->elements.value = 0; } +} + +// ---- LoadMeasure's term, from the FRAMING alone (§2.9, §6.5) ---- +// +// N x sizeof( T ) rounded to alignof( T ), AT EVERY DEPTH. N is framing and +// not a value, so this reads no field: it walks the list's own header and, +// where a table element holds a list or a map of its own, the elements' +// headers under it. Every -1 carries its REASON (§6.5): the int32 cap first, +// because a count past it cannot fit any body, and then the body's own L. +inline bool TableListWireExtent( const uint8_t * body, int64_t length, int64_t & at, + int64_t elem_size, int64_t elem_align, uint8_t elem_kind, int64_t elem_floor, + TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } // no array header: nothing rides + if ( r.get8() != elem_kind ) { return true; } // another element kind: §4's ordinary kind mismatch, the field reads empty + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + if ( n > (uint64_t) INT32_MAX ) { reason = count_over_extent_cap; return false; } + const int64_t rest = length - r.offset; + if ( n > (uint64_t) ( rest / elem_floor ) ) { reason = count_over_length; return false; } // an N the list's L cannot carry + at = ( at + elem_align - 1 ) & ~( elem_align - 1 ); + at += (int64_t) n * elem_size; + if ( inner == NULL ) { return true; } // nothing below an element: one depth is the whole term + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } // framing damage: the load reports it + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_LIST + +#ifndef MAPDEMO_SCHEMA_BUILD_VERSION +#define MAPDEMO_SCHEMA_BUILD_VERSION + +namespace mapdemo { + +// THE BUILD VERSION (docs/SPEC-TABLES.md §20): one digest over every fact the bytes +// this build produces depend on — the type wire's protocol id, every record's +// layout as the compiler's own C ABI model computes it, and the facts that +// decide what a load PUTS in those slots. It is the number a cook's header +// carries and the number Open compares, and the number a block's prologue +// carries and BlockOpen compares: a build version answers "which build?" and +// not "which form?", and what separates the two forms is their MAGIC. +// +// There are TWO ids in the design and they are not interchangeable: the +// PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is +// what everything cooked or blocked is keyed by. A table edit moves this and +// never the protocol id; a type edit moves both. +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_BUILD_VERSION + +#ifndef MAPDEMO_SCHEMA_TABLE_COOK +#define MAPDEMO_SCHEMA_TABLE_COOK + +namespace mapdemo { + +// ---- the cooked form (docs/SPEC-TABLES.md §7) ---- +// +// A cooked file is a HEADER, a DATA part and an ATTRIBUTION part, in that +// order. Every word of the header is a u64 written in the byte order the cook +// was produced in, and the header is 64 bytes: +// +// 0 magic 0x4b4f4f434d484353, read BYTEWISE before anything else +// 8 build_version the unit's id (docs/SPEC-TABLES.md §20) +// 16 byte_order 1 little, 2 big — the order that WROTE the file +// 24 data_length the region's bytes, rounded up to alignment +// 32 attribution_length the directory's bytes, or 0 +// 40 alignment the region's alignment, never below eight +// 48 reserved zero +// 56 reserved zero +// +// The DATA part is Lock's region written verbatim (§7.2) — the root at its +// base — and it is what a runtime points at. The ATTRIBUTION part is the node +// directory (§6.3), and NOTHING THAT READS THE STRUCTURE TOUCHES IT: it is +// written beside the data for schema cook-check, so a build that ships no +// tooling need not carry it at all. +static const int64_t kTableCookHeaderBytes = 64; + +// THE MAGIC'S VALUE, and a consumer written from the page needs the constant +// rather than a description of one. It is "SCHMCOOK" read as ASCII in the byte +// order a little-endian store produces — the same shape the block form's +// SCHMABLK takes, so a hex dump of a little-endian cook is legible and the two +// accelerators sit in one vocabulary. +// +// IT IS STORED IN THE PRODUCER'S ORDER, which is what makes it the byte-order +// check as well as the form check: a consumer reads back this build's +// constant, or that constant byte-reversed — which identifies a cook of the +// OTHER order — or something that is not a cook. All three answers but the +// first refuse, and a cook and a BLOCK are separated here too, because a +// form's identity belongs in its magic rather than in a second digest. +static const uint64_t TableCookMagic = 0x4b4f4f434d484353ull; + +// THIS BUILD's byte order, as the header's own word carries it. The magic is +// what REFUSES a foreign order; this word is what RECORDS which order wrote +// the file, so a refusal names the order rather than inferring it and a tool +// dumping a cook reads the fact. A file whose magic matched and whose order +// word did not is corrupt, and there is no reading that recovers it. +// +// The BUILD VERSION cannot do either job: §20.1 digests byteorder as a +// GENERATION input, little for every target schema generates for today, so +// two builds of one schema for two orders emit the same id. +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +static const uint64_t TableCookByteOrder = 2; // big +#else +static const uint64_t TableCookByteOrder = 1; // little +#endif + +// The greatest region alignment a cooked file may name. The DATA part begins +// at align_up( 64, alignment ), which is 64 for every unit this language can +// declare — the largest alignment it has is sixteen — so a word past this cap +// describes a file no build of this schema wrote (docs/SPEC-TABLES.md §7.1). +static const uint64_t TableCookMaxAlign = 64; + +// The header read, BYTEWISE. memcpy is the portable spelling of "these eight +// bytes, in this machine's order"; every compiler this repo builds under folds +// it to one load, and it is the only read in the whole of Open that is not a +// comparison. +inline uint64_t table_cook_read64( const uint8_t * p ) +{ + uint64_t v; + memcpy( &v, p, sizeof( v ) ); + return v; +} + +// TableCookOpen: THE WHOLE CHECK, in one place, because §7 states the +// enumeration once and every generated Open is that one enumeration plus +// its own root's two layout facts. +// +// THE CHECK, in order: the magic read bytewise, the byte order it establishes, +// the build version against this build's own, both RESERVED words zero, the +// region alignment the header names, the two part lengths against the length +// the caller passed — a truncated file and a file with trailing bytes are the +// same refusal — the root's own storage inside the data part, and the +// alignment of the base. +// +// AND THAT IS ALL OF IT. On a match the bytes ARE what this build wrote, in +// this build's layout and this build's byte order, so there is nothing to +// validate and nothing to fix up: the caller gets the root. Nothing per node +// happens here, which is what makes open O(1) in the file's size; a walk of +// any shape would forfeit that, and validating an untrusted file is schema +// cook-check's job and a person's decision (§7.4). +// +// EVERY NUMBER BELOW COMES OUT OF THE FILE, so the arithmetic is unsigned and +// each term is BOUNDED BEFORE IT IS ADDED: a forged length near 2^64 must +// refuse, and an addition that wrapped would be the defect the comparison +// after it was supposed to catch. Nothing past length is read on any path, +// including every refusing one. +// A REFUSAL NAMES ITSELF, beside the null (docs/SPEC-TABLES.md §7): the reason +// is written on the refusal path only, so a match costs nothing and a caller +// that passed no out-parameter pays nothing. +inline const uint8_t * TableCookRefuse( TableRefuseReason * reason, TableRefuseReason why ) +{ + if ( reason != NULL ) { *reason = why; } + return NULL; +} + +inline uint64_t table_cook_byteswap64( uint64_t v ) +{ + return ( v >> 56 ) | ( ( v >> 40 ) & 0xff00ull ) | ( ( v >> 24 ) & 0xff0000ull ) | ( ( v >> 8 ) & 0xff000000ull ) + | ( ( v << 8 ) & 0xff00000000ull ) | ( ( v << 24 ) & 0xff0000000000ull ) | ( ( v << 40 ) & 0xff000000000000ull ) + | ( v << 56 ); +} + +inline const uint8_t * TableCookOpen( const void * bytes, uint64_t length, uint64_t root_size, uint64_t root_align, TableRefuseReason * reason ) +{ + // a null buffer is the CALLER's defect, as an unaligned base is; a buffer + // shorter than the header has no header to read and is truncated + if ( bytes == NULL ) { return TableCookRefuse( reason, unaligned_base ); } + if ( length < (uint64_t) kTableCookHeaderBytes ) { return TableCookRefuse( reason, truncated ); } + const uint8_t * raw = (const uint8_t *) bytes; + // the MAGIC, bytewise and first: it is what establishes the byte order + // every other header word is read in, so nothing else may be read before + // it. A byte-reversed constant is a cook of the other order and refuses + // here, which is why the order never reaches a fix-up pass; anything else + // is not a cook at all, a BLOCK's magic included. + const uint64_t magic = table_cook_read64( raw ); + if ( magic != TableCookMagic ) + { + return TableCookRefuse( reason, magic == table_cook_byteswap64( TableCookMagic ) ? foreign_order : not_a_cook ); + } + // a byte-order word that contradicts its own magic describes no cook in + // EITHER order, so it shares the magic's own value (§7.1) + if ( table_cook_read64( raw + 16 ) != TableCookByteOrder ) { return TableCookRefuse( reason, not_a_cook ); } + if ( table_cook_read64( raw + 8 ) != BuildVersion ) { return TableCookRefuse( reason, wrong_build_version ); } + // the RESERVED words: a non-zero one means a writer used a form this build + // does not understand, and Open refuses rather than ignoring it. + if ( table_cook_read64( raw + 48 ) != 0 ) { return TableCookRefuse( reason, reserved_not_zero ); } + if ( table_cook_read64( raw + 56 ) != 0 ) { return TableCookRefuse( reason, reserved_not_zero ); } + const uint64_t data_length = table_cook_read64( raw + 24 ); + const uint64_t attribution_length = table_cook_read64( raw + 32 ); + const uint64_t alignment = table_cook_read64( raw + 40 ); + // THE ALIGNMENT WORD IS DATA, and it is the one header field the rest of + // the check does arithmetic WITH rather than only comparison against. A + // region's alignment is a power of two, never below eight (the floor that + // puts the attribution part on an eight-byte boundary without a second + // padding rule) and never past the cap above; a word that is none of those + // rounds nothing and aligns nothing, so it is refused before it is used, + // which is why bad_alignment precedes both truncated clauses (§7). + if ( alignment < 8 || alignment > TableCookMaxAlign ) { return TableCookRefuse( reason, bad_alignment ); } + if ( ( alignment & ( alignment - 1 ) ) != 0 ) { return TableCookRefuse( reason, bad_alignment ); } + // and it must be an alignment THE ROOT CAN SIT AT, since the root is at + // the region's base: both are powers of two, so "at least the root's" + // is one division. + if ( ( alignment % root_align ) != 0 ) { return TableCookRefuse( reason, bad_alignment ); } + // The DATA part begins at align_up( 64, alignment ). It is DERIVED and not + // a header field, because a fact a reader computes is a fact two writers + // cannot disagree about. + const uint64_t data_offset = ( (uint64_t) kTableCookHeaderBytes + alignment - 1 ) & ~( alignment - 1 ); + if ( length < data_offset ) { return TableCookRefuse( reason, truncated ); } + // the two part lengths against the length the caller passed. The whole + // file is data_offset + data_length + attribution_length, and a length + // that is not EXACTLY that refuses — truncation and trailing bytes are one + // refusal, and both terms are subtracted rather than added so no sum can + // carry. + if ( data_length > length - data_offset ) { return TableCookRefuse( reason, truncated ); } + if ( attribution_length != length - data_offset - data_length ) { return TableCookRefuse( reason, truncated ); } + // the ROOT sits at the region's base, so the region has to hold it: a + // shorter data part describes a root partly outside the file, which is the + // one way a match-and-point reader could hand back storage it never + // received. It is the second clause on truncated (§7). + if ( data_length < root_size ) { return TableCookRefuse( reason, truncated ); } + const uint8_t * base = raw + data_offset; + // the alignment of the BASE, LAST, because it is the only clause that reads + // nothing out of the file. The header pads the data part to the region's + // alignment, so a base an allocator or mmap gave you is already aligned — + // mmap gives page alignment for free — and a base that is not is a caller's + // buffer this form cannot be read out of: the caller's defect, not the file's. + if ( ( (uintptr_t) base % (uintptr_t) alignment ) != 0 ) { return TableCookRefuse( reason, unaligned_base ); } + return base; +} + +// ---- the cooked form, the WRITE side (docs/SPEC-TABLES.md §7.6) ---- +// +// THE BYTE ORDER IS THE TARGET'S, NOT THE HOST'S. A cook is produced in the +// byte order of the build that will read it (§7), so the fixing happens here — +// offline, once, on the writing side — and never at Open. Passing +// TableByteOrder::Big on a little-endian machine produces a big-endian build's +// file, and nothing about the writing host reaches the bytes. +enum class TableByteOrder +{ + Little = 1, // the header's byte_order word, and the order every scalar is written in + Big = 2, +}; + +// One store, width as an argument. Every call site passes a literal width, so +// the loop folds to a store (and a byte swap on the foreign order); a name per +// width would claim four §11 names to save nothing. +inline void table_cook_put( uint8_t * at, uint64_t value, int32_t width, TableByteOrder order ) +{ + if ( order == TableByteOrder::Little ) + { + for ( int32_t i = 0; i < width; i++ ) { at[i] = (uint8_t) ( value >> ( 8 * i ) ); } + } + else + { + for ( int32_t i = 0; i < width; i++ ) { at[i] = (uint8_t) ( value >> ( 8 * ( width - 1 - i ) ) ); } + } +} + +// A 128-bit store as two lanes: sixteen bytes, the low lane first in the +// little order and the high lane first — each lane big-endian — in the big +// order, exactly as a u64 is one lane of eight (docs/SPEC-TABLES.md §7.2). +inline void table_cook_put128( uint8_t * at, uint64_t lo, uint64_t hi, TableByteOrder order ) +{ + if ( order == TableByteOrder::Little ) { table_cook_put( at, lo, 8, order ); table_cook_put( at + 8, hi, 8, order ); } + else { table_cook_put( at, hi, 8, order ); table_cook_put( at + 8, lo, 8, order ); } +} + +// A buffer piece: the USED bytes and nothing else. The tail is already zero — +// the whole extent was zeroed before any field was written — so this copies the +// used prefix and leaves the rest, which is what makes a string's unused tail a +// consequence of one memset rather than a rule per buffer. A used length past +// the buffer, or below zero, is a value no reader could have produced and it is +// clamped rather than trusted: this writes inside the caller's buffer on every +// input. +inline void table_cook_bytes( uint8_t * at, const void * source, int64_t used, int64_t capacity ) +{ + if ( used <= 0 ) { return; } + const int64_t n = used < capacity ? used : capacity; + memcpy( at, source, (size_t) n ); +} + +// A WIDE TEXT buffer piece (docs/SPEC-TABLES.md §7.2): the USED code units, +// each a TWO-BYTE SCALAR in the cook's byte order. A record is written piece +// by piece and never memcpy'd, and a swap has to know where every scalar +// begins — a char16_t is one, so the units go one store each rather than as +// bytes. The tail is already zero, as the narrow twin's is, so the terminating +// zero unit at index used costs nothing here. +inline void table_cook_units( uint8_t * at, const char16_t * source, int64_t used, int64_t capacity, TableByteOrder order ) +{ + if ( used <= 0 ) { return; } + const int64_t n = used < capacity ? used : capacity; + for ( int64_t i = 0; i < n; i++ ) { table_cook_put( at + i * 2, (uint64_t) (uint16_t) source[i], 2, order ); } +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_COOK + +#ifndef MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE +#define MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE + +namespace mapdemo { + +// ---- the cooked form's WRITE side for a POINTERED root (docs/SPEC-TABLES.md §7.6) ---- +// +// A pointered root's cook is the region of §7.2: every node the numbering +// reached (§3.1), once, at its own type's alignment, in index order, the root +// at offset zero. This is that region while it is being laid out and written — +// the tool's own Layout and Write, in one struct. +// +// The OFFSETS are one per node, the root's zero at position 0 and node index k +// at position k - 1, which is the directory's own order (§6.3); they are the +// one allocation the write makes beyond the numbering, and they go through the +// same pair. A measure needs no offsets and leaves the pointer NULL. +struct TableCookRegion +{ + const TableNumbering * numbering = NULL; // node -> index, from the walk that placed it + int64_t * offsets = NULL; // index - 1 -> the node's region offset; NULL while measuring + int64_t count = 0; // nodes, the root included + int64_t bytes = 0; // the data part's length, rounded to align + int64_t align = 0; // the region's alignment: the nodes' greatest, never below eight + uint8_t * base = NULL; // where the data part is being written; NULL while measuring +}; + +// A reference slot: the SELF-RELATIVE delta from the slot's own address to the +// node's start (§6.3), and zero for null. The node is found by the address the +// numbering keyed it under, which is the same address the walk resolved through +// the same context — so a reference the numbering does not carry is a slot the +// walk never reached (a counted array's slot past its count, an absent +// optional's value) holding a node the region will not hold, and it is refused +// rather than written as a delta to nowhere. +inline bool table_cook_ref( const TableCookRegion & region, uint8_t * at, const void * pointee, TableByteOrder order ) +{ + if ( pointee == NULL ) { table_cook_put( at, 0, 8, order ); return true; } + uint64_t index = 0; + if ( !TableNumberingIndex( *region.numbering, pointee, index ) ) { return false; } + if ( index == 0 || index > (uint64_t) region.count ) { return false; } + const int64_t delta = region.offsets[index - 1] - (int64_t) ( at - region.base ); + table_cook_put( at, (uint64_t) delta, 8, order ); + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE + +namespace mapdemo { + +// table CrewsMembersEntry — TABLE-wire storage: relocatable, bounded, defaults in the +// member initializers (docs/SPEC-TABLES.md) +struct CrewsMembersEntry { + uint32_t key = 0; + TableRef value[2]; // [..2]*Item — used count beside it; every slot null until assigned + int32_t value_count = 0; +}; + +// table Crews — TABLE-wire storage: relocatable, bounded, defaults in the +// member initializers (docs/SPEC-TABLES.md) +struct Crews { + TableMap members; // map[uint32]*Item — the sorted entry array, empty until an insert + int32_t after = 0; +}; + +// ---- prefill: the declared defaults, in place (docs/SPEC-TABLES.md) ---- + +inline void CrewsMembersEntryReset( CrewsMembersEntry & value ); +inline void CrewsReset( Crews & value ); + +inline void CrewsMembersEntryReset( CrewsMembersEntry & value ) +{ + value.key = 0; + for ( int32_t i = 0; i < 2; i++ ) { value.value[i].value = 0; } // [2]*Item — every slot null + value.value_count = 0; +} + +inline void CrewsReset( Crews & value ) +{ + value.members.entries.value = 0; // map[uint32]*Item: empty + value.members.count = 0; + value.members.padding = 0; + value.after = 0; +} + +template inline int64_t CrewsMembersEntryMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const CrewsMembersEntry & value ); +template inline bool CrewsMembersEntrySaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const CrewsMembersEntry & value ); +inline bool CrewsMembersEntryLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, CrewsMembersEntry & value ); +template inline int64_t CrewsMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Crews & value ); +template inline bool CrewsSaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Crews & value ); +inline bool CrewsLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Crews & value ); +inline bool CrewsMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ); + +// CrewsMembersEntryMessageKeyRead: the key of one entry on the message wire, before the +// slot is chosen (docs/SPEC-TABLES.md §2.8, §3.3), and the bit the entry's +// body ends at. Field order inside a body is not contractual, so this scans +// the whole body by its announced shapes rather than assuming a position. +struct CrewsMembersEntryMessageKeyRead +{ + uint32_t key; + int64_t end; // the bit after the entry's own zero reference + bool found; // the body carried the key's id + bool kind_bad; // it carried it under another kind: the MAP's event + bool widened; // under a kind the declaration WIDENS (§4): decoded exactly, the MAP counts one + bool over; // longer than this reader's bound: the ENTRY is dropped + bool malformed; // the entry's framing gave out +}; + +inline CrewsMembersEntryMessageKeyRead CrewsMembersEntryMessageReadKey( TableBitReader r, const TableVocabulary & vocabulary, int64_t index_bits ) +{ + CrewsMembersEntryMessageKeyRead out = { 0, 0, false, false, false, false, false }; + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { out.malformed = true; return out; } + if ( ref == 0 ) { out.end = r.offset; return out; } // the terminator: no key field is the key's DEFAULT + if ( ref > (uint64_t) vocabulary.count ) { out.malformed = true; return out; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { out.malformed = true; return out; } + if ( entry.id == 0x3dc94a19365b10ecull ) // `key`, the ordinary hash of an ordinary name + { + const bool kind_bad = entry.kind != 8 && !TableKindWidens( entry.kind, 8 ); // THE KEY KIND IS THE READER'S DECLARATION + out.kind_bad = kind_bad; + out.found = !kind_bad; + out.widened = entry.kind != 8; + if ( kind_bad ) + { + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { out.malformed = true; return out; } + continue; + } + { + uint64_t raw = 0; + const int64_t width = entry.value_bits; + if ( width < 0 || !r.get( raw, width ) ) { out.malformed = true; return out; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + out.key = (uint32_t) decoded_wide; + } + continue; // the LAST occurrence is the one §3 keeps + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { out.malformed = true; return out; } + } +} + +// ---- the arena's reset hook (docs/SPEC-TABLES.md §6) ---- +// +// TableWorker::Alloc is a template and cannot name a member's Reset, so +// the arena reaches it through this overload set by argument-dependent +// lookup. It is how a node born in raw arena storage comes to hold the +// declared defaults without value-initialising the whole aggregate. + +inline void TableReset( CrewsMembersEntry & value ) { CrewsMembersEntryReset( value ); } +inline void TableReset( Crews & value ) { CrewsReset( value ); } + +// ---- pointer targets: allocation and resolution (docs/SPEC-TABLES.md §2) ---- +// +// A reference resolves differently in the two forms, and the CONTEXT says +// which: in the arena it is an offset; in a region it is a self-relative +// delta, so the const deref below is one add and needs no base pointer. + +// ---- codecs: measure/save/load per closure member ---- + +template inline int64_t CrewsMembersEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const CrewsMembersEntry & value ); +template inline bool CrewsMembersEntrySaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const CrewsMembersEntry & value ); +template inline bool CrewsMembersEntrySaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const CrewsMembersEntry & value ); +inline bool CrewsMembersEntryLoadBody( TableReader & r, const TableNodeMap & nodes, CrewsMembersEntry & value ); +template inline int64_t CrewsMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Crews & value ); +template inline bool CrewsSaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Crews & value ); +template inline bool CrewsSaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Crews & value ); +inline bool CrewsLoadBody( TableReader & r, const TableNodeMap & nodes, Crews & value ); + +// ---- pointer-graph walkers: number (measure/save), pack (Lock) ---- + +template inline bool CrewsMembersEntryNumber( const Ctx & ctx, TableNumbering & numbering, const CrewsMembersEntry & value ); +template inline int64_t CrewsMembersEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const CrewsMembersEntry & value ); +template inline bool CrewsMembersEntryPack( const Ctx & ctx, TablePackMap & seen, const CrewsMembersEntry & src, CrewsMembersEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); +template inline bool CrewsNumber( const Ctx & ctx, TableNumbering & numbering, const Crews & value ); +template inline int64_t CrewsPackMeasure( const Ctx & ctx, TablePackMap & seen, const Crews & value ); +template inline bool CrewsPack( const Ctx & ctx, TablePackMap & seen, const Crews & src, Crews & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +// ---- the numbering's bridge to each member's codec (docs/SPEC-TABLES.md §3.1) ---- + +template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const CrewsMembersEntry & value ) { return CrewsMembersEntryMeasureBody( ctx, numbering, ids, value ); } +template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const CrewsMembersEntry & value ) { return CrewsMembersEntrySaveBody( ctx, numbering, w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const CrewsMembersEntry & value ) { return CrewsMembersEntryMeasureMessageBody( ctx, numbering, index_bits, at, value ); } +template inline bool TableNodeMessageSave( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const CrewsMembersEntry & value ) { return CrewsMembersEntrySaveMessageBody( ctx, numbering, index_bits, w, value ); } +template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Crews & value ) { return CrewsMeasureBody( ctx, numbering, ids, value ); } +template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Crews & value ) { return CrewsSaveBody( ctx, numbering, w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Crews & value ) { return CrewsMeasureMessageBody( ctx, numbering, index_bits, at, value ); } +template inline bool TableNodeMessageSave( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Crews & value ) { return CrewsSaveMessageBody( ctx, numbering, index_bits, w, value ); } + +// ---- CrewsMembersEntry: the order, the key and the value (docs/SPEC-TABLES.md §2.8) ---- +// +// The four overloads the map runtime's templates reach by argument-dependent +// lookup. Nothing outside this file names them. +static_assert( alignof( CrewsMembersEntry ) <= kTableAlign, "a map entry's alignment must fit the arena's" ); + +inline int TableEntryOrder( const CrewsMembersEntry & a, const CrewsMembersEntry & b ) +{ + return TableKeyOrder( (uint64_t) a.key, (uint64_t) b.key ); // integers compare by VALUE, unsigned here +} +inline int TableEntryOrder( const CrewsMembersEntry & entry, uint32_t key ) +{ + return TableKeyOrder( (uint64_t) entry.key, (uint64_t) key ); +} +inline uint32_t TableEntryKey( const CrewsMembersEntry & entry ) { return entry.key; } +inline void TableEntrySetKey( CrewsMembersEntry & entry, uint32_t key ) { entry.key = key; } +// THIS VALUE'S STORAGE IS A PAIR (§2.8, §4.2, §7.2): `value` beside +// `value_count`, so the handle is the ENTRY and not one member of it. +// Fill both; `key` belongs to the map's own order. +inline const CrewsMembersEntry * TableEntryFound( const CrewsMembersEntry * entry ) { return entry; } +inline CrewsMembersEntry * TableEntryValue( CrewsMembersEntry * entry ) { return entry; } +struct CrewsMembersEntryEach { uint32_t key; decltype( TableEntryValue( (CrewsMembersEntry *) NULL ) ) value; }; +inline CrewsMembersEntryEach TableEntryEach( CrewsMembersEntry * entry ) { return CrewsMembersEntryEach{ TableEntryKey( *entry ), TableEntryValue( entry ) }; } +inline void TableResetMapValue( CrewsMembersEntry & value ) +{ + for ( int32_t i = 0; i < 2; i++ ) { value.value[i].value = 0; } // [2]*Item — every slot null + value.value_count = 0; +} + +// CrewsMembersEntryReadKey: the key, before the slot is chosen (docs/SPEC-TABLES.md §2.8). +// Field order inside a body is not contractual (§3), so this scans rather +// than assuming a position — and this implementation writes the key first, +// so on any wire it wrote the scan ends at the first header. +struct CrewsMembersEntryKeyRead +{ + uint32_t key; + bool found; // the body carried the key's id + bool kind_bad; // it carried it under another kind: the MAP's event + bool widened; // under a kind the declaration WIDENS (§4): decoded exactly, the MAP counts one + bool over; // longer than this reader's bound: the ENTRY is dropped + bool malformed; // the entry's framing gave out +}; + +inline CrewsMembersEntryKeyRead CrewsMembersEntryReadKey( const uint8_t * body, int64_t length, const TableIdTable * ids ) +{ + CrewsMembersEntryKeyRead out = { 0, false, false, false, false, false }; + TableReport scratch; // the scan's own framing damage is the MAP's, raised by the caller + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { out.malformed = true; return out; } + if ( field_ref == 0 ) { return out; } // the terminator: no key field is the key's DEFAULT + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { out.malformed = true; return out; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { out.malformed = true; return out; } + uint8_t field_kind = r.get8(); + if ( field_id == 0x3dc94a19365b10ecull ) // `key`, the ordinary hash of an ordinary name + { + if ( field_kind != 8 && TableKindWidens( field_kind, 8 ) ) + { + out.widened = true; + out.found = true; + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, field_kind, widened_v ) ) { out.malformed = true; return out; } + out.key = (uint32_t) widened_v; + continue; // the LAST occurrence is the one §3 keeps + } + out.kind_bad = field_kind != 8; // THE KEY KIND IS THE READER'S DECLARATION + out.found = !out.kind_bad; + if ( !out.kind_bad ) + { + if ( !r.has( 4 ) ) { out.malformed = true; return out; } + out.key = (uint32_t) r.get32(); + continue; // the LAST occurrence is the one §3 keeps + } + } + if ( !r.skip( field_kind ) ) { out.malformed = true; return out; } + } +} + +// ---- retain-unknown: the second family (docs/SPEC-TABLES.md §6.6) ---- +// +// The same walks, with the PATH threaded and the unknown arm capturing. The +// three above are untouched and cost nothing for these being here: a caller +// that does not ask instantiates none of them. + +template inline int64_t CrewsMembersEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool CrewsMembersEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool CrewsMembersEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool CrewsMembersEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline int64_t CrewsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool CrewsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool CrewsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ); +inline bool CrewsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Crews & value, TableRetain * retain, const TableRetainPath & path ); + +template +inline int64_t CrewsMembersEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const CrewsMembersEntry & value ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + if ( value.value_count < 0 || value.value_count > 2 ) { return -1; } // storage invariant + if ( value.value_count > 0 ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( value.value_count ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < value.value_count; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return -1; } + body_value += TableLebBytes( slot_index ); + } + } + bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); // value: [..2]*Item + } + return bytes; +} + +template +inline bool CrewsMembersEntrySaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const CrewsMembersEntry & value ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 8 ); // key + w.put32( uint32_t( value.key ) ); + } + if ( value.value_count < 0 || value.value_count > 2 ) { return false; } // storage invariant + if ( value.value_count > 0 ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( value.value_count ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < value.value_count; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + body_value += TableLebBytes( slot_index ); + } + } + w.putleb( ref_value ); w.put8( 14 ); w.putleb( (uint64_t) body_value ); // value + w.put8( 17 ); w.putleb( (uint64_t) ( value.value_count ) ); + for ( int32_t elem_i = 0; elem_i < value.value_count; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + w.putleb( slot_index ); + } + } + } + return !w.overflow; +} + +template +inline bool CrewsMembersEntrySaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const CrewsMembersEntry & value ) +{ + if ( !CrewsMembersEntrySaveBodyFields( ctx, numbering, w, ids, value ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool CrewsMembersEntryLoadBody( TableReader & r, const TableNodeMap & nodes, CrewsMembersEntry & value ) +{ + CrewsMembersEntryReset( value ); // prefill declared defaults in place, then overlay + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x3dc94a19365b10ecull: // key + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.key = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.key = decoded_v; + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER — the element kind byte and the + // count, so fewer than two bytes — is INERT (§4): the field keeps the + // value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + // A DAMAGED COUNT stops the elements and nothing else: the field + // RODE, so an optional is still PRESENT (§2.3) — only a foreign + // ELEMENT KIND says the payload is not this array's at all. + if ( !counted_ok ) { r.report->malformed = true; } + else if ( elem_kind != 17 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + uint64_t keep = count; + if ( keep > 2 ) { keep = 2; r.report->clamped++; } + // elements are BOUNDED by the field body: a count the length + // cannot cover keeps the decoded prefix, flags malformed, and + // the parent continues at the next field — following fields' + // bytes are never fabricated into elements + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint64_t decoded = 0; + for ( uint64_t i = 0; i < keep; i++ ) + { + { + uint64_t node_index = 0; + if ( !sub.getleb( node_index ) ) { r.report->malformed = true; break; } + TableNodeResolve( nodes, value.value[(int32_t) i], node_index, 0x52cfa1d198476806ull, r.report ); // *Item + } + decoded = i + 1; + } + value.value_count = (int32_t) decoded; + } + } + r.offset = body_end; // excess elements and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's cost, in BITS (docs/SPEC-TABLES.md §3.3). `at` is the +// body's own bit position in the batch, because a `string(N)` ALIGNS before +// its bytes and an align costs what the position says it costs. +template +inline int64_t CrewsMembersEntryMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const CrewsMembersEntry & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + int64_t bits = 0; + if ( value.key != 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + } + if ( value.value_count < 0 || value.value_count > 2 ) { return -1; } // storage invariant + if ( value.value_count > 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 2; + bits += (int64_t) ( value.value_count ) * index_bits; + } + bits += kTableMessageRefBitsHere; // the ZERO REFERENCE that ends the body + (void) at; + return bits; +} + +// The BITPACKED body: the fields, then the ZERO REFERENCE that ends it. No +// kind byte rides at all, and no length frames a nested body, because a +// body is self-delimiting: it is written where the file form put an L. +template +inline bool CrewsMembersEntrySaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const CrewsMembersEntry & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + if ( value.key != 0 ) + { + w.put( 9, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.key ), 32 ); + } + if ( value.value_count < 0 || value.value_count > 2 ) { return false; } // storage invariant + if ( value.value_count > 0 ) + { + w.put( 21, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.value_count ) - 0, 2 ); + for ( int32_t i = 0; i < value.value_count; i++ ) + { + const Item * pointee_value = ItemAt( ctx, value.value[i] ); // *Item + uint64_t index_value = 0; + if ( pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) pointee_value, index_value ) ) { return false; } + w.put( index_value, index_bits ); + } + } + w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +inline bool CrewsMembersEntryLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, CrewsMembersEntry & value ) +{ + (void) nodes; (void) index_bits; + CrewsMembersEntryReset( value ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 2 ) { kept = 2; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + TableRef scratch; + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( in_bounds ? value.value[i] : scratch ), node_index_2, 0x52cfa1d198476806ull, report ); // *Item + } + } + value.value_count = kept; + } + break; + } + default: + report->unknown++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + } +} + +template +inline int64_t CrewsMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Crews & value ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // members: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_members = TableMapOrder( ctx, value.members ); + if ( !order_members.ok ) { return -1; } // the sort could not run + if ( order_members.count > 0 ) + { + const uint64_t ref_members = ids.ref( 0x79d594675e391090ull ); + int64_t body_members = 1 + TableLebBytes( (uint64_t) order_members.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_members = CrewsMembersEntryMeasureBody( ctx, numbering, ids, *order_members[i] ); + if ( elem_members < 0 ) { TableMapRelease( order_members ); return -1; } + body_members += TableLebBytes( (uint64_t) ( elem_members ) ) + ( elem_members ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_members ) + 1 + TableLebBytes( (uint64_t) ( body_members ) ) + ( body_members ); + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + return bytes; +} + +template +inline bool CrewsSaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Crews & value ) +{ + { + TableMapCursor order_members = TableMapOrder( ctx, value.members ); // members + if ( !order_members.ok ) { return false; } + if ( order_members.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_members = ids.ref( 0x79d594675e391090ull ); + int64_t body_members = 1 + TableLebBytes( (uint64_t) order_members.count ); + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_members = CrewsMembersEntryMeasureBody( ctx, numbering, ids, *order_members[i] ); + if ( elem_members < 0 ) { TableMapRelease( order_members ); return false; } + body_members += TableLebBytes( (uint64_t) ( elem_members ) ) + ( elem_members ); + } + w.putleb( ref_members ); w.put8( 14 ); w.putleb( (uint64_t) body_members ); + w.put8( 13 ); w.putleb( (uint64_t) order_members.count ); + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_len_members = CrewsMembersEntryMeasureBody( ctx, numbering, ids, *order_members[i] ); + if ( elem_len_members < 0 ) { TableMapRelease( order_members ); return false; } + w.putleb( (uint64_t) elem_len_members ); + if ( !CrewsMembersEntrySaveBody( ctx, numbering, w, ids, *order_members[i] ) ) { TableMapRelease( order_members ); return false; } + } + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + return !w.overflow; +} + +template +inline bool CrewsSaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Crews & value ) +{ + if ( !CrewsSaveBodyFields( ctx, numbering, w, ids, value ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool CrewsLoadBody( TableReader & r, const TableNodeMap & nodes, Crews & value ) +{ + CrewsReset( value ); // prefill declared defaults in place, then overlay + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x79d594675e391090ull: // members + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + TableMapFill fill = TableMapFillBegin( nodes, value.members, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + CrewsMembersEntryKeyRead read = CrewsMembersEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + CrewsMembersEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + CrewsMembersEntryLoadBody( elem, nodes, *slot ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's cost, in BITS (docs/SPEC-TABLES.md §3.3). `at` is the +// body's own bit position in the batch, because a `string(N)` ALIGNS before +// its bytes and an align costs what the position says it costs. +template +inline int64_t CrewsMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Crews & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + int64_t bits = 0; + { + TableMapCursor order_members = TableMapOrder( ctx, value.members ); // members + if ( !order_members.ok ) { return -1; } // the sort could not run + if ( order_members.count > 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; // the count the data decides + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_members = CrewsMembersEntryMeasureMessageBody( ctx, numbering, index_bits, at + bits, *order_members[i] ); + if ( elem_members < 0 ) { TableMapRelease( order_members ); return -1; } + bits += elem_members; // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + } + bits += kTableMessageRefBitsHere; // the ZERO REFERENCE that ends the body + (void) at; + return bits; +} + +// The BITPACKED body: the fields, then the ZERO REFERENCE that ends it. No +// kind byte rides at all, and no length frames a nested body, because a +// body is self-delimiting: it is written where the file form put an L. +template +inline bool CrewsSaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Crews & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + { + TableMapCursor order_members = TableMapOrder( ctx, value.members ); // members + if ( !order_members.ok ) { return false; } // the sort could not run + if ( order_members.count > 0 ) + { + w.put( 20, kTableMessageRefBitsHere ); + w.put( (uint64_t) order_members.count, 32 ); // the count the data decides + for ( int32_t i = 0; i < order_members.count; i++ ) + { + if ( !CrewsMembersEntrySaveMessageBody( ctx, numbering, index_bits, w, *order_members[i] ) ) { TableMapRelease( order_members ); return false; } + } + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) + { + w.put( 18, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.after ), 32 ); + } + w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +// CrewsMessageExtent: the extent Crews's maps command on the message wire, from +// the FRAMING alone (docs/SPEC-TABLES.md §2.8, §3.3, §6.5). +inline bool CrewsMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { return false; } + if ( entry.id == 0x79d594675e391090ull && entry.kind == 14 && entry.elem_kind == 13 ) // members + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { return false; } + n += (uint64_t) entry.min; + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( CrewsMembersEntry ) + at += (int64_t) n * (int64_t) sizeof( CrewsMembersEntry ); // the whole array FIRST + for ( uint64_t i = 0; i < n; i++ ) // then, entry by entry in key order + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { return false; } + } + continue; + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { return false; } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +inline bool CrewsLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Crews & value ) +{ + (void) nodes; (void) index_bits; + CrewsReset( value ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x79d594675e391090ull: // members + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + TableMapFill fill = TableMapFillBegin( nodes, value.members, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const CrewsMembersEntryMessageKeyRead read = CrewsMembersEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + CrewsMembersEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !CrewsMembersEntryLoadMessageBody( r, vocabulary, report, nodes, index_bits, *slot ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + } +} + +// CrewsMembersEntryWireExtent: the extent CrewsMembersEntry's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool CrewsMembersEntryWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + (void) body; (void) length; (void) at; (void) ids; (void) reason; // no list or map below this record + return true; +} + +// CrewsMembersEntryExtentAt: the node extent CrewsMembersEntry's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as CrewsMembersEntryExtentPack advances it (§2.8, §2.9). +template +inline bool CrewsMembersEntryExtentAt( const Ctx & ctx, const CrewsMembersEntry & value, int64_t & at ) +{ + (void) ctx; (void) value; (void) at; // no list or map below this record + return true; +} + +template +inline int64_t CrewsMembersEntryExtent( const Ctx & ctx, const CrewsMembersEntry & value ) +{ + (void) ctx; (void) value; // no list or map below this record + return 0; +} + +// CrewsMembersEntryExtentPack: carve CrewsMembersEntry's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset CrewsMembersEntryExtentAt advances (§2.8, §2.9). +template +inline bool CrewsMembersEntryExtentPack( const Ctx & ctx, const CrewsMembersEntry & src, CrewsMembersEntry & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + (void) ctx; (void) src; (void) dst; (void) extent; (void) at; (void) capacity; // no list or map below this record + return true; +} + +// CrewsWireExtent: the extent Crews's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool CrewsWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; // the scan's framing damage is the LOAD's to report + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { return true; } + if ( field_ref == 0 ) { return true; } + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { return true; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { return true; } + uint8_t field_kind = r.get8(); + if ( field_id == 0x79d594675e391090ull && field_kind == 14 ) // members + { + uint64_t map_len = 0; + if ( !r.getleb( map_len ) || !r.room( map_len ) ) { return true; } + const uint8_t * map_body = r.buffer + r.offset; + r.offset += (int64_t) map_len; + if ( !TableMapWireExtent( map_body, (int64_t) map_len, at, (int64_t) sizeof( CrewsMembersEntry ), (int64_t) alignof( CrewsMembersEntry ), NULL, ids, reason ) ) { return false; } + continue; + } + if ( !r.skip( field_kind ) ) { return true; } + } +} + +// CrewsExtentAt: the node extent Crews's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as CrewsExtentPack advances it (§2.8, §2.9). +template +inline bool CrewsExtentAt( const Ctx & ctx, const Crews & value, int64_t & at ) +{ + { + TableMapCursor cursor = TableMapOrder( ctx, value.members ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( CrewsMembersEntry ) + at += (int64_t) cursor.count * (int64_t) sizeof( CrewsMembersEntry ); // the whole array FIRST + for ( int32_t i = 0; i < cursor.count; i++ ) // then, entry by entry in key order + { + if ( !CrewsMembersEntryExtentAt( ctx, *cursor[i], at ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// the whole extent of one node, from a fresh offset: what a pack reserves +// for it beside the record's own storage. +template +inline int64_t CrewsExtent( const Ctx & ctx, const Crews & value ) +{ + int64_t at = 0; + if ( !CrewsExtentAt( ctx, value, at ) ) { return -1; } + return at; +} + +// CrewsExtentPack: carve Crews's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset CrewsExtentAt advances (§2.8, §2.9). +template +inline bool CrewsExtentPack( const Ctx & ctx, const Crews & src, Crews & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + { + TableMapCursor cursor = TableMapOrder( ctx, src.members ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; + const int64_t bytes = (int64_t) cursor.count * (int64_t) sizeof( CrewsMembersEntry ); + if ( at + bytes > capacity ) { TableMapRelease( cursor ); return false; } + CrewsMembersEntry * placed = (CrewsMembersEntry *) ( extent + at ); + at += bytes; + dst.members.count = cursor.count; + dst.members.padding = 0; + dst.members.entries.value = cursor.count > 0 ? (int64_t) ( (uint8_t *) placed - (const uint8_t *) &dst.members.entries ) : 0; + for ( int32_t i = 0; i < cursor.count; i++ ) + { + memcpy( (void *) ( placed + i ), (const void *) cursor[i], sizeof( CrewsMembersEntry ) ); // trivially copyable, by construction + } + for ( int32_t i = 0; i < cursor.count; i++ ) + { + if ( !CrewsMembersEntryExtentPack( ctx, *cursor[i], placed[i], extent, at, capacity ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// ---- Crews.members: the builder's five and the side index (§2.8) ---- + +// INSERT: the key is copied, the value is handed back at its defaults to +// fill. A DUPLICATE key REPLACES — the value is reset and the same entry +// handed back, key and address unchanged — so a caller that wants to know +// writes Find first. NULL is NOT INSERTED: a key longer than the bound, +// because a truncated key would be a merged entry, and an arena that +// cannot carve another segment, alike. +// +// It is a WRAPPER: TableMapPlace owns the lookup, the reset, the +// allocation and the key copy, and this half is the bound and the +// const char * key's length. Nothing here mutates an entry (§2.8). +inline CrewsMembersEntry * CrewsMembersInsert( TableWorker & worker, TableMap & map, uint32_t key ) +{ + CrewsMembersEntry * entry = TableMapPlace( worker, map, key ); + return entry != NULL ? TableEntryValue( entry ) : NULL; +} + +// FIND on the builder: the same linear scan, O( n ) key compares over the +// segments in insertion order. NULL when absent. The builder builds NO +// INDEX, and that is a rule — the sort happens once, at Lock, Save or +// Cook, and every lookup that matters runs over the sorted region. +inline CrewsMembersEntry * CrewsMembersFind( TableArena & arena, TableMap & map, uint32_t key ) +{ + CrewsMembersEntry * found = TableMapScan( arena, map, key ); + return found != NULL ? TableEntryValue( found ) : NULL; +} + +// ERASE: marks the entry DEAD, one bit in the segment's slot and not in the +// entry table. False when absent. Its storage is held until the builder +// resets and never reused mid-build, because reusing a slot would make "an +// entry's address is stable" false for exactly one case. +inline bool CrewsMembersErase( TableArena & arena, TableMap & map, uint32_t key ) +{ + return TableMapErase( arena, map, key ); +} + +// EACH on the builder: INSERTION order, live entries only. +inline TableMapEach CrewsMembersEach( const TableArena & arena, const TableMap & map ) +{ + return TableMapEachOf( arena, map ); +} + +// ---- the OPTIONAL INDEX: caller-owned, built at load, never stored ---- +// +// Open addressing with linear probing over the sorted array, for a map large +// enough that log n compares over a cold array cost more than one hash and a +// probe. ITS HASH AND ITS LOAD FACTOR ARE NOT A CROSS-PORT CONTRACT: the +// index is never stored, so no golden, no cook-check rule and no +// build-version line ever names either. What a port is held to is the +// CONTRACT of the lookup — the same value the sorted array's Find returns +// for the same key, and no allocation past the storage the caller handed in. +inline int64_t CrewsMembersIndexMeasure( const TableMap & map ) +{ + return (int64_t) TableMapIndexSlots( map.count ) * (int64_t) sizeof( int32_t ); +} + +inline TableMapIndex CrewsMembersIndex( const TableMap & map, void * storage, int64_t bytes ) +{ + TableMapIndex index; + const int32_t slots = TableMapIndexSlots( map.count ); + if ( storage == NULL || bytes < (int64_t) slots * (int64_t) sizeof( int32_t ) ) { return index; } + index.slots = (int32_t *) storage; + index.capacity = slots; + for ( int32_t i = 0; i < slots; i++ ) { index.slots[i] = 0; } + const CrewsMembersEntry * entries = map.Entries(); + for ( int32_t i = 0; i < map.count; i++ ) // ONE PASS over the sorted array + { + int32_t at = (int32_t) ( TableMapHash( (uint64_t) entries[i].key ) & (uint64_t) ( slots - 1 ) ); + while ( index.slots[at] != 0 ) { at = ( at + 1 ) & ( slots - 1 ); } + index.slots[at] = i + 1; // slots are ENTRY INDICES; 0 is an empty slot + } + index.good = true; + return index; +} + +inline const CrewsMembersEntry * CrewsMembersIndexFind( const TableMapIndex & index, const TableMap & map, uint32_t key ) +{ + if ( !index.good ) { return map.Find( key ); } // an index that did not build is not a wrong answer + const CrewsMembersEntry * entries = map.Entries(); + int32_t at = (int32_t) ( TableMapHash( (uint64_t) key ) & (uint64_t) ( index.capacity - 1 ) ); + for ( int32_t probe = 0; probe < index.capacity; probe++ ) + { + const int32_t slot = index.slots[at]; + if ( slot == 0 ) { return NULL; } + if ( TableEntryOrder( entries[slot - 1], key ) == 0 ) { return TableEntryFound( entries + slot - 1 ); } + at = ( at + 1 ) & ( index.capacity - 1 ); + } + return NULL; +} + +// CrewsMembersEntryNumber: number everything CrewsMembersEntry POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool CrewsMembersEntryNumber( const Ctx & ctx, TableNumbering & numbering, const CrewsMembersEntry & value ) +{ + for ( int32_t k = 0; k < value.value_count && k < 2; k++ ) // value: [..2]*Item + { + { + const Item * pointee = ItemAt( ctx, value.value[k] ); // value + if ( pointee != NULL ) + { + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( numbering.seen, (const void *) pointee, + (int64_t) ( numbering.count + 2 ), taken, slot ); // its index, if this is its first visit + if ( entry == NULL ) { return false; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return false; } // a data cycle + } + else + { + TableNodeEntry node; + node.node = (const void *) pointee; + node.type_id = 0x52cfa1d198476806ull; // fnv1a64( "Item" ) + node.type_slot = 65; // its slot in the unit's vocabulary (§3.3) + node.measure = &TableNodeMeasureThunk; + node.save = &TableNodeSaveThunk; + node.message_measure = &TableNodeMessageMeasureThunk; + node.message_save = &TableNodeMessageSaveThunk; + if ( !TableNumberingAppend( numbering, node ) ) { return false; } + if ( !ItemNumber( ctx, numbering, *pointee ) ) { return false; } + TablePackMapClose( numbering.seen, (const void *) pointee, slot ); + } + } + } + } + return true; +} + +// CrewsMembersEntryPackMeasure: the packed region bytes of everything CrewsMembersEntry POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t CrewsMembersEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const CrewsMembersEntry & value ) +{ + int64_t bytes = 0; + for ( int32_t k = 0; k < value.value_count && k < 2; k++ ) // value: [..2]*Item + { + { + const Item * pointee = ItemAt( ctx, value.value[k] ); // value + if ( pointee != NULL ) + { + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( seen, (const void *) pointee, 0, taken, slot ); + if ( entry == NULL ) { return -1; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return -1; } // a data cycle + } + else + { + int64_t inner = ItemPackMeasure( ctx, seen, *pointee ); + if ( inner < 0 ) { return -1; } + TablePackMapClose( seen, (const void *) pointee, slot ); + bytes += TableAlignUp64( (int64_t) sizeof( Item ) ) + inner; + } + } + } + } + return bytes; +} + +// CrewsMembersEntryPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool CrewsMembersEntryPackEdges( const Ctx & ctx, TablePackMap & seen, const CrewsMembersEntry & src, CrewsMembersEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool CrewsMembersEntryPack( const Ctx & ctx, TablePackMap & seen, const CrewsMembersEntry & src, CrewsMembersEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( CrewsMembersEntry ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( CrewsMembersEntry ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !CrewsMembersEntryExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return CrewsMembersEntryPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool CrewsMembersEntryPackEdges( const Ctx & ctx, TablePackMap & seen, const CrewsMembersEntry & src, CrewsMembersEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + for ( int32_t k = 0; k < src.value_count && k < 2; k++ ) // value: [..2]*Item + { + { + dst.value[k].value = 0; // value + const Item * pointee = ItemAt( ctx, src.value[k] ); + if ( pointee != NULL ) + { + int64_t at = TableAlignUp64( used ); // where it WOULD land, if this is its first visit + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( seen, (const void *) pointee, at, taken, slot ); + if ( entry == NULL ) { return false; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return false; } // a data cycle + dst.value[k].value = (int64_t) ( ( base + entry->offset ) - (const uint8_t *) &dst.value[k] ); // the one body it already has + } + else + { + if ( at + (int64_t) sizeof( Item ) > capacity ) { return false; } + used = at + TableAlignUp64( (int64_t) sizeof( Item ) ); + Item * child = new ( base + at ) Item; // lifetime only: the Pack below memcpy's the whole node over it + dst.value[k].value = (int64_t) ( ( base + at ) - (const uint8_t *) &dst.value[k] ); + if ( !ItemPack( ctx, seen, *pointee, *child, base, capacity, used ) ) { return false; } + TablePackMapClose( seen, (const void *) pointee, slot ); + } + } + } + } + return true; +} + +// CrewsNumber: number everything Crews POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool CrewsNumber( const Ctx & ctx, TableNumbering & numbering, const Crews & value ) +{ + { // members: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_members = TableMapOrder( ctx, value.members ); + if ( !cursor_members.ok ) { return false; } + for ( int32_t i = 0; i < cursor_members.count; i++ ) + { + if ( !CrewsMembersEntryNumber( ctx, numbering, *cursor_members[i] ) ) { TableMapRelease( cursor_members ); return false; } + } + TableMapRelease( cursor_members ); + } + return true; +} + +// CrewsPackMeasure: the packed region bytes of everything Crews POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t CrewsPackMeasure( const Ctx & ctx, TablePackMap & seen, const Crews & value ) +{ + int64_t bytes = 0; + { // members: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_members = TableMapOrder( ctx, value.members ); + if ( !cursor_members.ok ) { return -1; } + for ( int32_t i = 0; i < cursor_members.count; i++ ) + { + int64_t inner = CrewsMembersEntryPackMeasure( ctx, seen, *cursor_members[i] ); + if ( inner < 0 ) { TableMapRelease( cursor_members ); return -1; } + bytes += inner; + } + TableMapRelease( cursor_members ); + } + return bytes; +} + +// CrewsPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool CrewsPackEdges( const Ctx & ctx, TablePackMap & seen, const Crews & src, Crews & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool CrewsPack( const Ctx & ctx, TablePackMap & seen, const Crews & src, Crews & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( Crews ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( Crews ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !CrewsExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return CrewsPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool CrewsPackEdges( const Ctx & ctx, TablePackMap & seen, const Crews & src, Crews & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + { // members: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_members = TableMapOrder( ctx, src.members ); + if ( !cursor_members.ok ) { return false; } + CrewsMembersEntry * placed_members = (CrewsMembersEntry *) ( dst.members.entries.value != 0 ? ( (uint8_t *) &dst.members.entries + dst.members.entries.value ) : NULL ); + for ( int32_t i = 0; i < cursor_members.count; i++ ) + { + if ( !CrewsMembersEntryPackEdges( ctx, seen, *cursor_members[i], placed_members[i], base, capacity, used ) ) { TableMapRelease( cursor_members ); return false; } + } + TableMapRelease( cursor_members ); + } + return true; +} + +// ---- Crews: the variable-length life (docs/SPEC-TABLES.md §2, §6, §9) ---- +// +// MUTABLE: CrewsBuilder — allocate nodes, wire them together, then Lock. +// CONST: one packed region, root at its base. Lock produces it and Load +// produces it, so a locked structure and a loaded one are the +// SAME representation with one view API. There is no unlock: +// re-editing means loading the const form into a fresh builder. +// Crews is never held by value — a file-format-scale structure is a region +// and a root pointer, not a struct you copy. + +struct CrewsBuilder +{ + TableArena arena; + TableWorker main; // the calling thread's allocation front + TableRef root_ref; + uint8_t * region = NULL; // the packed const form, produced by Lock() + int64_t region_bytes = 0; + + // THE ALLOCATOR IS THE BUILDER'S, and everything this structure ever + // allocates goes through it: the arena's segments, Lock's identity map, + // the packed region, the wire walks' numbering, and the tool path's node + // directory. Name your own and a profiler sees every byte under it. + CrewsBuilder( TableAllocator allocator = TableDefaultAllocator() ) + { + TableArenaInit( arena, allocator ); + main.arena = &arena; + TableSlot slot = main.Alloc(); + root_ref = slot.ref; + } + ~CrewsBuilder() { TableArenaShutdown( arena ); arena.allocator.free( arena.allocator.context, region ); } + CrewsBuilder( const CrewsBuilder & ) = delete; + CrewsBuilder & operator=( const CrewsBuilder & ) = delete; + + // Alloc a node in THIS thread's slab: no lock, no atomic per node. + // The result is usable both as the node pointer and as the reference + // to store in a pointer field. + template TableSlot Alloc() { return main.Alloc(); } + // a BYTE BUFFER's node of exactly `length` bytes (docs/SPEC-TABLES.md §2.5): + // the bytes to write through, and the reference to store in a *bytes + // or *string slot; a blob past a slab takes a span of its own + TableBytesSlot AllocBytes( int64_t length ) { return main.AllocBytes( length ); } + TableStringSlot AllocString( int64_t length ) { return main.AllocString( length ); } + // one worker per thread; allocate on your own, and synchronize your own + // writes to nodes another worker allocated + TableWorker Worker() { TableWorker worker; worker.arena = &arena; return worker; } + + // GetRoot/AsConst, not Root/Const: a member function hides the type + // name it shares, and `table Root` is this spec's own canonical + // example. The checker refuses a table named after any member here, + // so the remaining spellings cannot collide either. + Crews * GetRoot() { return arena.locked ? NULL : (Crews *) TableArenaAt( arena, (uint32_t) root_ref.value ); } + bool Locked() const { return arena.locked; } + const Crews * AsConst() const { return (const Crews *) region; } + const uint8_t * Region() const { return region; } + int64_t RegionBytes() const { return region_bytes; } + + // Lock is ONE WAY and it is the compaction: the segmented arena becomes + // one exact-packed region with zero slack, references rewritten + // self-relative, and the mutable life released. Single-threaded: call + // it after the workers have joined. + bool Lock(); +}; + +inline bool CrewsBuilder::Lock() +{ + if ( arena.locked ) { return region != NULL; } + if ( root_ref.null() ) { return false; } + TableArenaCtx ctx = { &arena }; + const Crews & root = *(const Crews *) TableArenaAt( arena, (uint32_t) root_ref.value ); + // The ROOT takes the map's first entry: it is packed at offset 0, and its + // descent is open for the whole walk (docs/SPEC-TABLES.md §3.1). + TablePackMap seen; + TablePackMapInit( seen, arena.allocator ); + bool root_taken = false; + int64_t root_slot = 0; + int64_t below = -1; + if ( TablePackMapReach( seen, (const void *) &root, 0, root_taken, root_slot ) != NULL ) + { + below = CrewsPackMeasure( ctx, seen, root ); + } + if ( below < 0 ) { TablePackMapShutdown( seen ); return false; } // a data cycle, named at the reference that closes it + int64_t root_extent = CrewsExtent( ctx, root ); + if ( root_extent < 0 ) { TablePackMapShutdown( seen ); return false; } // the sort could not run + int64_t total = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ) + below; + // the AUTHORING path may allocate (§6.5), and it does so through the + // builder's own pair. The region comes back ZEROED, which is the + // allocator's contract: a packed region carries node padding. + uint8_t * packed = (uint8_t *) arena.allocator.alloc( arena.allocator.context, total ); + if ( packed == NULL ) { TablePackMapShutdown( seen ); return false; } + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + Crews * destination = new ( packed ) Crews; // lifetime only: the Pack below memcpy's the whole node over it + // The pack walk RE-DERIVES the same numbering rather than carrying the + // measure's — nothing passes between them, which is what makes + // `used == total` below a real check and not a tautology (§3.1). The + // map keeps the capacity the measure paid for, so the second walk + // rehashes nothing. + TablePackMapReset( seen ); + if ( TablePackMapReach( seen, (const void *) &root, 0, root_taken, root_slot ) == NULL || + !CrewsPack( ctx, seen, root, *destination, packed, total, used ) || used != total ) + { + TablePackMapShutdown( seen ); + arena.allocator.free( arena.allocator.context, packed ); + return false; + } + TablePackMapShutdown( seen ); + region = packed; + region_bytes = total; + arena.locked = true; // MONOTONIC: there is no unlock + TableArenaShutdown( arena ); + return true; +} + +// ---- Crews on the wire: the FLAT NODE TABLE (docs/SPEC-TABLES.md §3.1) ---- +// +// A pointered save writes every reachable node ONCE, into a node table under +// the reserved id 0xFFFF, and a pointer field rides as a u32 INDEX into it +// under kind 17. No pointer edge is a nesting level, so a chain's length is +// not a depth and two references to one node are one node. + +// CrewsNodeStorage: the region bytes one record commands, or -1 for a type id +// this build cannot name — which keeps its index and reads null. A BYTE +// BUFFER's record commands its header and its bytes (docs/SPEC-TABLES.md §2.5), +// which is the one answer the record's LENGTH decides, and a blob past the +// size cap answers kTableNodeRefused with its reason (§3.1, §6.5). +// A MAP'S ENTRIES RIDE IN THEIR HOLDER'S EXTENT (docs/SPEC-TABLES.md §2.8), +// so a record's storage is its type's PLUS N x sizeof( Entry ) at every +// depth, summed from the FRAMING: N is framing and not a value, and this +// reads no field. kTableNodeRefused is a wire whose N its L cannot carry. +inline int64_t CrewsNodeStorage( uint64_t type_id, int64_t length, TableRefuseReason & reason ) +{ + (void) length; // no byte buffer below this root: every node's storage is its type's + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( (int64_t) sizeof( Item ) ); // Item + default: break; + } + (void) reason; // no blob and no extent below this root: nothing here refuses + return -1; +} + +// CrewsNodePlace: start one record's node's lifetime in the storage pass one +// reserved for it, holding exactly the declared defaults — a byte buffer's +// header holds its length, and its bytes come in pass two. +inline void CrewsNodePlace( uint64_t type_id, uint8_t * at, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: { Item * node = new ( at ) Item; ItemReset( *node ); break; } // Item + default: break; + } +} + +// CrewsNodeRecordBytes: one record's OWN storage, before the extent its maps +// take (docs/SPEC-TABLES.md §2.8) — where a node's extent begins. +inline int64_t CrewsNodeRecordBytes( uint64_t type_id ) +{ + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( (int64_t) sizeof( Item ) ); // Item + default: break; + } + return 0; +} + +// CrewsNodeAlloc: the TOOL's path — one record's node in the builder's arena. +// Zero is the arena's null, and it is also what a type id this build cannot +// name answers. +inline uint32_t CrewsNodeAlloc( uint64_t type_id, TableWorker & worker, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return (uint32_t) worker.Alloc().ref.value; // Item + default: break; + } + return 0; +} + +// CrewsNodeBody: PASS TWO's half — decode one record's body into the storage it +// already owns. +inline void CrewsNodeBody( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = CrewsNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? CrewsNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ItemLoadBody( r, *(Item *) at ); break; // Item + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// CrewsNodeMessageStorage: the region bytes one record commands on the message +// wire, or -1 for a type id this build cannot name. A table's is its own +// storage plus the extent its maps take; a byte buffer's is its header and +// its bytes, which is the one answer the record's LENGTH decides. +inline int64_t CrewsNodeMessageStorage( uint64_t type_id, int64_t extent, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Item ) ) + extent ); // Item + default: break; + } + return -1; +} + +// CrewsNodeMessageExtent: step over one TABLE record's body, tallying the extent +// its maps take where its type has any (§2.8). A type this build cannot +// name is stepped over by its announced shapes and takes no extent. +inline bool CrewsNodeMessageExtent( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & extent ) +{ + extent = 0; + (void) type_id; // no map below any node this root can name + return TableMessageSkipBody( r, vocabulary, index_bits ); +} + +// CrewsNodeMessageBody: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +inline bool CrewsNodeMessageBody( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at ) +{ + TableExtentCarve carve; + carve.at = at + CrewsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !CrewsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemLoadMessageBody( r, vocabulary, report, index_bits, *(Item *) at ); break; // Item + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// The numbering both wire walks derive, and NEITHER CARRIES THE OTHER'S: the +// root takes index 1 and its entry stays open for the whole walk, so a +// reference back at it is the cycle it is (§3.1). +template +inline bool CrewsNumberFrom( const Ctx & ctx, TableNumbering & numbering, const Crews & root ) +{ + bool taken = false; + int64_t slot = 0; + if ( TablePackMapReach( numbering.seen, (const void *) &root, (int64_t) kTableNodeIndexRoot, taken, slot ) == NULL ) { return false; } + return CrewsNumber( ctx, numbering, root ); +} + +template +inline int64_t CrewsMeasureWire( const Ctx & ctx, const Crews & root, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bytes = -1; + if ( CrewsNumberFrom( ctx, numbering, root ) ) + { + TableIds ids; + bytes = CrewsMeasureBody( ctx, numbering, ids, root ); + if ( bytes >= 0 ) + { + const int64_t table = TableNodeTableMeasure( ctx, ids, numbering ); + // the FORM BYTE, the ROOT BODY — its own fields, the node table + // and the terminator — and the ID TABLE (docs/SPEC-TABLES.md §3) + bytes = table < 0 || ids.overflow ? -1 : 1 + bytes + table + TableIdsBytes( ids ); + } + } + TableNumberingShutdown( numbering ); + return bytes; +} + +template +inline int64_t CrewsSaveWire( const Ctx & ctx, const Crews & root, uint8_t * buffer, int64_t capacity, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + if ( !CrewsNumberFrom( ctx, numbering, root ) ) { TableNumberingShutdown( numbering ); return -1; } + TableWriter w( buffer, capacity ); + TableIds ids; + w.put8( kTableWireForm ); // the FORM BYTE is the whole header (§3) + // the root's own fields, then the node table's field, then the + // terminator: a reader that gives up inside the table has already + // decoded the ROOT'S OWN FIELDS (§3.1) + bool ok = CrewsSaveBodyFields( ctx, numbering, w, ids, root ) && TableNodeTableSave( ctx, w, ids, numbering ); + TableNumberingShutdown( numbering ); + if ( !ok || ids.overflow ) { return -1; } + w.put8( 0 ); // the ZERO REFERENCE that ends the root body + TableIdsWrite( w, ids ); + if ( w.overflow ) { return -1; } // the caller's buffer was too small + return w.offset; // == CrewsMeasure( root ) +} + +inline int64_t CrewsMeasure( const Crews * root, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return CrewsMeasureWire( ctx, *root, allocator ); +} + +inline int64_t CrewsSave( const Crews * root, uint8_t * buffer, int64_t capacity, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return CrewsSaveWire( ctx, *root, buffer, capacity, allocator ); +} + +inline int64_t CrewsMeasure( const CrewsBuilder & builder ) +{ + if ( builder.region != NULL ) { return CrewsMeasure( builder.AsConst(), builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return CrewsMeasureWire( ctx, *(const Crews *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), builder.arena.allocator ); +} + +inline int64_t CrewsSave( const CrewsBuilder & builder, uint8_t * buffer, int64_t capacity ) +{ + if ( builder.region != NULL ) { return CrewsSave( builder.AsConst(), buffer, capacity, builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return CrewsSaveWire( ctx, *(const Crews *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), buffer, capacity, builder.arena.allocator ); +} + +// ---- Crews on the MESSAGE wire: the batch over a region (docs/SPEC-TABLES.md §3.3) ---- + +// CrewsMessageBodyBits: one root body's bits at bit position `at` of the batch, +// with the numbering derived from the graph, the node table FIRST, then the +// fields, then the zero reference. Measure derives the numbering and save +// derives the same one, and nothing passes between them (§3.1). +template +inline int64_t CrewsMessageBodyBits( const Ctx & ctx, const Crews & root, int64_t at, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bits = -1; + if ( CrewsNumberFrom( ctx, numbering, root ) ) + { + const int64_t index_bits = TableBitsRequired( 0, numbering.count + 1 ); + const int64_t table = TableMessageNodeTableMeasure( ctx, numbering, index_bits, at ); + if ( table >= 0 ) + { + const int64_t body = CrewsMeasureMessageBody( ctx, numbering, index_bits, at + table, root ); + bits = body < 0 ? -1 : table + body; + } + } + TableNumberingShutdown( numbering ); + return bits; +} + +template +inline bool CrewsMessageBodySave( const Ctx & ctx, const Crews & root, TableBitWriter & w, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + bool ok = false; + if ( CrewsNumberFrom( ctx, numbering, root ) ) + { + const int64_t index_bits = TableBitsRequired( 0, numbering.count + 1 ); + ok = TableMessageNodeTableSave( ctx, numbering, index_bits, w ) && CrewsSaveMessageBody( ctx, numbering, index_bits, w, root ); + } + TableNumberingShutdown( numbering ); + return ok && !w.overflow; +} + +// THE PRIMITIVE IS A BATCH (§3.3): a number of ROOTS in one buffer, one count +// and one continuous bit stream, each body carrying its own numbering. A +// root is a locked region's, `builder.AsConst()`, or a loaded one's. M above +// 256 is a refusal by name, batch_too_large, with nothing written. +inline int64_t CrewsMeasureMessages( const Crews * const * roots, int64_t count, TableReport * report, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( roots == NULL || count < 1 ) { return -1; } + if ( count > kTableMessageBatchMax ) { TableMessageRefuseBatch( report ); return -1; } + TableRegionCtx ctx; + int64_t bits = 8; // the body count + for ( int64_t i = 0; i < count; i++ ) + { + if ( roots[i] == NULL ) { return -1; } + const int64_t body = CrewsMessageBodyBits( ctx, *roots[i], bits, allocator ); + if ( body < 0 ) { return -1; } + bits += body; + } + return 1 + ( bits + 7 ) / 8; +} + +inline int64_t CrewsSaveMessages( const Crews * const * roots, int64_t count, uint8_t * buffer, int64_t capacity, TableReport * report, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( roots == NULL || count < 1 ) { return -1; } + if ( count > kTableMessageBatchMax ) { TableMessageRefuseBatch( report ); return -1; } + TableMessageBatch batch; + if ( !TableMessageBatchBegin( batch, buffer, capacity, count ) ) { return -1; } + TableRegionCtx ctx; + for ( int64_t i = 0; i < count; i++ ) + { + if ( roots[i] == NULL || !CrewsMessageBodySave( ctx, *roots[i], batch.w, allocator ) ) { return -1; } + batch.written++; + } + return TableMessageBatchEnd( batch ); // == CrewsMeasureMessages( roots, count, report, allocator ) +} + +// CrewsMessageRecordScan: one node record's type id and the extent its maps +// take, or a blob's length, the reader left after the record. A type id +// reference of 0, one past E, or one naming anything but a kind-0 entry is +// damage, as §3.1 and §3.3 say. +inline bool CrewsMessageRecordScan( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, uint64_t & type_id, int64_t & extent, int64_t & length ) +{ + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { return false; } + TableMessageEntry type_entry; + if ( !TableMessageNameEntry( vocabulary, type_ref, type_entry ) ) { return false; } + type_id = type_entry.id; + extent = 0; + length = 0; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + // A BLOB RECORD CARRIES A LENGTH AT THIRTY-TWO RAW BITS, then ALIGNS, + // then the bytes verbatim (§3.3) + uint64_t n = 0; + if ( !r.get( n, 32 ) || !r.align() || !r.skip( (int64_t) n * 8 ) ) { return false; } + length = (int64_t) n; + return true; + } + return CrewsNodeMessageExtent( type_id, r, vocabulary, index_bits, extent ); +} + +// CrewsMessageBodyStorage: one body's node count and data bytes from the FRAMING +// alone, the reader left at the next body. The node table is walked record +// by record, a table record's body stepped over by its announced shapes and +// a blob's by its length, then the root's own fields. False is a numbering +// that could not be sized; `complete` false is a ROOT body whose own framing +// gave out, which the load meets as damage inside this body after the +// bodies before it were delivered, so the batch is sized through this body +// and no further (§3.3). +inline bool CrewsMessageBodyStorage( TableBitReader & r, const TableVocabulary & vocabulary, int64_t & records, int64_t & data, bool & complete ) +{ + complete = true; + records = 0; + data = 0; + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { return false; } + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !CrewsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { return false; } + const int64_t storage = CrewsNodeMessageStorage( type_id, extent, length ); + if ( storage > 0 ) { data += storage; } // a type id this build cannot name commands none + records++; + } + int64_t root_extent = 0; + if ( !CrewsMessageExtent( r, vocabulary, index_bits, root_extent ) ) { complete = false; } + data += TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + return true; +} + +// CrewsLoadMeasure's MESSAGE overload: the exact region bytes ONE BATCH needs, +// which is one measurement, one allocation and one bounds check for however +// many bodies ride (§3.3, §6.5). It is a scan by the announced shapes and +// reads no field value. The answer is the data bytes plus the attribution, +// one node directory a body, and -1 for a wire it cannot size: no vocabulary, +// another form, or framing that gives out. +inline int64_t CrewsLoadMeasure( const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, int64_t * attribution_bytes = NULL ) +{ + TableReport ignored; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, &ignored ); + if ( bodies < 0 ) { return -1; } + int64_t data = 0, attribution = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + int64_t records = 0, body_data = 0; + bool complete = true; + if ( !CrewsMessageBodyStorage( br.r, vocabulary, records, body_data, complete ) ) { return -1; } + data += body_data; + attribution += ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( !complete ) { break; } // damage inside this body: the load delivers the ones before it + } + if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; } + return data + attribution; +} + +// CrewsLoadMessageBodyInto: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +inline bool CrewsLoadMessageBodyInto( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Crews * & root_out ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !CrewsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = CrewsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + CrewsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !CrewsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xc85b940060088651ull; + Crews * root = new ( region + used ) Crews; // lifetime only: LoadMessageBody's first act is CrewsReset + CrewsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Crews ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + out->unknown += unknown_records; + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !CrewsNodeMessageBody( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return CrewsLoadMessageBody( r, vocabulary, out, nodes, index_bits, *root ); +} + +// CrewsLoadMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +inline bool CrewsLoadMessages( const Crews ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !CrewsLoadMessageBodyInto( br.r, vocabulary, out, region, region_bytes, used, roots[b] ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + +// CrewsLoadMeasure: the exact region bytes a wire buffer will need, and it is +// ONE SCAN — a record's type id gives its storage size, its length gives the +// next record — reading no field value at all, so the caller owns the +// allocation and can refuse a number it did not expect (§6.5). +// +// It reports the DATA bytes and the ATTRIBUTION bytes separately, because the +// attribution is the wire's numbering made resident (§6.3) and a caller may +// release it once Load returns. The answer is their sum. +inline int64_t CrewsLoadMeasure( const uint8_t * wire_file, int64_t wire_file_bytes, int64_t * attribution_bytes = NULL, TableRefuseReason * reason_out = NULL ) +{ + TableReport ignored; + TableIdTable ids_table; + int64_t body_bytes = 0; + // a FORM BYTE this build does not carry is refused by name (§3, §6.5); + // a trailer that cannot be read whole is damage and names no reason + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict == TableOpenRefused ) { if ( reason_out != NULL ) { *reason_out = unknown_form; } return -1; } + if ( verdict != TableOpenOk ) { return -1; } + // ANY BYTE BETWEEN THE ROOT'S TERMINATOR AND THE TABLE'S FIRST ENTRY + // IS MALFORMED (docs/SPEC-TABLES.md §3): the two ends of the file have + // met, nothing is decoded, and no region is sized from it. + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) { return -1; } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &ignored, &ids_table ); + TableRefuseReason reason = count_over_length; + int64_t root_extent = 0; + if ( !CrewsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { if ( reason_out != NULL ) { *reason_out = reason; } return -1; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + int64_t records = 0; + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = CrewsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { if ( reason_out != NULL ) { *reason_out = reason; } return -1; } // an N the record's framing cannot carry, or a blob past the cap (§2.8, §2.9, §3.1) + if ( storage > 0 ) { data += storage; } // a type id this build cannot name commands none + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; } + return data + attribution; +} + +// CrewsLoad: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +inline const Crews * CrewsLoad( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Crews ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !CrewsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = CrewsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0xc85b940060088651ull; + Crews * root = new ( region ) Crews; // lifetime only: LoadBody's first act is CrewsReset + CrewsReset( *root ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = CrewsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + CrewsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + if ( nodes.good ) { out->unknown += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + CrewsNodeBody( type_id, sub, nodes, region + directory[k + 1].offset ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Crews ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + CrewsLoadBody( r, nodes, *root ); + return root; +} + +// CrewsLoadBuilder: the TOOL's path — the same tolerant decode into a fresh +// builder, so loaded data can be edited and locked again. The numbering is +// the same one; what differs is where a node lives and therefore what a +// resolved slot holds — an arena offset here, a self-relative delta there. +inline bool CrewsLoadBuilder( CrewsBuilder & builder, const uint8_t * wire_file, int64_t wire_file_bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; } + return false; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return false; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + Crews * root = builder.GetRoot(); + if ( root == NULL ) { out->malformed = true; return false; } + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) { records++; } + } + // the AUTHORING side may allocate (§6.5), and this is the tool's path. + // It goes through the builder's own pair, like everything else the + // builder reaches, and the entries come back zeroed. + const TableAllocator allocator = builder.arena.allocator; + TableNodeDirEntry * directory = (TableNodeDirEntry *) allocator.alloc( allocator.context, ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ) ); + if ( directory == NULL ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) builder.root_ref.value; + directory[0].type_id = 0xc85b940060088651ull; + TableNodeMap nodes; + nodes.base = NULL; + nodes.entries = directory; + nodes.count = records + 1; + nodes.arena = true; // a resolved slot holds the node's ARENA OFFSET here + nodes.worker = &builder.main; // and a map's entries and a list's elements are the arena's, not a node extent's (§2.8, §2.9) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + uint32_t at = CrewsNodeAlloc( type_id, builder.main, length ); + if ( at == 0 ) + { + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + } + else + { + directory[k + 1].offset = (uint64_t) at; + } + directory[k + 1].type_id = type_id; + k++; + } + nodes.good = TableNodeScanWhole( scan ); + if ( nodes.good ) { out->unknown += unknown_records; } else { out->malformed = true; } + } + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + CrewsNodeBody( type_id, sub, nodes, TableArenaAt( builder.arena, (uint32_t) directory[k + 1].offset ) ); + } + k++; + } + } + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.worker = &builder.main; + nodes.carve = &root_carve; + bool ok = CrewsLoadBody( r, nodes, *root ); + // A COUNT ABOVE THE int32 CAP is this path's refusal (docs/SPEC-TABLES.md + // §2.9): the partial builder is the caller's to discard, and the report + // holds what it held when the count was met + ok = ok && !nodes.refused; + allocator.free( allocator.context, directory ); + return ok; +} + +template +inline int64_t CrewsMembersEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + if ( value.value_count < 0 || value.value_count > 2 ) { return -1; } // storage invariant + if ( value.value_count > 0 ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( value.value_count ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < value.value_count; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return -1; } + body_value += TableLebBytes( slot_index ); + } + } + bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); // value: [..2]*Item + } + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool CrewsMembersEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 8 ); // key + w.put32( uint32_t( value.key ) ); + } + if ( value.value_count < 0 || value.value_count > 2 ) { return false; } // storage invariant + if ( value.value_count > 0 ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( value.value_count ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < value.value_count; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + body_value += TableLebBytes( slot_index ); + } + } + w.putleb( ref_value ); w.put8( 14 ); w.putleb( (uint64_t) body_value ); // value + w.put8( 17 ); w.putleb( (uint64_t) ( value.value_count ) ); + for ( int32_t elem_i = 0; elem_i < value.value_count; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + w.putleb( slot_index ); + } + } + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool CrewsMembersEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !CrewsMembersEntrySaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool CrewsMembersEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + CrewsMembersEntryReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x3dc94a19365b10ecull: // key + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.key = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.key = decoded_v; + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER — the element kind byte and the + // count, so fewer than two bytes — is INERT (§4): the field keeps the + // value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + // A DAMAGED COUNT stops the elements and nothing else: the field + // RODE, so an optional is still PRESENT (§2.3) — only a foreign + // ELEMENT KIND says the payload is not this array's at all. + if ( !counted_ok ) { r.report->malformed = true; } + else if ( elem_kind != 17 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + uint64_t keep = count; + if ( keep > 2 ) { keep = 2; r.report->clamped++; } + // elements are BOUNDED by the field body: a count the length + // cannot cover keeps the decoded prefix, flags malformed, and + // the parent continues at the next field — following fields' + // bytes are never fabricated into elements + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint64_t decoded = 0; + for ( uint64_t i = 0; i < keep; i++ ) + { + { + uint64_t node_index = 0; + if ( !sub.getleb( node_index ) ) { r.report->malformed = true; break; } + TableNodeResolve( nodes, value.value[(int32_t) i], node_index, 0x52cfa1d198476806ull, r.report ); // *Item + } + decoded = i + 1; + } + value.value_count = (int32_t) decoded; + } + } + r.offset = body_end; // excess elements and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +template +inline int64_t CrewsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // members: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_members = TableMapOrder( ctx, value.members ); + if ( !order_members.ok ) { return -1; } // the sort could not run + if ( order_members.count > 0 ) + { + const uint64_t ref_members = ids.ref( 0x79d594675e391090ull ); + int64_t body_members = 1 + TableLebBytes( (uint64_t) order_members.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_members = CrewsMembersEntryMeasureBodyRetain( ctx, numbering, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_members < 0 ) { TableMapRelease( order_members ); return -1; } + body_members += TableLebBytes( (uint64_t) ( elem_members ) ) + ( elem_members ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_members ) + 1 + TableLebBytes( (uint64_t) ( body_members ) ) + ( body_members ); + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool CrewsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + { + TableMapCursor order_members = TableMapOrder( ctx, value.members ); // members + if ( !order_members.ok ) { return false; } + if ( order_members.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_members = ids.ref( 0x79d594675e391090ull ); + int64_t body_members = 1 + TableLebBytes( (uint64_t) order_members.count ); + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_members = CrewsMembersEntryMeasureBodyRetain( ctx, numbering, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_members < 0 ) { TableMapRelease( order_members ); return false; } + body_members += TableLebBytes( (uint64_t) ( elem_members ) ) + ( elem_members ); + } + w.putleb( ref_members ); w.put8( 14 ); w.putleb( (uint64_t) body_members ); + w.put8( 13 ); w.putleb( (uint64_t) order_members.count ); + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_len_members = CrewsMembersEntryMeasureBodyRetain( ctx, numbering, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_len_members < 0 ) { TableMapRelease( order_members ); return false; } + w.putleb( (uint64_t) elem_len_members ); + if ( !CrewsMembersEntrySaveBodyRetain( ctx, numbering, w, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_members ); return false; } + } + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool CrewsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !CrewsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool CrewsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + CrewsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x79d594675e391090ull: // members + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.members, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + CrewsMembersEntryKeyRead read = CrewsMembersEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + CrewsMembersEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + CrewsMembersEntryLoadBodyRetain( elem, nodes, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// CrewsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void CrewsNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = CrewsNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? CrewsNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ItemLoadBodyRetain( r, *(Item *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Item + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// CrewsLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +// UNDER RETENTION it also fills the caller's two stores with the fields +// this build cannot name, and the report carries what it could not keep +// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the +// reader's data is exactly what it would have been with retention off. +inline const Crews * CrewsLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Crews ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !CrewsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = CrewsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0xc85b940060088651ull; + Crews * root = new ( region ) Crews; // lifetime only: LoadBody's first act is CrewsReset + CrewsReset( *root ); + + // LoadRetain RESETS BOTH STORES and writes into neither id list: a + // retained record carries its field's identity in the record itself, + // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer + // belongs to this region from here on. + TableRetainReset( retain, nodes, region ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = CrewsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + CrewsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + CrewsNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Crews ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + CrewsLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// CrewsMeasureRetain and CrewsSaveRetain: the pair, with the retained tail in +// every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same +// records under the same walk, so Measure's answer is the size the save +// writes even where a record could not be placed. +template +inline int64_t CrewsMeasureWireRetain( const Ctx & ctx, const Crews & root, TableRetain * retain, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bytes = -1; + auto retain_measure = []( const Ctx & c, const TableNumbering & nn, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> int64_t + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemMeasureBodyRetain( ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return -1; + }; + if ( CrewsNumberFrom( ctx, numbering, root ) ) + { + TableRetainIds ids( retain ); + if ( retain != NULL ) { retain->id_used = 0; } // one walk fills the list, and the save's own walk refills it + bytes = CrewsMeasureBodyRetain( ctx, numbering, ids, root, retain, TableRetainPathRoot( (const void *) &root, 1 ) ); + if ( bytes >= 0 ) + { + const int64_t table = TableNodeTableMeasureRetain( ctx, ids, numbering, retain, retain_measure ); + bytes = table < 0 || ids.overflow ? -1 : 1 + bytes + table + TableRetainIdsBytes( ids ); + } + } + TableNumberingShutdown( numbering ); + return bytes; +} + +template +inline int64_t CrewsSaveWireRetain( const Ctx & ctx, const Crews & root, TableRetain * retain, uint8_t * buffer, int64_t capacity, TableReport * report ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, TableDefaultAllocator() ); + auto retain_measure = []( const Ctx & c, const TableNumbering & nn, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> int64_t + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemMeasureBodyRetain( ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return -1; + }; + auto retain_save = []( const Ctx & c, const TableNumbering & nn, TableWriter & ww, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> bool + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ww; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemSaveBodyRetain( ww, ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return false; + }; + if ( !CrewsNumberFrom( ctx, numbering, root ) ) { TableNumberingShutdown( numbering ); return -1; } + TableWriter w( buffer, capacity ); + TableRetainIds ids( retain ); + if ( retain != NULL ) { retain->id_used = 0; } + TableRetainClearPlaced( retain ); + w.put8( kTableWireForm ); // the FORM BYTE is the whole header (§3) + // the root's own fields, then the RETAINED TAIL, then the node table's + // field: a retained field is one of the root's own values, and the tail + // is pinned before the large and damage-prone part (§6.6, §3.1) + bool ok = CrewsSaveBodyFieldsRetain( ctx, numbering, w, ids, root, retain, TableRetainPathRoot( (const void *) &root, 1 ) ) && + TableNodeTableSaveRetain( ctx, w, ids, numbering, retain, retain_measure, retain_save ); + TableNumberingShutdown( numbering ); + if ( !ok || ids.overflow ) { return -1; } + w.put8( 0 ); // the ZERO REFERENCE that ends the root body + TableRetainIdsWrite( w, ids ); + if ( w.overflow ) { return -1; } // the caller's buffer was too small + // THE SAVE'S OWN SHARE OF retain_lost, read after the save (§6.6): every + // record the walk did not place, counted once. + TableRetainCountLost( retain, report ); + return w.offset; +} + +inline int64_t CrewsMeasureRetain( const Crews * root, TableRetain * retain, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return CrewsMeasureWireRetain( ctx, *root, retain, allocator ); +} + +// SaveRetain REFUSES A NULL REPORT and returns -1 (docs/SPEC-TABLES.md +// §6.6): the save is the only place a caller learns that a record was +// dropped, so the report is required here where it is optional everywhere +// else. A surface that let a caller retain, save and never find out would +// be a promise it could not check. +inline int64_t CrewsSaveRetain( const Crews * root, TableRetain * retain, uint8_t * buffer, int64_t capacity, TableReport * report ) +{ + if ( root == NULL || report == NULL ) { return -1; } + TableRegionCtx ctx; + return CrewsSaveWireRetain( ctx, *root, retain, buffer, capacity, report ); +} + +// CrewsSaveRetainMessages: RETENTION WRITING FORM 2 IS REFUSED BY NAME +// (docs/SPEC-TABLES.md §3.3). It is a MISUSE refusal on §6.6's own +// precedent and never a silent drop, and the two answers are named: a +// caller that must carry unknowns across a rewrite writes the FILE form, +// which carries its own table and takes §6.6 unchanged, and a RELAY +// forwards the sending peer's announcement and its batch bytes verbatim. +template +inline int64_t CrewsSaveRetainMessages( Args &&... ) +{ + static_assert( sizeof...( Args ) == (size_t) -1, + "Crews: a form 2 writer names entries through slots of a vocabulary the compiler settled, and a retained id is one this build's closure does not contain, so it has neither a slot nor an announced shape. Retention writing the MESSAGE form is refused by name (docs/SPEC-TABLES.md §3.3). Write the FILE form, which carries its own table and takes §6.6 unchanged, or relay the sender's announcement and batch bytes verbatim." ); + return -1; +} + +// ---- the cooked form: point at a cook (docs/SPEC-TABLES.md §7) ---- + +// CrewsOpen: match the header and POINT. On a match the bytes ARE what this +// build wrote, in this build's layout and this build's byte order, so there +// is nothing to validate and nothing to fix up and the root comes back as it +// lies. On ANY refusal it returns NULL and NAMES the refusal in the caller's +// TableRefuseReason, the first failing clause in §7's order (a wrong build +// version is a re-cook, a foreign order a cross-endian cook, a truncated +// file a bad download, an unaligned base the caller's own buffer), and the +// caller falls back to a wire load, which is the path that carries every +// version. The reason is written on the refusal path only; a caller that +// passes nothing gets the null alone. +// +// It is O(1) IN THE FILE'S SIZE — the header and nothing per node — so a one +// megabyte cook and a one gigabyte cook open in the same time, and a mapped +// file's pages are touched only as they are used. That is a property of +// touching nothing at open rather than a separate mechanism. +// +// A REFERENCE INSIDE THE REGION IS DEREFERENCED THROUGH CrewsAt: the slot holds +// the signed self-relative byte delta of §6.3, so a deref is one add and +// needs no base pointer, a whole region relocates by plain memcpy, and a +// delta of zero is null. +// +// There is ONE entry point and no tolerant twin: a build either wrote this +// file or it did not, and the build version is what says which. Validating a +// file whose provenance a person doubts is schema cook-check, offline, +// over the ATTRIBUTION part beside the data — a person's decision, never a +// parameter on a load. +inline const Crews * CrewsOpen( const void * bytes, uint64_t length, TableRefuseReason * reason = NULL ) +{ + return (const Crews *) TableCookOpen( bytes, length, (uint64_t) sizeof( Crews ), (uint64_t) alignof( Crews ), reason ); +} + +// ---- the cooked form: WRITE a cook (docs/SPEC-TABLES.md §7.6) ---- +// +// The bytes are `schema cook`'s, and the tool stays the reference: the two +// writers are held to one file, byte for byte, in both byte orders. A cook is +// content-addressed by (asset hash, build version), so two writers of one +// instance produce ONE artifact or the pair means nothing. + +template inline bool CrewsMembersEntryCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const CrewsMembersEntry & value, TableByteOrder order ); +template inline bool CrewsCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Crews & value, TableByteOrder order ); + +template inline bool CrewsMembersEntryCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const CrewsMembersEntry & value, TableByteOrder order ) +{ + table_cook_put( at + 0, (uint64_t) value.key, 4, order ); + for ( int32_t i = 0; i < 2; i++ ) // value: an array of pointers, every slot + { + if ( !table_cook_ref( region, at + 8 + i * 8, (const void *) ItemAt( ctx, value.value[ i ] ), order ) ) { return false; } + } + table_cook_put( at + 24, (uint64_t) (uint32_t) value.value_count, 4, order ); + return true; +} + +template inline bool CrewsCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Crews & value, TableByteOrder order ) +{ + (void) ctx; (void) region; // no reference resolves in this body: a list's and a map's slots are the extent writer's, and the class was decided elsewhere in the closure + table_cook_put( at + 0, 0, 8, order ); // members: the array's delta, filled by the extent writer + table_cook_put( at + 8, 0, 4, order ); // and its count + table_cook_put( at + 16, (uint64_t) value.after, 4, order ); + return true; +} + +template inline bool CrewsMembersEntryCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const CrewsMembersEntry & value, TableByteOrder order ); +template inline bool CrewsCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const Crews & value, TableByteOrder order ); + +// CrewsMembersEntryCookExtent: CrewsMembersEntry's arrays into the node's extent, PRE-ORDER, a map's entries +// in ASCENDING key order and a list's elements in INDEX order, each through its +// own cook writer (§2.8, §2.9, §7.6). +template inline bool CrewsMembersEntryCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const CrewsMembersEntry & value, TableByteOrder order ) +{ + (void) ctx; (void) region; (void) extent; (void) at; (void) record; (void) value; (void) order; + return true; // no list or map below this record +} + +// CrewsCookExtent: Crews's arrays into the node's extent, PRE-ORDER, a map's entries +// in ASCENDING key order and a list's elements in INDEX order, each through its +// own cook writer (§2.8, §2.9, §7.6). +template inline bool CrewsCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const Crews & value, TableByteOrder order ) +{ + (void) region; // a table element's and an entry's references resolve through their own bodies + { // members + TableMapCursor cursor = TableMapOrder( ctx, value.members ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( CrewsMembersEntry ) + uint8_t * array = extent + at; + at += (int64_t) cursor.count * 32; // the whole array FIRST + // the SIXTEEN BYTES of the slot: the self-relative delta, then the count + table_cook_put( record + 0, cursor.count > 0 ? (uint64_t) (int64_t) ( array - ( record + 0 ) ) : 0, 8, order ); + table_cook_put( record + 8, (uint64_t) (uint32_t) cursor.count, 4, order ); + for ( int32_t i = 0; i < cursor.count; i++ ) + { + if ( !CrewsMembersEntryCookBody( ctx, region, array + i * 32, *cursor[i], order ) ) { return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// CrewsMembersEntryCookNode: one node, the record, then the extent its lists and maps take (§2.8, §2.9). +template inline bool CrewsMembersEntryCookNode( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const CrewsMembersEntry & value, TableByteOrder order ) +{ + if ( !CrewsMembersEntryCookBody( ctx, region, at, value, order ) ) { return false; } + int64_t extent_at = 0; + return CrewsMembersEntryCookExtent( ctx, region, at + 32, extent_at, at, value, order ); +} + +// CrewsCookNode: one node, the record, then the extent its lists and maps take (§2.8, §2.9). +template inline bool CrewsCookNode( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Crews & value, TableByteOrder order ) +{ + if ( !CrewsCookBody( ctx, region, at, value, order ) ) { return false; } + int64_t extent_at = 0; + if ( !CrewsCookExtent( ctx, region, at + 24, extent_at, at, value, order ) ) { return false; } + return extent_at == CrewsExtent( ctx, value ); // the extent written is the extent measured, or no header is written +} + +// CrewsCookLayout: the tool's own Layout (docs/SPEC-TABLES.md §7.2) over one +// numbering — the root at zero, then every node in index order at +// align_up( offset, alignof ) for its OWN type, no slack between them, the +// data length rounded to the greatest alignment among them and never below +// eight. The offsets go into the region's table when it has one, and are only +// summed when it does not (a measure). A type id the numbering carries that +// this root cannot name is the two walks disagreeing, and it is refused. +// A NODE'S SIZE DEPENDS ON ITS VALUE where a list or a map rides in its extent +// (docs/SPEC-TABLES.md §2.8), so the layout takes the resolution context +// the numbering walked and reads the same arrays that walk read. +template +inline bool CrewsCookLayout( const Ctx & ctx, const Crews & root, const TableNumbering & numbering, TableCookRegion & region ) +{ + region.numbering = &numbering; + region.count = numbering.count + 1; + const int64_t root_extent = CrewsExtent( ctx, root ); + if ( root_extent < 0 ) { return false; } + int64_t offset = 24 + root_extent; // the root at zero, its extent behind it + int64_t align = 8; + if ( region.offsets != NULL ) { region.offsets[0] = 0; } + for ( int64_t k = 0; k < numbering.count; k++ ) + { + int64_t size = 0; + int64_t node_align = 0; + switch ( numbering.entries[k].type_id ) + { + case 0x52cfa1d198476806ull: size = 4; node_align = 4; break; // Item + default: return false; + } + offset = ( offset + node_align - 1 ) & ~( node_align - 1 ); + if ( region.offsets != NULL ) { region.offsets[k + 1] = offset; } + offset += size; + if ( node_align > align ) { align = node_align; } + } + region.bytes = ( offset + align - 1 ) & ~( align - 1 ); + region.align = align; + return true; +} + +// CrewsCookMeasureFrom: the whole cooked file's bytes for one graph — the header, +// the data part and the attribution part (§7.1). IT DEPENDS ON THE VALUE, +// because the answer is the numbering: the depth-first walk of §3.1 is run +// here and run again by the write, and neither carries the other's (§7.6). A +// data cycle is refused by the walk and answers -1. +template +inline int64_t CrewsCookMeasureFrom( const Ctx & ctx, const Crews & root, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + TableCookRegion region; + int64_t bytes = -1; + if ( CrewsNumberFrom( ctx, numbering, root ) && CrewsCookLayout( ctx, root, numbering, region ) ) + { + const int64_t data_offset = ( kTableCookHeaderBytes + region.align - 1 ) & ~( region.align - 1 ); + bytes = data_offset + region.bytes + region.count * (int64_t) sizeof( TableNodeDirEntry ); + } + TableNumberingShutdown( numbering ); + return bytes; +} + +// CrewsCookFrom: write one cooked file of a pointered graph, in the byte order +// the caller names. The bytes are `schema cook`'s, byte for byte (§7.6). +// +// THE CALLER OWNS THE OUTPUT and nothing is allocated toward it. What is +// allocated is the numbering — the identity map, the entry array and one +// offset per node — through the pair handed in, and released before this +// returns (§6.5, §13.9). A capacity short of the measure writes nothing. +// +// THE HEADER IS WRITTEN LAST. A reference the numbering did not carry is +// found while a body is being written, and a write that refuses there has +// already put bytes in the buffer; with no magic ahead of them, no Open can +// mistake them for a cook. +template +inline bool CrewsCookFrom( const Ctx & ctx, const Crews & root, void * out, uint64_t capacity, TableByteOrder order, TableAllocator allocator ) +{ + if ( out == NULL ) { return false; } + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + TableCookRegion region; + bool ok = CrewsNumberFrom( ctx, numbering, root ); + if ( ok ) + { + region.offsets = (int64_t *) allocator.alloc( allocator.context, ( numbering.count + 1 ) * (int64_t) sizeof( int64_t ) ); + ok = region.offsets != NULL && CrewsCookLayout( ctx, root, numbering, region ); + } + if ( ok ) + { + const int64_t data_offset = ( kTableCookHeaderBytes + region.align - 1 ) & ~( region.align - 1 ); + const int64_t attribution = region.count * (int64_t) sizeof( TableNodeDirEntry ); + const int64_t need = data_offset + region.bytes + attribution; + ok = (uint64_t) need <= capacity; + if ( ok ) + { + uint8_t * raw = (uint8_t *) out; + memset( raw, 0, (size_t) need ); // EVERY BYTE NO FIELD COVERS IS ZERO (§7.2) + region.base = raw + data_offset; + // the DATA part: the root at the region's base, then every numbered + // node at the offset the layout gave it, each through its own writer + ok = CrewsCookNode( ctx, region, region.base, root, order ); + for ( int64_t k = 0; ok && k < numbering.count; k++ ) + { + uint8_t * at = region.base + region.offsets[k + 1]; + const void * node = numbering.entries[k].node; + switch ( numbering.entries[k].type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemCookNode( ctx, region, at, *(const Item *) node, order ); break; // Item + default: ok = false; break; + } + } + // the ATTRIBUTION part: the node directory (§6.3), one entry per node + // in index order, for `schema cook-check` + uint8_t * entry = raw + data_offset + region.bytes; + table_cook_put( entry, 0, 8, order ); + table_cook_put( entry + 8, 0xc85b940060088651ull, 8, order ); // the root: fnv1a64( "Crews" ) + for ( int64_t k = 0; k < numbering.count; k++ ) + { + entry += sizeof( TableNodeDirEntry ); + table_cook_put( entry, (uint64_t) region.offsets[k + 1], 8, order ); + table_cook_put( entry + 8, numbering.entries[k].type_id, 8, order ); + } + // and the HEADER (§7.1), every word a u64 in the order the file is + // produced in; the two RESERVED words are the memset's zeros + if ( ok ) + { + table_cook_put( raw + 0, TableCookMagic, 8, order ); + table_cook_put( raw + 8, BuildVersion, 8, order ); + table_cook_put( raw + 16, (uint64_t) ( order == TableByteOrder::Big ? 2 : 1 ), 8, order ); + table_cook_put( raw + 24, (uint64_t) region.bytes, 8, order ); + table_cook_put( raw + 32, (uint64_t) attribution, 8, order ); + table_cook_put( raw + 40, (uint64_t) region.align, 8, order ); + } + } + } + allocator.free( allocator.context, region.offsets ); + TableNumberingShutdown( numbering ); + return ok; +} + +// CrewsCookMeasure / CrewsCook over a REGION root — a locked builder's AsConst, a +// region CrewsLoad produced, or an opened cook — with the pair the numbering +// allocates through as an optional last argument, as the wire's own entries +// take it (§13.9). +inline int64_t CrewsCookMeasure( const Crews * root, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return CrewsCookMeasureFrom( ctx, *root, allocator ); +} + +inline bool CrewsCook( const Crews * root, void * out, uint64_t capacity, TableByteOrder order, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return false; } + TableRegionCtx ctx; + return CrewsCookFrom( ctx, *root, out, capacity, order, allocator ); +} + +// and over a BUILDER, locked or not: the builder's own pair, and the arena +// encoding while it is still mutable (§6.3). +inline int64_t CrewsCookMeasure( const CrewsBuilder & builder ) +{ + if ( builder.region != NULL ) { return CrewsCookMeasure( builder.AsConst(), builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return CrewsCookMeasureFrom( ctx, *(const Crews *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), builder.arena.allocator ); +} + +inline bool CrewsCook( const CrewsBuilder & builder, void * out, uint64_t capacity, TableByteOrder order ) +{ + if ( builder.region != NULL ) { return CrewsCook( builder.AsConst(), out, capacity, order, builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return false; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return CrewsCookFrom( ctx, *(const Crews *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), out, capacity, order, builder.arena.allocator ); +} + +// ---- relocatability, enforced: the wire is a pure length-prefixed +// stream AND the decoded storage is pointer-free — every closure type +// must stay trivially copyable and standard-layout, so instances can be +// memcpy'd, mmap'd, shared across processes, and walked through +// descriptor offsets. A failure here means a pointer, virtual or +// non-trivial member crept into generated storage. +// +// They ask the COMPILER ITSELF, which is what every C++ standard library +// answers the same two questions with — and it costs this header no +// include at all. +// A pointer FIELD is a TableRef — eight bytes and no address — so the +// property holds in BOTH forms: a fixed-size table is one relocatable +// struct, and a packed region is one relocatable block whose references +// are self-relative and therefore survive a plain memcpy. +static_assert( __is_trivially_copyable( CrewsMembersEntry ), "CrewsMembersEntry must stay relocatable" ); +static_assert( __is_standard_layout( CrewsMembersEntry ), "CrewsMembersEntry must stay standard-layout for offsetof" ); +static_assert( __is_trivially_copyable( Crews ), "Crews must stay relocatable" ); +static_assert( __is_standard_layout( Crews ), "Crews must stay standard-layout for offsetof" ); + +// ---- the cook's layout contract (docs/SPEC-TABLES.md §20.3) ---- +// +// The compiler derived every number below from the declaration and folded it +// into the BUILD VERSION; these asserts are this compiler saying whether it +// agrees. The model is not self-evidently right — on 32-bit System V +// alignof(uint64_t) is 4, not 8 — which is precisely why it is asserted +// rather than assumed. +static_assert( sizeof( CrewsMembersEntry ) == 32, "CrewsMembersEntry's sizeof moved: the build version was taken over 32, so a cook of it would not be this build's file (docs/SPEC-TABLES.md §20.3)" ); +static_assert( alignof( CrewsMembersEntry ) == 8, "CrewsMembersEntry's alignof moved: the build version was taken over 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( CrewsMembersEntry, key ) == 0, "CrewsMembersEntry's field key moved: the build version was taken over offset 0 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( CrewsMembersEntry, value ) == 8, "CrewsMembersEntry's field value moved: the build version was taken over offset 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( sizeof( Crews ) == 24, "Crews's sizeof moved: the build version was taken over 24, so a cook of it would not be this build's file (docs/SPEC-TABLES.md §20.3)" ); +static_assert( alignof( Crews ) == 8, "Crews's alignof moved: the build version was taken over 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( Crews, members ) == 0, "Crews's field members moved: the build version was taken over offset 0 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( Crews, after ) == 16, "Crews's field after moved: the build version was taken over offset 16 (docs/SPEC-TABLES.md §20.3)" ); + + +// ---- reflection descriptors (tables only, docs/SPEC-TABLES.md) ---- + +inline const TableTypeInfo * CrewsMembersEntryTableType(); +inline const TableTypeInfo * CrewsTableType(); +// The descriptors are CONSTANT-INITIALISED data, and a field's target is +// the ADDRESS of another descriptor. These declarations are what let a +// self- or mutually-referential graph — Node naming itself through *Node — +// be expressed as constant data instead of a lazy link, which could not +// have been written race-free OR recursion-safe. The whole reflection +// surface is therefore immutable: read it from any thread, any time. +extern const TableTypeInfo CrewsMembersEntryTableInfo; +extern const TableTypeInfo CrewsTableInfo; + +inline const TableFieldInfo CrewsMembersEntryTableFields[] = { + { "key", "key", "uint32", 0x3dc94a19365b10ecull, 8, false, false, NULL, NULL, false, false, 0, (uint32_t) offsetof( CrewsMembersEntry, key ), (uint32_t) sizeof( CrewsMembersEntry::key ), 0xffffffffu, 0xffffffffu, NULL, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, + { "value", "value", "Item", 0x7ce4fd9430e80ceaull, 17, true, true, []( const void * slot ) -> const void * { return (const void *) ItemAt( *(const TableRef *) slot ); }, []( TableWorker & worker, void * slot ) -> void * { return (void *) ItemEmplace( worker, *(TableRef *) slot ); }, true, false, 2, (uint32_t) offsetof( CrewsMembersEntry, value ), (uint32_t) sizeof( CrewsMembersEntry::value[0] ), (uint32_t) offsetof( CrewsMembersEntry, value_count ), 0xffffffffu, &ItemTableInfo, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, +}; +inline const TableTypeInfo CrewsMembersEntryTableInfo = { "CrewsMembersEntry", (uint32_t) sizeof( CrewsMembersEntry ), 2, CrewsMembersEntryTableFields, +[]( void * p ) { CrewsMembersEntryReset( *(CrewsMembersEntry *) p ); }, true, TableDocNone, 0, NULL }; +inline const TableTypeInfo * CrewsMembersEntryTableType() { return &CrewsMembersEntryTableInfo; } + +inline const TableFieldInfo CrewsTableFields[] = { + { "members", "members", "map[uint32]*Item", 0x79d594675e391090ull, 13, true, false, NULL, NULL, true, false, 0, (uint32_t) offsetof( Crews, members ), (uint32_t) sizeof( CrewsMembersEntry ), (uint32_t) offsetof( Crews, members.count ), 0xffffffffu, &CrewsMembersEntryTableInfo, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, []( TableWorker & worker, void * slot, const char *, int32_t, int64_t key_value ) -> void * { return (void *) TableMapPlace( worker, *(TableMap *) slot, (uint32_t) key_value ); }, "", TableDocNone, 0, NULL }, + { "after", "after", "int32", 0xbf82010f6f71eae9ull, 4, false, false, NULL, NULL, false, false, 0, (uint32_t) offsetof( Crews, after ), (uint32_t) sizeof( Crews::after ), 0xffffffffu, 0xffffffffu, NULL, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, +}; +inline const TableTypeInfo CrewsTableInfo = { "Crews", (uint32_t) sizeof( Crews ), 2, CrewsTableFields, +[]( void * p ) { CrewsReset( *(Crews *) p ); }, true, TableDocNone, 0, NULL }; +inline const TableTypeInfo * CrewsTableType() { return &CrewsTableInfo; } + +// ---- the text form (docs/SPEC-TABLES.md §16) ---- + +// Crews in and out of a JSON text (docs/SPEC-TABLES.md §16.7): read into a +// builder, written from a region's const root. A node named more than once +// carries `&node` in the text. Defined in CrewsTable.cpp; link it to use them. +bool CrewsFromJson( CrewsBuilder & builder, const char * text, int64_t bytes, TableReport * report ); +int64_t CrewsToJsonMeasure( const Crews * root, TableAllocator allocator = TableDefaultAllocator() ); +int64_t CrewsToJson( const Crews * root, char * buffer, int64_t capacity, TableAllocator allocator = TableDefaultAllocator() ); + +} // namespace mapdemo diff --git a/testdata/golden/tables/maps/DepthTable.h b/testdata/golden/tables/maps/DepthTable.h index bc2fa8711..7cf956f68 100644 --- a/testdata/golden/tables/maps/DepthTable.h +++ b/testdata/golden/tables/maps/DepthTable.h @@ -377,7 +377,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -879,13 +879,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1407,7 +1407,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1425,10 +1425,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1437,6 +1437,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1445,54 +1446,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3430,25 +3440,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -5692,7 +5704,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6105,8 +6117,8 @@ inline bool TableEnumSlot( Slot value, uint64_t & slot ) switch ( value ) { case Slot::None: slot = 0; return true; - case Slot::Alpha: slot = 44; return true; - case Slot::Beta: slot = 45; return true; + case Slot::Alpha: slot = 50; return true; + case Slot::Beta: slot = 51; return true; default: return false; // no variant names this value: no wire identity } } @@ -6822,7 +6834,7 @@ inline bool SquadSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_roster.ok ) { return false; } // the sort could not run if ( order_roster.count > 0 ) { - w.put( 42, kTableMessageRefBitsHere ); + w.put( 46, kTableMessageRefBitsHere ); w.put( (uint64_t) order_roster.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_roster.count; i++ ) { @@ -7543,14 +7555,14 @@ inline bool DepthSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( body_one < 0 ) { return false; } if ( body_one > kTableMessageRefBitsHere ) // an all-default nested table elides { - w.put( 19, kTableMessageRefBitsHere ); + w.put( 22, kTableMessageRefBitsHere ); if ( !SquadSaveMessageBody( ctx, numbering, index_bits, w, value.one ) ) { return false; } } } if ( value.many_count < 0 || value.many_count > 3 ) { return false; } // storage invariant if ( value.many_count > 0 ) { - w.put( 20, kTableMessageRefBitsHere ); + w.put( 23, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.many_count ) - 0, 2 ); for ( int32_t i = 0; i < value.many_count; i++ ) { @@ -7569,7 +7581,7 @@ inline bool DepthSaveMessageBody( const Ctx & ctx, const TableNumbering & number } if ( pairs_keyed > 0 ) { - w.put( 21, kTableMessageRefBitsHere ); + w.put( 24, kTableMessageRefBitsHere ); w.put( (uint64_t) pairs_keyed, 2 ); // ASCENDING BY VARIANT ORDINAL, which is slot order. It is // this writer's choice and a reader must not rely on it: every @@ -7589,18 +7601,18 @@ inline bool DepthSaveMessageBody( const Ctx & ctx, const TableNumbering & number } if ( value.arm.type != ForceType::None ) { - w.put( 22, kTableMessageRefBitsHere ); + w.put( 25, kTableMessageRefBitsHere ); switch ( value.arm.type ) { case ForceType::Squad: { - w.put( 46, kTableMessageRefBitsHere ); + w.put( 52, kTableMessageRefBitsHere ); if ( !SquadSaveMessageBody( ctx, numbering, index_bits, w, value.arm.squad ) ) { return false; } break; } case ForceType::Plain: { - w.put( 47, kTableMessageRefBitsHere ); + w.put( 53, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.arm.plain ), 32 ); break; } @@ -7609,7 +7621,7 @@ inline bool DepthSaveMessageBody( const Ctx & ctx, const TableNumbering & number } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/DocsTable.h b/testdata/golden/tables/maps/DocsTable.h index acdfed2b6..00d6087fb 100644 --- a/testdata/golden/tables/maps/DocsTable.h +++ b/testdata/golden/tables/maps/DocsTable.h @@ -376,7 +376,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -878,13 +878,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1406,7 +1406,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1424,10 +1424,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1436,6 +1436,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1444,54 +1445,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3429,25 +3439,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -3752,6 +3764,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4128,6 +4150,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -5676,7 +5703,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6458,15 +6485,13 @@ inline bool DocsPagesEntryLoadMessageBody( TableBitReader & r, const TableVocabu { uint64_t n = 0; if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } - int32_t kept = 0; - if ( n > (uint64_t) 8 ) { kept = 8; report->clamped++; } else { kept = (int32_t) n; } - for ( uint64_t i = 0; i < n; i++ ) - { - uint64_t by = 0; - if ( !r.get( by, 8 ) ) { report->malformed = true; return false; } - if ( (int32_t) i < kept ) { value.key[i] = (char) by; } - } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); value.key[kept] = 0; + r.offset += (int64_t) n * 8; value.key_length = kept; } break; @@ -6766,7 +6791,7 @@ inline bool DocsSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi if ( !order_pages.ok ) { return false; } // the sort could not run if ( order_pages.count > 0 ) { - w.put( 23, kTableMessageRefBitsHere ); + w.put( 26, kTableMessageRefBitsHere ); w.put( (uint64_t) order_pages.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_pages.count; i++ ) { @@ -6777,7 +6802,7 @@ inline bool DocsSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body @@ -7193,7 +7218,7 @@ inline bool DocsPagesEntryNumber( const Ctx & ctx, TableNumbering & numbering, c TableNodeEntry node; node.node = (const void *) blob; node.type_id = kTableStringTypeId; - node.type_slot = 50; // its slot in the unit's vocabulary (§3.3) + node.type_slot = 56; // its slot in the unit's vocabulary (§3.3) node.measure = &TableBlobMeasureThunk; node.save = &TableBlobSaveThunk; node.message_measure = &TableBlobMessageMeasureThunk; diff --git a/testdata/golden/tables/maps/FleetTable.h b/testdata/golden/tables/maps/FleetTable.h index 96d00d68b..67ee3f150 100644 --- a/testdata/golden/tables/maps/FleetTable.h +++ b/testdata/golden/tables/maps/FleetTable.h @@ -376,7 +376,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -878,13 +878,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1406,7 +1406,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1424,10 +1424,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1436,6 +1436,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1444,54 +1445,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3429,25 +3439,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -5691,7 +5703,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6467,6 +6479,37 @@ inline ShipConfig * ShipConfigEmplace( TableWorker & worker, TableRef & slot ) return allocated.ptr; } +// Item is a pointer target. +inline const Item * ItemAt( const TableRef & ref ) // the const form's hot path: one add, no base +{ + return ref.value != 0 ? (const Item *) ( (const uint8_t *) &ref + ref.value ) : NULL; +} +inline Item * ItemAt( TableRef & ref ) +{ + return ref.value != 0 ? (Item *) ( (uint8_t *) &ref + ref.value ) : NULL; +} +inline const Item * ItemAt( const TableRegionCtx &, const TableRef & ref ) { return ItemAt( ref ); } +inline const Item * ItemAt( const TableArenaCtx & ctx, const TableRef & ref ) +{ + return ref.value != 0 ? (const Item *) TableArenaAt( *ctx.arena, (uint32_t) ref.value ) : NULL; +} +// while the builder is mutable, resolve against the arena itself +inline Item * ItemAt( TableArena & arena, const TableRef & ref ) +{ + return ref.value != 0 ? (Item *) TableArenaAt( arena, (uint32_t) ref.value ) : NULL; +} +inline const Item * ItemAt( const TableArena & arena, const TableRef & ref ) +{ + return ref.value != 0 ? (const Item *) TableArenaAt( arena, (uint32_t) ref.value ) : NULL; +} +// allocate one Item in the arena; the slot holds the arena offset +inline Item * ItemEmplace( TableWorker & worker, TableRef & slot ) +{ + TableSlot allocated = worker.Alloc(); + slot = allocated.ref; + return allocated.ptr; +} + // ---- codecs: measure/save/load per closure member ---- inline int64_t ShipConfigMeasureBody( TableIds & ids, const ShipConfig & value ); @@ -6502,6 +6545,9 @@ inline bool FleetLoadBody( TableReader & r, const TableNodeMap & nodes, Fleet & template inline bool ShipConfigNumber( const Ctx & ctx, TableNumbering & numbering, const ShipConfig & value ); template inline int64_t ShipConfigPackMeasure( const Ctx & ctx, TablePackMap & seen, const ShipConfig & value ); template inline bool ShipConfigPack( const Ctx & ctx, TablePackMap & seen, const ShipConfig & src, ShipConfig & dst, uint8_t * base, int64_t capacity, int64_t & used ); +template inline bool ItemNumber( const Ctx & ctx, TableNumbering & numbering, const Item & value ); +template inline int64_t ItemPackMeasure( const Ctx & ctx, TablePackMap & seen, const Item & value ); +template inline bool ItemPack( const Ctx & ctx, TablePackMap & seen, const Item & src, Item & dst, uint8_t * base, int64_t capacity, int64_t & used ); template inline bool FleetByIdEntryNumber( const Ctx & ctx, TableNumbering & numbering, const FleetByIdEntry & value ); template inline int64_t FleetByIdEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const FleetByIdEntry & value ); template inline bool FleetByIdEntryPack( const Ctx & ctx, TablePackMap & seen, const FleetByIdEntry & src, FleetByIdEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); @@ -6518,6 +6564,10 @@ template inline int64_t TableNodeMeasure( const Ctx &, const Tabl template inline bool TableNodeSave( const Ctx &, const TableNumbering &, TableWriter & w, TableIds & ids, const ShipConfig & value ) { return ShipConfigSaveBody( w, ids, value ); } template inline int64_t TableNodeMessageMeasure( const Ctx &, const TableNumbering &, int64_t, int64_t at, const ShipConfig & value ) { return ShipConfigMeasureMessageBody( at, value ); } template inline bool TableNodeMessageSave( const Ctx &, const TableNumbering &, int64_t, TableBitWriter & w, const ShipConfig & value ) { return ShipConfigSaveMessageBody( w, value ); } +template inline int64_t TableNodeMeasure( const Ctx &, const TableNumbering &, TableIds & ids, const Item & value ) { return ItemMeasureBody( ids, value ); } +template inline bool TableNodeSave( const Ctx &, const TableNumbering &, TableWriter & w, TableIds & ids, const Item & value ) { return ItemSaveBody( w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx &, const TableNumbering &, int64_t, int64_t at, const Item & value ) { return ItemMeasureMessageBody( at, value ); } +template inline bool TableNodeMessageSave( const Ctx &, const TableNumbering &, int64_t, TableBitWriter & w, const Item & value ) { return ItemSaveMessageBody( w, value ); } template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const FleetByIdEntry & value ) { return FleetByIdEntryMeasureBody( ctx, numbering, ids, value ); } template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const FleetByIdEntry & value ) { return FleetByIdEntrySaveBody( ctx, numbering, w, ids, value ); } template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const FleetByIdEntry & value ) { return FleetByIdEntryMeasureMessageBody( ctx, numbering, index_bits, at, value ); } @@ -7180,14 +7230,14 @@ inline bool ShipConfigSaveMessageBody( TableBitWriter & w, const ShipConfig & va if ( value.name_length < 0 || value.name_length > 64 ) { return false; } // storage invariant if ( value.name_length > 0 ) { - w.put( 37, kTableMessageRefBitsHere ); + w.put( 41, kTableMessageRefBitsHere ); w.put( (uint64_t) value.name_length, 7 ); w.align(); // a string or a bytes ALIGNS before its bytes w.putbytes( (const uint8_t *) value.name, value.name_length ); } if ( value.health != 0 ) { - w.put( 38, kTableMessageRefBitsHere ); + w.put( 42, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.health ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body @@ -7543,7 +7593,7 @@ inline bool ItemSaveMessageBody( TableBitWriter & w, const Item & value ) { if ( value.count != 0 ) { - w.put( 33, kTableMessageRefBitsHere ); + w.put( 36, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.count ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body @@ -9880,7 +9930,7 @@ inline bool FleetSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_ships.ok ) { return false; } // the sort could not run if ( order_ships.count > 0 ) { - w.put( 28, kTableMessageRefBitsHere ); + w.put( 31, kTableMessageRefBitsHere ); w.put( (uint64_t) order_ships.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_ships.count; i++ ) { @@ -9894,7 +9944,7 @@ inline bool FleetSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_by_id.ok ) { return false; } // the sort could not run if ( order_by_id.count > 0 ) { - w.put( 29, kTableMessageRefBitsHere ); + w.put( 32, kTableMessageRefBitsHere ); w.put( (uint64_t) order_by_id.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_by_id.count; i++ ) { @@ -9909,7 +9959,7 @@ inline bool FleetSaveMessageBody( const Ctx & ctx, const TableNumbering & number { uint64_t index_flagship = 0; if ( !TableNumberingIndex( numbering, (const void *) pointee_flagship, index_flagship ) ) { return false; } - w.put( 30, kTableMessageRefBitsHere ); + w.put( 33, kTableMessageRefBitsHere ); w.put( index_flagship, index_bits ); } } @@ -9918,7 +9968,7 @@ inline bool FleetSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_loadouts.ok ) { return false; } // the sort could not run if ( order_loadouts.count > 0 ) { - w.put( 31, kTableMessageRefBitsHere ); + w.put( 34, kTableMessageRefBitsHere ); w.put( (uint64_t) order_loadouts.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_loadouts.count; i++ ) { @@ -9932,7 +9982,7 @@ inline bool FleetSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_tiers.ok ) { return false; } // the sort could not run if ( order_tiers.count > 0 ) { - w.put( 32, kTableMessageRefBitsHere ); + w.put( 35, kTableMessageRefBitsHere ); w.put( (uint64_t) order_tiers.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_tiers.count; i++ ) { @@ -10344,6 +10394,41 @@ inline bool ShipConfigExtentPack( const Ctx & ctx, const ShipConfig & src, ShipC return true; } +// ItemWireExtent: the extent Item's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool ItemWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + (void) body; (void) length; (void) at; (void) ids; (void) reason; // no list or map below this record + return true; +} + +// ItemExtentAt: the node extent Item's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as ItemExtentPack advances it (§2.8, §2.9). +template +inline bool ItemExtentAt( const Ctx & ctx, const Item & value, int64_t & at ) +{ + (void) ctx; (void) value; (void) at; // no list or map below this record + return true; +} + +template +inline int64_t ItemExtent( const Ctx & ctx, const Item & value ) +{ + (void) ctx; (void) value; // no list or map below this record + return 0; +} + +// ItemExtentPack: carve Item's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset ItemExtentAt advances (§2.8, §2.9). +template +inline bool ItemExtentPack( const Ctx & ctx, const Item & src, Item & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + (void) ctx; (void) src; (void) dst; (void) extent; (void) at; (void) capacity; // no list or map below this record + return true; +} + // FleetByIdEntryWireExtent: the extent FleetByIdEntry's lists and maps command, from the FRAMING alone. // It reads no field value, so a caller can refuse a number it did not // expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). @@ -11168,6 +11253,59 @@ inline bool ShipConfigPackEdges( const Ctx & ctx, TablePackMap & seen, const Shi return true; } +// ItemNumber: number everything Item POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool ItemNumber( const Ctx & ctx, TableNumbering & numbering, const Item & value ) +{ + (void) ctx; (void) numbering; (void) value; // no pointers below this node + return true; +} + +// ItemPackMeasure: the packed region bytes of everything Item POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t ItemPackMeasure( const Ctx & ctx, TablePackMap & seen, const Item & value ) +{ + int64_t bytes = 0; + (void) ctx; (void) seen; (void) value; // no pointers below this node + return bytes; +} + +// ItemPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool ItemPackEdges( const Ctx & ctx, TablePackMap & seen, const Item & src, Item & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool ItemPack( const Ctx & ctx, TablePackMap & seen, const Item & src, Item & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( Item ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( Item ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !ItemExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return ItemPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool ItemPackEdges( const Ctx & ctx, TablePackMap & seen, const Item & src, Item & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + (void) ctx; (void) seen; (void) src; (void) dst; (void) base; (void) capacity; (void) used; + return true; +} + // FleetByIdEntryNumber: number everything FleetByIdEntry POINTS AT, in first-visit order — // the fields in declaration order, a by-value edge descended in place. // A reference to an entry whose descent is still OPEN is a data cycle, @@ -11193,7 +11331,7 @@ inline bool FleetByIdEntryNumber( const Ctx & ctx, TableNumbering & numbering, c TableNodeEntry node; node.node = (const void *) pointee; node.type_id = 0x758252d2d1b14f0dull; // fnv1a64( "ShipConfig" ) - node.type_slot = 61; // its slot in the unit's vocabulary (§3.3) + node.type_slot = 69; // its slot in the unit's vocabulary (§3.3) node.measure = &TableNodeMeasureThunk; node.save = &TableNodeSaveThunk; node.message_measure = &TableNodeMessageMeasureThunk; @@ -11381,7 +11519,7 @@ inline bool FleetNumber( const Ctx & ctx, TableNumbering & numbering, const Flee TableNodeEntry node; node.node = (const void *) pointee; node.type_id = 0x758252d2d1b14f0dull; // fnv1a64( "ShipConfig" ) - node.type_slot = 61; // its slot in the unit's vocabulary (§3.3) + node.type_slot = 69; // its slot in the unit's vocabulary (§3.3) node.measure = &TableNodeMeasureThunk; node.save = &TableNodeSaveThunk; node.message_measure = &TableNodeMessageMeasureThunk; diff --git a/testdata/golden/tables/maps/PairsTable.cpp b/testdata/golden/tables/maps/PairsTable.cpp new file mode 100644 index 000000000..fc3e9f2db --- /dev/null +++ b/testdata/golden/tables/maps/PairsTable.cpp @@ -0,0 +1,3448 @@ +// Code generated by the schema compiler from Pairs.schema. DO NOT EDIT. +// SPDX-License-Identifier: NONE — this generated output is yours, under terms of +// your choice. See the LICENSE exception in the schema compiler; the compiler is +// AGPL-3.0, its output is not. +// package mapdemo — the TABLE wire's text form (docs/SPEC-TABLES.md §16). +// Compile this file to use FromJson / ToJson; a project that +// never reads or writes a text does not compile it and pays nothing. + +#include "PairsTable.h" + +#include // the text form: number formatting +#include // the text form: exact number conversion +#include // the text form: the runtime's decimal point + +// The guard is not vestigial. Several mapdemo Table.cpp files may be +// concatenated into ONE translation unit — a unity build — and without it +// each would redefine the walk. It is also why the walk's functions may be +// weak (vague linkage) across separate objects: ODR requires their +// definitions to be token-identical, and the generic-walk gate is what +// proves that, byte for byte, across every generated .cpp. +#ifndef MAPDEMO_SCHEMA_TABLE_JSON +#define MAPDEMO_SCHEMA_TABLE_JSON + +namespace mapdemo { + +// ---- the pointer adapters (docs/SPEC-TABLES.md §16.7) ---- +// +// The walk below is ONE walk, byte-identical in every generated .cpp, and a +// pointer is the one kind it cannot walk alone: reading one needs the +// builder's arena and writing one needs a region's deref, and neither exists +// in a unit that declares no pointer. So the walk calls these three and does +// not define them. A unit with no pointer defines them as stubs no field ever +// reaches; a pointered unit defines them in the graph half that follows the +// walk. + +struct TableJsonIn; +struct TableJsonOut; + +// a pointer field's object, or the `&node` reference standing in for it, into +// the slot; the cursor is on the opening brace +inline bool TableJsonReadPointer( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); +// the node a pointer slot names, in place — or as `&node` when it is shared +inline bool TableJsonWritePointer( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// the FIRST key of an object the walk is skipping begins with `&`: the cursor is +// on its value. A dropped definition still takes its label (§16.7); a fixed reader +// skips the value whole, as it skips everything else it does not place. +inline bool TableJsonSkippedAmpersand( TableJsonIn & in, const char * key, int32_t depth ); + +// ---- the map and list adapters (docs/SPEC-TABLES.md §2.8, §2.9, §16) ---- +// +// A MAP and an UNBOUNDED ARRAY are the other constructs the walk cannot walk +// alone: their arrays live behind a TableMap or a TableList this +// walk has no name for, reading one needs the builder's arena, and neither +// exists in a unit that declares neither construct. Same shape as the +// pointer's three: declared here, defined after the walk by whichever half +// the unit carries. Both are OUT-OF-LINE ARRAYS to the descriptors (§8.1): +// array_bound = 0 is the tell, and the type name says which of the two. + +// a map field: an out-of-line array whose type name spells the map +inline bool TableJsonIsMap( const TableFieldInfo * f ); +// the map as a plain JSON object keyed by the KEY, in ASCENDING key order +inline bool TableJsonWriteMap( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// that object back into the slot, in whatever order the text gives it +inline bool TableJsonReadMap( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); +// an unbounded array: the other out-of-line array +inline bool TableJsonIsList( const TableFieldInfo * f ); +// the list as a JSON array, in INDEX order +inline bool TableJsonWriteList( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// that array back into the slot, every element the text carries +inline bool TableJsonReadList( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); + +// ---- json walk: begin ---- +// +// The TEXT form (docs/SPEC-TABLES.md §16): one table, one text, one walk over the +// reflection descriptors (§8). Reading fills ONE caller-owned instance and +// allocates nothing beyond it; writing targets a caller buffer with the +// wire's measure/write symmetry. Everything AROUND this — which file goes +// with which instance, what key an instance is filed under, how instances +// link into a root table's collections — is a packer's opinion and stays +// with the tool that holds it. +// +// The dialect: trailing commas are accepted on read (the authoring files +// this exists for carry them) and never written; comments are not JSON and +// are refused; unknown keys are skipped and counted; a duplicate key is +// last-wins and counted; a key present with the wrong JSON type is skipped +// and counted, never coerced. + +static const int32_t kTableJsonMaxDepth = 128; + +// A key longer than this cannot name a field, so it is skipped as unknown. +static const int32_t kTableJsonMaxKey = 256; + +// The longest numeric token the walk will convert. Anything longer is a +// value no field can hold and counts as a kind mismatch. +static const int32_t kTableJsonMaxNumber = 512; + +// The decimal point the C runtime is CURRENTLY using. Number conversion is +// the one locale-sensitive corner of the grammar — JSON's point is always +// '.', the runtime's is whatever the program set — so every number crosses +// this one character on the way out and on the way back in. Nothing else in +// the walk consults the locale. +inline char TableJsonDecimalPoint() +{ + const struct lconv * conv = localeconv(); + if ( conv != NULL && conv->decimal_point != NULL && conv->decimal_point[0] != 0 ) + { + return conv->decimal_point[0]; + } + return '.'; +} + +// ---- storage access: the descriptors give an offset and a width, and the +// ---- storage is the HOST's, so every load and store goes through a width +// ---- switch rather than a memcpy into the low bytes of a wider word + +// finite: not a NaN, not an infinity. Written without — the walk's +// runtime surface stays the handful of functions it already names. +// A vocabulary entry the descriptor could not spell. The generated name +// functions answer "???" for a value outside the declared set, and that is +// not a name — writing it would put a spelling in the text that the reader +// then counts as unknown, turning a refusal into a silent loss. +inline bool TableJsonNamed( const char * name ) +{ + return name != NULL && strcmp( name, "???" ) != 0; +} + +inline bool TableJsonFinite( double v ) +{ + return v == v && v <= 1.7976931348623157e308 && v >= -1.7976931348623157e308; +} + +inline uint64_t TableJsonGetRaw( const void * storage, uint32_t width ) +{ + switch ( width ) + { + case 1: { uint8_t v = 0; memcpy( &v, storage, 1 ); return v; } + case 2: { uint16_t v = 0; memcpy( &v, storage, 2 ); return v; } + case 4: { uint32_t v = 0; memcpy( &v, storage, 4 ); return v; } + case 8: { uint64_t v = 0; memcpy( &v, storage, 8 ); return v; } + } + return 0; +} + +inline void TableJsonSetRaw( void * storage, uint32_t width, uint64_t value ) +{ + switch ( width ) + { + case 1: { uint8_t v = (uint8_t) value; memcpy( storage, &v, 1 ); break; } + case 2: { uint16_t v = (uint16_t) value; memcpy( storage, &v, 2 ); break; } + case 4: { uint32_t v = (uint32_t) value; memcpy( storage, &v, 4 ); break; } + case 8: { uint64_t v = value; memcpy( storage, &v, 8 ); break; } + } +} + +inline int64_t TableJsonGetSigned( const void * storage, uint32_t width ) +{ + uint64_t raw = TableJsonGetRaw( storage, width ); + if ( width < 8 ) + { + uint64_t sign = uint64_t( 1 ) << ( width * 8 - 1 ); + if ( ( raw & sign ) != 0 ) + { + raw |= ~( ( sign << 1 ) - 1 ); + } + } + return (int64_t) raw; +} + +// ---- the WIDE kinds (docs/SPEC-TABLES.md §3, §16.2) ---- +// +// The 128-bit integers and the fixed-point family convert EXACTLY, over two +// 64-bit lanes: a 128-bit integer is a decimal integer, a fixed value a +// decimal in WHOLE UNITS (1.0, -0.25, 3.0000152587890625) and nothing +// on either path passes through a double. Nothing here needs a 128-bit type +// either, which is what keeps this walk one text for every unit. +struct TableJsonWide +{ + uint64_t lo; + uint64_t hi; +}; + +inline bool TableJsonKindWide( uint8_t kind ) { return kind >= 18 && kind <= 29; } +inline bool TableJsonKindWideSigned( uint8_t kind ) { return kind == 18 || ( kind >= 20 && kind <= 24 ); } +inline bool TableJsonKindFixed( uint8_t kind ) { return kind >= 20 && kind <= 29; } + +inline bool TableJsonWideZero( TableJsonWide v ) { return v.lo == 0 && v.hi == 0; } +inline bool TableJsonWideNegative( TableJsonWide v ) { return ( v.hi >> 63 ) != 0; } + +inline int TableJsonWideCompare( TableJsonWide a, TableJsonWide b, bool is_signed ) +{ + if ( is_signed && TableJsonWideNegative( a ) != TableJsonWideNegative( b ) ) { return TableJsonWideNegative( a ) ? -1 : 1; } + if ( a.hi != b.hi ) { return a.hi < b.hi ? -1 : 1; } + if ( a.lo != b.lo ) { return a.lo < b.lo ? -1 : 1; } + return 0; +} + +inline TableJsonWide TableJsonWideShl( TableJsonWide v, int n ) +{ + TableJsonWide r = { 0, 0 }; + if ( n <= 0 ) { return v; } + if ( n >= 128 ) { return r; } + if ( n >= 64 ) { r.hi = v.lo << ( n - 64 ); return r; } + r.hi = ( v.hi << n ) | ( v.lo >> ( 64 - n ) ); + r.lo = v.lo << n; + return r; +} + +inline TableJsonWide TableJsonWideShr( TableJsonWide v, int n ) +{ + TableJsonWide r = { 0, 0 }; + if ( n <= 0 ) { return v; } + if ( n >= 128 ) { return r; } + if ( n >= 64 ) { r.lo = v.hi >> ( n - 64 ); return r; } + r.lo = ( v.lo >> n ) | ( v.hi << ( 64 - n ) ); + r.hi = v.hi >> n; + return r; +} + +inline TableJsonWide TableJsonWideNeg( TableJsonWide v ) +{ + TableJsonWide r; + r.lo = ~v.lo + 1; + r.hi = ~v.hi + ( r.lo == 0 ? 1 : 0 ); + return r; +} + +// v = v * m + a; the return is the carry out of 128 bits +inline uint32_t TableJsonWideMulAdd( TableJsonWide * v, uint32_t m, uint32_t a ) +{ + uint64_t limb[4] = { v->lo & 0xffffffffull, v->lo >> 32, v->hi & 0xffffffffull, v->hi >> 32 }; + uint64_t carry = a; + for ( int i = 0; i < 4; i++ ) + { + uint64_t p = limb[i] * m + carry; + limb[i] = p & 0xffffffffull; + carry = p >> 32; + } + v->lo = limb[0] | ( limb[1] << 32 ); + v->hi = limb[2] | ( limb[3] << 32 ); + return (uint32_t) carry; +} + +// v = v / d; the return is the remainder +inline uint32_t TableJsonWideDiv( TableJsonWide * v, uint32_t d ) +{ + uint64_t limb[4] = { v->lo & 0xffffffffull, v->lo >> 32, v->hi & 0xffffffffull, v->hi >> 32 }; + uint64_t rem = 0; + for ( int i = 3; i >= 0; i-- ) + { + uint64_t cur = ( rem << 32 ) | limb[i]; + limb[i] = cur / d; + rem = cur % d; + } + v->lo = limb[0] | ( limb[1] << 32 ); + v->hi = limb[2] | ( limb[3] << 32 ); + return (uint32_t) rem; +} + +// The storage of a wide kind, as lanes. A sixteen-byte storage is serialize's +// pair — native __int128 in the host's byte order, or the emulated struct with +// its low lane first — so the lanes are read in the host's order; a narrower +// storage is one lane, sign-extended for a signed kind. +inline TableJsonWide TableJsonWideLoad( const void * storage, uint32_t width, bool is_signed ) +{ + TableJsonWide v = { 0, 0 }; + if ( width == 16 ) + { + uint64_t half[2]; + memcpy( half, storage, 16 ); + uint16_t probe = 1; + bool little = *(const uint8_t *) &probe == 1; + v.lo = little ? half[0] : half[1]; + v.hi = little ? half[1] : half[0]; + return v; + } + v.lo = is_signed ? (uint64_t) TableJsonGetSigned( storage, width ) : TableJsonGetRaw( storage, width ); + v.hi = ( is_signed && ( v.lo >> 63 ) != 0 ) ? ~uint64_t( 0 ) : 0; + return v; +} + +inline void TableJsonWideStore( void * storage, uint32_t width, TableJsonWide v ) +{ + if ( width == 16 ) + { + uint16_t probe = 1; + bool little = *(const uint8_t *) &probe == 1; + uint64_t half[2]; + half[0] = little ? v.lo : v.hi; + half[1] = little ? v.hi : v.lo; + memcpy( storage, half, 16 ); + return; + } + TableJsonSetRaw( storage, width, v.lo ); +} + +// a counted field's companion: a string's length, a bytes' length, a counted +// array's count. Bounded by the declared extent on the way out, so a storage +// invariant a caller broke cannot walk off the end of the array. +inline int32_t TableJsonCount( const void * base, const TableFieldInfo * f ) +{ + if ( !f->counted ) + { + return f->array_bound; + } + int32_t count = 0; + memcpy( &count, (const uint8_t *) base + f->count_offset, sizeof( count ) ); + if ( count < 0 ) { count = 0; } + if ( count > f->array_bound ) { count = f->array_bound; } + return count; +} + +inline void TableJsonSetCount( void * base, const TableFieldInfo * f, int32_t count ) +{ + if ( f->counted ) + { + memcpy( (uint8_t *) base + f->count_offset, &count, sizeof( count ) ); + } +} + +// ---- what a field's kind expects to see in the text ---- +// +// One classifier, consulted by both directions, so a reader and a writer can +// never disagree about a kind's JSON form. 'o' object, 'a' array, 's' +// string, 'n' number, 'b' boolean. +// +// A vocabulary field is spelled by NAME: an enum is one name, a flags mask +// is the array of the names of its set bits. The two are told apart by the +// id column — an enum variant rides under a wire id, a flags BIT never does +// (docs/SPEC-TABLES.md §4), so a name function with no id function is flags. +// +// bytes(N) is the one kind whose element kind does not decide its form: it +// shares u8 with a plain array of u8, and rides as base64. The schema type +// name settles it, and "bytes" is a keyword no declaration can claim. +inline bool TableJsonIsBytes( const TableFieldInfo * f ) +{ + return f->is_array && f->kind == 6 && strcmp( f->type_name, "bytes" ) == 0; +} + +// An ENUM-KEYED array (docs/SPEC-TABLES.md §2.4): its JSON form is an OBJECT +// keyed by variant name, not a positional array, because that is what the +// storage is — one slot per variant, addressed by the variant. +inline bool TableJsonIsKeyed( const TableFieldInfo * f ) +{ + return f->key_name != NULL; +} + +// THE KEY A STORAGE SLOT HOLDS (§2.4, §8): the storage shifts left, so slot i +// holds the key i + 1 and nothing is stored for None. This is the ONE place +// the walker spells the shift. +inline uint64_t TableJsonKeyedSlotKey( int64_t slot ) +{ + return (uint64_t) ( slot + 1 ); +} + +// A slot whose key names a variant of the keying enum. Every slot in +// [0, array_bound) does, unless the enum carries max-headroom variants outside +// a table closure, where a reserved value names nothing and its key id is 0 — +// the reserved id no declared name can fold to (§5). +inline bool TableJsonKeyedSlotValid( const TableFieldInfo * f, int64_t slot ) +{ + return f->key_id( TableJsonKeyedSlotKey( slot ) ) != 0; +} + +inline bool TableJsonIsFlags( const TableFieldInfo * f ) +{ + return f->enum_name != NULL && f->variant_id == NULL; +} + +inline bool TableJsonIsEnum( const TableFieldInfo * f ) +{ + return f->variant_id != NULL && f->arms == NULL; +} + +inline char TableJsonShape( const TableFieldInfo * f ) +{ + if ( TableJsonIsMap( f ) ) return 'o'; // a MAP: an object keyed by the KEY (§2.8) + if ( f->kind == 12 ) return 's'; // string + if ( f->kind == 33 ) return 's'; // wstring: the same text, transcoded (§16.2) + if ( TableJsonIsBytes( f ) ) return 's'; // bytes: base64 + if ( TableJsonIsKeyed( f ) ) return 'o'; // an object keyed by variant NAME + if ( f->is_array ) return 'a'; + if ( f->arms != NULL ) return 'o'; // union: an object with ONE key + if ( f->kind == 13 ) return 'o'; // nested table or type + if ( f->kind == 17 ) return f->table != NULL ? 'o' : 's'; // a pointer: the pointee's object in place, or null (§16.7); a byte buffer's string (§2.5) + if ( TableJsonIsEnum( f ) ) return 's'; + if ( TableJsonIsFlags( f ) ) return 'a'; + if ( f->kind == 1 ) return 'b'; + return 'n'; +} + +// the ELEMENT shape of an array field — the same classifier one level down +inline char TableJsonElementShape( const TableFieldInfo * f ) +{ + if ( f->arms != NULL ) return 'o'; // an element of an array of unions: one key, the arm (§2.6) + if ( f->kind == 13 ) return 'o'; + if ( TableJsonIsEnum( f ) ) return 's'; + if ( TableJsonIsFlags( f ) ) return 'a'; + if ( f->kind == 1 ) return 'b'; + return 'n'; +} + +// A guarded group rides only when its guard reads true — the wire's own +// elision (§4), carried into the text so a text and a wire written from one +// instance say the same thing. The guard is spelled as its branch condition +// over bool fields of the SAME type ("at_rest", "!at_rest", +// "active && has_target"), so evaluating it is a walk of the same +// descriptor. Nothing is inferred in the other direction: reading places +// every key it can name, and the guard is a plain bool key (§16.2). +inline bool TableJsonGuardHolds( const void * base, const TableTypeInfo * info, const char * guard ) +{ + const char * p = guard; + for ( ;; ) + { + while ( *p == ' ' || *p == '&' ) { p++; } + if ( *p == 0 ) { return true; } + bool want = true; + if ( *p == '!' ) { want = false; p++; } + const char * start = p; + while ( *p != 0 && *p != ' ' && *p != '&' ) { p++; } + size_t length = (size_t) ( p - start ); + bool value = false; + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + const TableFieldInfo * f = &info->fields[i]; + if ( strlen( f->name ) == length && strncmp( f->name, start, length ) == 0 ) + { + value = TableJsonGetRaw( (const uint8_t *) base + f->offset, f->elem_size ) != 0; + break; + } + } + if ( value != want ) { return false; } + } +} + +// ---- writing ---- + +// The writer sink MEASURES when the buffer is NULL and WRITES when it is +// not, over one code path — so measure and write agree byte for byte, the +// wire's invariant (§9) carried across. +struct TableJsonOut +{ + char * buffer; + int64_t capacity; + int64_t offset; + bool overflow; + void * graph; // the pointered write's identity map (§16.7); NULL for a fixed table + + void raw( const char * data, int64_t count ) + { + if ( buffer != NULL ) + { + if ( offset + count > capacity ) { overflow = true; return; } + memcpy( buffer + offset, data, (size_t) count ); + } + offset += count; + } + void put( char c ) { raw( &c, 1 ); } + void text( const char * s ) { raw( s, (int64_t) strlen( s ) ); } + void line( int32_t depth ) + { + put( '\n' ); + for ( int32_t i = 0; i < depth; i++ ) { raw( " ", 2 ); } + } +}; + +inline const char * TableJsonBase64Alphabet() +{ + return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +} + +inline void TableJsonWriteBase64( TableJsonOut & out, const uint8_t * data, int32_t length ) +{ + const char * alphabet = TableJsonBase64Alphabet(); + out.put( '"' ); + int32_t i = 0; + for ( ; i + 3 <= length; i += 3 ) + { + uint32_t triple = ( uint32_t( data[i] ) << 16 ) | ( uint32_t( data[i+1] ) << 8 ) | uint32_t( data[i+2] ); + char quad[4] = { alphabet[ ( triple >> 18 ) & 0x3f ], alphabet[ ( triple >> 12 ) & 0x3f ], + alphabet[ ( triple >> 6 ) & 0x3f ], alphabet[ triple & 0x3f ] }; + out.raw( quad, 4 ); + } + if ( i < length ) + { + int32_t left = length - i; + uint32_t triple = uint32_t( data[i] ) << 16; + if ( left == 2 ) { triple |= uint32_t( data[i+1] ) << 8; } + char quad[4] = { alphabet[ ( triple >> 18 ) & 0x3f ], alphabet[ ( triple >> 12 ) & 0x3f ], '=', '=' }; + if ( left == 2 ) { quad[2] = alphabet[ ( triple >> 6 ) & 0x3f ]; } + out.raw( quad, 4 ); + } + out.put( '"' ); +} + +// One UTF-8 sequence at s, or -1 when the bytes there are not one. Rejects +// the lot: a stray continuation, an overlong form, a surrogate half, and +// anything past U+10FFFF. +inline int32_t TableJsonUtf8( const char * s, int32_t remaining, int32_t * width ) +{ + unsigned char lead = (unsigned char) s[0]; + int32_t want = 0; + int32_t code = 0; + if ( lead < 0x80 ) { *width = 1; return lead; } + else if ( lead >= 0xc2 && lead <= 0xdf ) { want = 2; code = lead & 0x1f; } + else if ( lead >= 0xe0 && lead <= 0xef ) { want = 3; code = lead & 0x0f; } + else if ( lead >= 0xf0 && lead <= 0xf4 ) { want = 4; code = lead & 0x07; } + else { return -1; } + if ( remaining < want ) { return -1; } + for ( int32_t i = 1; i < want; i++ ) + { + unsigned char next = (unsigned char) s[i]; + if ( ( next & 0xc0 ) != 0x80 ) { return -1; } + code = ( code << 6 ) | ( next & 0x3f ); + } + if ( want == 3 && code < 0x800 ) { return -1; } // overlong + if ( want == 4 && code < 0x10000 ) { return -1; } // overlong + if ( code >= 0xd800 && code <= 0xdfff ) { return -1; } // a surrogate half + if ( code > 0x10ffff ) { return -1; } + *width = want; + return code; +} + +// The inverse: one code point encoded as UTF-8 into unit, its length +// answered. Both text kinds' writers reach it, and so does the escape +// grammar's U+FFFD replacement. +inline int32_t TableJsonEncodeUtf8( uint32_t code, char * unit ) +{ + if ( code < 0x80 ) { unit[0] = (char) code; return 1; } + if ( code < 0x800 ) + { + unit[0] = (char) ( 0xc0 | ( code >> 6 ) ); + unit[1] = (char) ( 0x80 | ( code & 0x3f ) ); + return 2; + } + if ( code < 0x10000 ) + { + unit[0] = (char) ( 0xe0 | ( code >> 12 ) ); + unit[1] = (char) ( 0x80 | ( ( code >> 6 ) & 0x3f ) ); + unit[2] = (char) ( 0x80 | ( code & 0x3f ) ); + return 3; + } + unit[0] = (char) ( 0xf0 | ( code >> 18 ) ); + unit[1] = (char) ( 0x80 | ( ( code >> 12 ) & 0x3f ) ); + unit[2] = (char) ( 0x80 | ( ( code >> 6 ) & 0x3f ) ); + unit[3] = (char) ( 0x80 | ( code & 0x3f ) ); + return 4; +} + +// A JSON text MUST be valid UTF-8 (RFC 8259 §8.1). The read path is +// byte-transparent — the wire imposes no encoding (§3) and a string may hold +// anything — so the WRITER is where that obligation is met: a byte that is +// not part of a well-formed sequence is written as U+FFFD, one per bad byte, +// and never raw. A text this walk writes is therefore readable by any +// conforming parser, which a raw byte would not be. The cost is stated +// plainly: for a string holding invalid UTF-8, the round trip is NOT +// byte-identical, because the alternative is emitting a text that is not +// JSON. +inline void TableJsonWriteString( TableJsonOut & out, const char * s, int32_t length ) +{ + static const char hex[] = "0123456789abcdef"; + out.put( '"' ); + for ( int32_t i = 0; i < length; i++ ) + { + unsigned char c = (unsigned char) s[i]; + switch ( c ) + { + case '"': out.raw( "\\\"", 2 ); break; + case '\\': out.raw( "\\\\", 2 ); break; + case '\b': out.raw( "\\b", 2 ); break; + case '\f': out.raw( "\\f", 2 ); break; + case '\n': out.raw( "\\n", 2 ); break; + case '\r': out.raw( "\\r", 2 ); break; + case '\t': out.raw( "\\t", 2 ); break; + default: + if ( c < 0x20 ) + { + char escape[6] = { '\\', 'u', '0', '0', hex[ c >> 4 ], hex[ c & 0xf ] }; + out.raw( escape, 6 ); + } + else if ( c < 0x80 ) + { + out.put( (char) c ); + } + else + { + int32_t width = 0; + if ( TableJsonUtf8( s + i, length - i, &width ) < 0 ) + { + out.raw( "\xef\xbf\xbd", 3 ); // U+FFFD, one per bad byte + } + else + { + out.raw( s + i, width ); + i += width - 1; + } + } + break; + } + } + out.put( '"' ); +} + +// A WIDE field's text: the code units transcoded back to UTF-8 (§16.2). A +// SURROGATE PAIR is one code point; an UNPAIRED SURROGATE is not a code point +// at all, encodes to nothing, and writes one U+FFFD per ill-formed unit; a +// ZERO UNIT is U+0000, which JSON has an escape for, and writes \u0000 +// (§16.3). No wire can put either into storage (§3), so both answer for +// storage a PROGRAM built. +inline void TableJsonWriteWString( TableJsonOut & out, const char16_t * s, int32_t length ) +{ + static const char hex[] = "0123456789abcdef"; + out.put( '"' ); + for ( int32_t i = 0; i < length; i++ ) + { + uint32_t code = (uint32_t) (uint16_t) s[i]; + if ( code >= 0xd800 && code <= 0xdbff && i + 1 < length ) + { + const uint32_t low = (uint32_t) (uint16_t) s[i + 1]; + if ( low >= 0xdc00 && low <= 0xdfff ) + { + code = 0x10000 + ( ( code - 0xd800 ) << 10 ) + ( low - 0xdc00 ); + i++; + } + } + if ( code >= 0xd800 && code <= 0xdfff ) { code = 0xfffd; } // an unpaired surrogate + switch ( code ) + { + case '"': out.raw( "\\\"", 2 ); continue; + case '\\': out.raw( "\\\\", 2 ); continue; + case '\b': out.raw( "\\b", 2 ); continue; + case '\f': out.raw( "\\f", 2 ); continue; + case '\n': out.raw( "\\n", 2 ); continue; + case '\r': out.raw( "\\r", 2 ); continue; + case '\t': out.raw( "\\t", 2 ); continue; + default: break; + } + if ( code < 0x20 ) + { + char escape[6] = { '\\', 'u', '0', '0', hex[ code >> 4 ], hex[ code & 0xf ] }; + out.raw( escape, 6 ); + continue; + } + char encoded[4]; + const int32_t encoded_length = TableJsonEncodeUtf8( code, encoded ); + out.raw( encoded, encoded_length ); + } + out.put( '"' ); +} + +inline void TableJsonWriteUnsigned( TableJsonOut & out, uint64_t value ) +{ + char digits[24]; + int32_t n = 0; + do + { + digits[n++] = (char) ( '0' + (int) ( value % 10 ) ); + value /= 10; + } while ( value != 0 ); + char text[24]; + for ( int32_t i = 0; i < n; i++ ) { text[i] = digits[n - 1 - i]; } + out.raw( text, n ); +} + +inline void TableJsonWriteSigned( TableJsonOut & out, int64_t value ) +{ + if ( value < 0 ) + { + out.put( '-' ); + TableJsonWriteUnsigned( out, uint64_t( 0 ) - (uint64_t) value ); + return; + } + TableJsonWriteUnsigned( out, (uint64_t) value ); +} + +// A wide kind writes its raw storage as §16.2's text: a 128-bit integer as a +// decimal integer; a fixed value in WHOLE UNITS as the shortest exact decimal +// with at least one fractional digit (1.0, -0.25), the spelling the schema text +// gives a fixed default. The fraction terminates because a dyadic fraction has +// a finite decimal expansion — at most F digits. +inline void TableJsonWriteWide( TableJsonOut & out, const void * storage, const TableFieldInfo * f ) +{ + bool is_signed = TableJsonKindWideSigned( f->kind ); + TableJsonWide v = TableJsonWideLoad( storage, f->elem_size, is_signed ); + if ( is_signed && TableJsonWideNegative( v ) ) + { + out.put( '-' ); + v = TableJsonWideNeg( v ); + } + int frac = f->frac_bits; + TableJsonWide whole = TableJsonWideShr( v, frac ); + char digits[40]; + int32_t n = 0; + do + { + digits[n++] = (char) ( '0' + (int) TableJsonWideDiv( &whole, 10 ) ); + } while ( !TableJsonWideZero( whole ) ); + char text[40]; + for ( int32_t i = 0; i < n; i++ ) { text[i] = digits[n - 1 - i]; } + out.raw( text, n ); + if ( !TableJsonKindFixed( f->kind ) ) { return; } + out.put( '.' ); + // the fraction bits alone: v with everything at and above bit F cleared + TableJsonWide fraction = v; + if ( frac < 64 ) { fraction.hi = 0; fraction.lo &= ( uint64_t( 1 ) << frac ) - 1; } + else { fraction.hi &= ( uint64_t( 1 ) << ( frac - 64 ) ) - 1; } + if ( frac == 0 ) { fraction.lo = 0; } + if ( TableJsonWideZero( fraction ) ) + { + out.put( '0' ); + return; + } + while ( !TableJsonWideZero( fraction ) ) + { + // ×10: the digit is what lands at and above bit F, including the + // carry out of 128 bits when F leaves no room for it below + uint32_t carry = TableJsonWideMulAdd( &fraction, 10, 0 ); + uint64_t digit = TableJsonWideShr( fraction, frac ).lo; + if ( frac > 64 ) { digit |= uint64_t( carry ) << ( 128 - frac ); } + out.put( (char) ( '0' + (int) digit ) ); + if ( frac < 64 ) { fraction.hi = 0; fraction.lo &= ( uint64_t( 1 ) << frac ) - 1; } + else { fraction.hi &= ( uint64_t( 1 ) << ( frac - 64 ) ) - 1; } + } +} + +// A float writes at the SHORTEST precision that reads back as the same value +// at the field's own width, so a round trip is exact and a text stays +// readable. Non-finite values have no JSON spelling at all, and the writer +// REFUSES rather than losing one silently — the same rule measure and save +// already apply to an enum value no variant names (§5). +inline bool TableJsonWriteFloat( TableJsonOut & out, double value, bool single ) +{ + if ( !TableJsonFinite( value ) ) { return false; } + char text[64]; + int low = single ? 6 : 15; + int high = single ? 9 : 17; + int length = 0; + for ( int digits = low; ; digits++ ) + { + length = snprintf( text, sizeof( text ), "%.*g", digits, value ); + if ( length <= 0 || length >= (int) sizeof( text ) ) { return false; } + if ( digits >= high ) { break; } + // the round-trip check runs BEFORE the decimal point is normalised: + // the token still carries whatever point snprintf just produced + if ( single ) + { + if ( (double) strtof( text, NULL ) == value ) { break; } + } + else + { + if ( strtod( text, NULL ) == value ) { break; } + } + } + char point = TableJsonDecimalPoint(); + if ( point != '.' ) + { + for ( int i = 0; i < length; i++ ) + { + if ( text[i] == point ) { text[i] = '.'; } + } + } + out.raw( text, length ); + return true; +} + +inline bool TableJsonWriteValue( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth ); +// a UNION ARM that names no declaration writes through the field walk one key +// down (docs/SPEC-TABLES.md §2.6, §16.2), which is defined below +inline bool TableJsonWriteField( TableJsonOut & out, const void * base, const TableFieldInfo * f, int32_t depth ); + +// one scalar, at one storage address: a nested object, a union, a +// vocabulary, or a number +inline bool TableJsonWriteScalar( TableJsonOut & out, const void * storage, const TableFieldInfo * f, int32_t depth ) +{ + if ( f->arms != NULL ) + { + // a union is an object with ONE key, the arm's name; None is {} + const TableUnionInfo * arms = f->arms(); + uint64_t tag = TableJsonGetRaw( (const uint8_t *) storage + arms->tag_offset, arms->tag_size ); + if ( tag == 0 ) + { + out.raw( "{}", 2 ); + return true; + } + if ( (int64_t) tag > f->enum_max ) + { + return false; // a tag no arm names, exactly as measure refuses it + } + const char * arm = f->enum_name( tag ); + // and refuse on the NAME, not merely on the bound: §16.2 says a value + // no variant NAMES is refused, so the check is the name. Writing + // whatever came back would emit "???", a spelling the reader counts + // as unknown — a silent round-trip loss in place of a refusal. + if ( !TableJsonNamed( arm ) ) { return false; } + out.put( '{' ); + out.line( depth + 1 ); + TableJsonWriteString( out, arm, (int32_t) strlen( arm ) ); + out.raw( ": ", 2 ); + // THE ARM'S VALUE TAKES THE ARM'S OWN ROW (§16.2): an arm that names + // no declaration carries the FIELD descriptor a field of its type + // would carry, offsets taken inside the union storage (§2.6), so the + // value walks through the field writer one key down. + if ( arms->arms[tag].field != NULL ) + { + if ( !TableJsonWriteField( out, storage, arms->arms[tag].field, depth + 1 ) ) + { + return false; + } + } + else if ( arms->arms[tag].table == NULL ) + { + out.raw( "null", 4 ); // a payload-free arm: the name selects it (§2.6) + } + else if ( !TableJsonWriteValue( out, (const uint8_t *) storage + arms->arms[tag].offset, arms->arms[tag].table, depth + 1 ) ) + { + return false; + } + out.line( depth ); + out.put( '}' ); + return true; + } + if ( f->kind == 13 ) + { + return TableJsonWriteValue( out, storage, f->table, depth ); + } + if ( TableJsonIsEnum( f ) ) + { + uint64_t value = TableJsonGetRaw( storage, f->elem_size ); + // a value no variant names has no text spelling, exactly as it has no + // wire identity: the writer REFUSES rather than writing None over it, + // the rule measure and save already apply (docs/SPEC-TABLES.md §5) + if ( (int64_t) value > f->enum_max ) { return false; } + if ( value != 0 && f->variant_id( value ) == 0 ) { return false; } + const char * name = f->enum_name( value ); + if ( !TableJsonNamed( name ) ) { return false; } + TableJsonWriteString( out, name, (int32_t) strlen( name ) ); + return true; + } + if ( TableJsonIsFlags( f ) ) + { + uint64_t bits = TableJsonGetRaw( storage, f->elem_size ); + if ( bits == 0 ) + { + out.raw( "[]", 2 ); + return true; + } + out.put( '[' ); + bool first = true; + for ( int64_t bit = 0; bit < 64; bit++ ) + { + if ( ( bits & ( uint64_t( 1 ) << bit ) ) == 0 ) { continue; } + if ( bit > f->enum_max ) + { + return false; // a bit no variant names has no text spelling + } + const char * name = f->enum_name( (uint64_t) bit ); + if ( !TableJsonNamed( name ) ) { return false; } + if ( !first ) { out.put( ',' ); } + first = false; + out.line( depth + 1 ); + TableJsonWriteString( out, name, (int32_t) strlen( name ) ); + } + out.line( depth ); + out.put( ']' ); + return true; + } + switch ( f->kind ) + { + case 1: + out.text( TableJsonGetRaw( storage, f->elem_size ) != 0 ? "true" : "false" ); + return true; + case 10: + { + float v = 0.0f; + memcpy( &v, storage, sizeof( v ) ); + return TableJsonWriteFloat( out, (double) v, true ); + } + case 11: + { + double v = 0.0; + memcpy( &v, storage, sizeof( v ) ); + return TableJsonWriteFloat( out, v, false ); + } + case 2: case 3: case 4: case 5: + TableJsonWriteSigned( out, TableJsonGetSigned( storage, f->elem_size ) ); + return true; + default: + if ( TableJsonKindWide( f->kind ) ) + { + TableJsonWriteWide( out, storage, f ); + return true; + } + TableJsonWriteUnsigned( out, TableJsonGetRaw( storage, f->elem_size ) ); + return true; + } +} + +inline bool TableJsonWriteField( TableJsonOut & out, const void * base, const TableFieldInfo * f, int32_t depth ) +{ + const uint8_t * storage = (const uint8_t *) base + f->offset; + if ( TableJsonIsMap( f ) ) + { + return TableJsonWriteMap( out, (const void *) storage, f, depth ); + } + if ( TableJsonIsList( f ) ) + { + return TableJsonWriteList( out, (const void *) storage, f, depth ); + } + if ( f->kind == 17 && !f->is_array ) + { + return TableJsonWritePointer( out, storage, f, depth ); + } + if ( f->kind == 17 ) + { + // an ARRAY OF POINTERS (§2.1): the pointer row per element — the + // pointee's object in place, null, or `&node` for a shared one (§16.7) + int32_t count = TableJsonCount( base, f ); + if ( count == 0 ) { out.raw( "[]", 2 ); return true; } + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + if ( !TableJsonWritePointer( out, storage + (int64_t) i * f->elem_size, f, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( ']' ); + return true; + } + if ( f->kind == 12 ) + { + TableJsonWriteString( out, (const char *) storage, TableJsonCount( base, f ) ); + return true; + } + if ( f->kind == 33 ) + { + TableJsonWriteWString( out, (const char16_t *) (const void *) storage, TableJsonCount( base, f ) ); + return true; + } + if ( TableJsonIsBytes( f ) ) + { + TableJsonWriteBase64( out, storage, TableJsonCount( base, f ) ); + return true; + } + if ( TableJsonIsKeyed( f ) ) + { + // one entry per SLOT, keyed by the variant that owns it, so inserting + // a variant next season moves nothing in the text either. Slot i holds + // the key i + 1: nothing is stored for None, so nothing is written for it. + out.put( '{' ); + bool first = true; + for ( int64_t slot = 0; slot < f->array_bound; slot++ ) + { + if ( !TableJsonKeyedSlotValid( f, slot ) ) { continue; } + if ( !first ) { out.put( ',' ); } + first = false; + out.line( depth + 1 ); + const char * key = f->key_name( TableJsonKeyedSlotKey( slot ) ); + TableJsonWriteString( out, key, (int32_t) strlen( key ) ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteScalar( out, storage + slot * f->elem_size, f, depth + 1 ) ) + { + return false; + } + } + if ( first ) { out.raw( "}", 1 ); return true; } + out.line( depth ); + out.put( '}' ); + return true; + } + if ( f->is_array ) + { + int32_t count = TableJsonCount( base, f ); + if ( count == 0 ) + { + out.raw( "[]", 2 ); + return true; + } + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + if ( !TableJsonWriteScalar( out, storage + (int64_t) i * f->elem_size, f, depth + 1 ) ) + { + return false; + } + } + out.line( depth ); + out.put( ']' ); + return true; + } + return TableJsonWriteScalar( out, storage, f, depth ); +} + +// One instance's fields, in DECLARATION ORDER, defaults included — a text is +// for people and tools, and a text that elides is a text a reader has to know +// the schema to complete. `any` says whether the object is already open on +// entry — a shared node's `&node` opens it before the fields (§16.7) — and +// whether it is open on return. +inline bool TableJsonWriteFields( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth, bool & any ) +{ + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + const TableFieldInfo * f = &info->fields[i]; + if ( f->guard[0] != 0 && !TableJsonGuardHolds( base, info, f->guard ) ) { continue; } + // an ABSENT optional writes no key: presence of the key IS the + // presence (§16.2), so an absent field is an absent key and nothing + // else would read back as absent + if ( f->optional && + TableJsonGetRaw( (const uint8_t *) base + f->present_offset, 1 ) == 0 ) + { + continue; + } + if ( !any ) { out.put( '{' ); } + else { out.put( ',' ); } + any = true; + out.line( depth + 1 ); + TableJsonWriteString( out, f->json, (int32_t) strlen( f->json ) ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteField( out, base, f, depth + 1 ) ) { return false; } + } + return true; +} + +// One instance as one object. The writer carries the reader's depth cap +// (§16.2): a pointer chain nests as deep as it is long (§16.7), and a text the +// writer produced past the cap would be a text the reader refuses. +inline bool TableJsonWriteValue( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { return false; } + bool any = false; + if ( !TableJsonWriteFields( out, base, info, depth, any ) ) { return false; } + if ( !any ) + { + out.raw( "{}", 2 ); + return true; + } + out.line( depth ); + out.put( '}' ); + return true; +} + +// ---- reading ---- + +struct TableJsonIn +{ + const char * text; + int64_t size; + int64_t pos; + TableReport * report; + bool bad; // the text is not JSON: the walk stops and keeps what it placed + void * graph; // the pointered read's builder and label map (§16.7); NULL for a fixed table +}; + +inline void TableJsonSpace( TableJsonIn & in ) +{ + while ( in.pos < in.size ) + { + char c = in.text[in.pos]; + if ( c == ' ' || c == '\t' || c == '\n' || c == '\r' ) { in.pos++; continue; } + // COMMENTS ARE ACCEPTED ON READ AND NEVER WRITTEN (docs/SPEC-TABLES.md + // §16.2): a line comment runs to the end of the line or of the input, + // a block comment to its closing delimiter, which does not nest, and + // an UNCLOSED block comment is malformed on the terms an unclosed + // string is. Both are legal wherever whitespace is; a lone slash is not JSON. + if ( c == '/' && in.pos + 1 < in.size && in.text[in.pos + 1] == '/' ) + { + in.pos += 2; + while ( in.pos < in.size && in.text[in.pos] != '\n' ) { in.pos++; } + continue; + } + if ( c == '/' && in.pos + 1 < in.size && in.text[in.pos + 1] == '*' ) + { + int64_t at = in.pos + 2; + while ( at + 1 < in.size && !( in.text[at] == '*' && in.text[at + 1] == '/' ) ) { at++; } + if ( at + 1 >= in.size ) { in.bad = true; in.pos = in.size; return; } + in.pos = at + 2; + continue; + } + if ( c == '/' ) { in.bad = true; } + return; + } +} + +inline char TableJsonPeek( TableJsonIn & in ) +{ + TableJsonSpace( in ); + return in.pos < in.size ? in.text[in.pos] : 0; +} + +// the shape of the value sitting at the cursor, without consuming it +inline char TableJsonValueShape( TableJsonIn & in ) +{ + char c = TableJsonPeek( in ); + switch ( c ) + { + case '{': return 'o'; + case '[': return 'a'; + case '"': return 's'; + case 't': case 'f': return 'b'; + case 'n': return 'z'; + case 0: return 0; + default: return 'n'; + } +} + +inline bool TableJsonLiteral( TableJsonIn & in, const char * word ) +{ + int64_t length = (int64_t) strlen( word ); + if ( in.pos + length > in.size || memcmp( in.text + in.pos, word, (size_t) length ) != 0 ) + { + in.bad = true; + return false; + } + in.pos += length; + return true; +} + +// one \uXXXX escape body; -1 when the four hex digits are not there +inline int TableJsonHex4( TableJsonIn & in ) +{ + if ( in.pos + 4 > in.size ) { return -1; } + int value = 0; + for ( int i = 0; i < 4; i++ ) + { + char c = in.text[in.pos + i]; + int digit; + if ( c >= '0' && c <= '9' ) { digit = c - '0'; } + else if ( c >= 'a' && c <= 'f' ) { digit = c - 'a' + 10; } + else if ( c >= 'A' && c <= 'F' ) { digit = c - 'A' + 10; } + else { return -1; } + value = ( value << 4 ) | digit; + } + in.pos += 4; + return value; +} + + +// One STRING BODY CHARACTER at the cursor, encoded into unit as UTF-8 and +// its length answered: an escape's code point, or a UTF-8 sequence read +// whole. It is ONE grammar serving both text kinds — the narrow scan places +// these bytes and the wide scan converts them back to code units — so the +// escape table, the lone-surrogate rule and the U+FFFD replacement are stated +// once. false means the text is not JSON and in.bad says so; a returned +// length of 0 means the closing quote was consumed and the string is done. +inline bool TableJsonScanUnit( TableJsonIn & in, char * unit, int32_t * unit_length_out ) +{ + int32_t unit_length = 0; + *unit_length_out = 0; + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos]; + if ( c == '"' ) { in.pos++; return true; } + if ( c == '\\' ) + { + in.pos++; + if ( in.pos >= in.size ) { in.bad = true; return false; } + char escape = in.text[in.pos++]; + switch ( escape ) + { + case '"': unit[0] = '"'; unit_length = 1; break; + case '\\': unit[0] = '\\'; unit_length = 1; break; + case '/': unit[0] = '/'; unit_length = 1; break; + case 'b': unit[0] = '\b'; unit_length = 1; break; + case 'f': unit[0] = '\f'; unit_length = 1; break; + case 'n': unit[0] = '\n'; unit_length = 1; break; + case 'r': unit[0] = '\r'; unit_length = 1; break; + case 't': unit[0] = '\t'; unit_length = 1; break; + case 'u': + { + int high = TableJsonHex4( in ); + if ( high < 0 ) { in.bad = true; return false; } + uint32_t code = (uint32_t) high; + if ( high >= 0xd800 && high <= 0xdbff && in.pos + 2 <= in.size && + in.text[in.pos] == '\\' && in.text[in.pos + 1] == 'u' ) + { + int64_t mark = in.pos; + in.pos += 2; + int low = TableJsonHex4( in ); + if ( low >= 0xdc00 && low <= 0xdfff ) + { + code = 0x10000 + ( ( (uint32_t) high - 0xd800 ) << 10 ) + ( (uint32_t) low - 0xdc00 ); + } + else + { + in.pos = mark; // a lone lead surrogate rides as itself + } + } + // a surrogate half that never found its partner has no + // UTF-8 encoding: encoding it anyway would manufacture + // CESU-8 — invalid UTF-8 — out of input that was valid + // JSON, so it reads as the replacement character + if ( code >= 0xd800 && code <= 0xdfff ) { code = 0xfffd; } + unit_length = TableJsonEncodeUtf8( code, unit ); + break; + } + default: in.bad = true; return false; + } + } + else if ( (unsigned char) c < 0x20 ) + { + in.bad = true; // a raw control character is not a JSON string body + return false; + } + else + { + // a UTF-8 sequence read WHOLE, so the clamp below can only land + // between code points. Only bytes that ACTUALLY look like + // continuations are taken: the wire imposes no encoding (§3), so + // a string may legitimately hold a stray lead byte, and one at + // the end of a text must not swallow the closing quote. + unsigned char lead = (unsigned char) c; + int32_t want = 1; + if ( ( lead & 0xe0 ) == 0xc0 ) { want = 2; } + else if ( ( lead & 0xf0 ) == 0xe0 ) { want = 3; } + else if ( ( lead & 0xf8 ) == 0xf0 ) { want = 4; } + unit[0] = c; + in.pos++; + unit_length = 1; + while ( unit_length < want && in.pos < in.size && + ( (unsigned char) in.text[in.pos] & 0xc0 ) == 0x80 ) + { + unit[unit_length++] = in.text[in.pos++]; + } + // A SEQUENCE THAT IS NOT A CODE POINT READS AS U+FFFD, which is + // §16.3's rule at the point the defect ENTERS rather than at the + // point it leaves: a lone surrogate escape already reads that way + // above, RFC 8259 requires a JSON text to be valid UTF-8, and a + // kind 12 payload is well-formed UTF-8 (§3), so storage the text + // form built has to be storage the wire can carry (§5). + if ( !TableUtf8Valid( (const uint8_t *) unit, unit_length ) ) + { + unit_length = TableJsonEncodeUtf8( 0xfffd, unit ); + } + } + } + *unit_length_out = unit_length; + return true; +} + +// Scan one JSON string into a caller buffer. Bytes are appended ONE CODE +// POINT AT A TIME — an escape's encoding, or a UTF-8 sequence read whole — +// so a string longer than the field is clamped AT A CODE POINT BOUNDARY and +// never cut through a multi-byte character. Clamping is counted, never +// fatal, exactly as it is on the wire (§4). A NULL destination scans past a +// string without keeping it. +// +// A CALLER THAT TAKES clamped_out OWNS THE COUNTER. The value paths leave it +// NULL, and the clamp is a value's clamp, counted here. A MAP KEY takes it, +// because a key never clamps: a key this buffer could not hold whole is not a +// shorter key, and its entry drops instead (§2.8). +inline bool TableJsonScanString( TableJsonIn & in, char * out, int32_t capacity, int32_t * length, + bool * clamped_out = NULL ) +{ + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + int32_t placed = 0; + bool clamped = false; + for ( ;; ) + { + char unit[4]; + int32_t unit_length = 0; + if ( !TableJsonScanUnit( in, unit, &unit_length ) ) { return false; } + if ( unit_length == 0 ) { break; } + if ( out == NULL ) + { + placed += unit_length; // measured and not kept: a byte buffer's read sizes its node this way (§2.5) + } + else if ( !clamped && placed + unit_length <= capacity ) + { + memcpy( out + placed, unit, (size_t) unit_length ); + placed += unit_length; + } + else + { + // A CLAMP IS A PREFIX. Once one code point does not fit, the scan + // stops placing: a later SHORTER code point sliding into the room + // the long one left would store a string the input never spelled, + // and one clamped count cannot tell the two apart. + clamped = true; + } + } + if ( clamped_out != NULL ) { *clamped_out = clamped; } + else if ( clamped ) { in.report->clamped++; } + if ( length != NULL ) { *length = placed; } + return true; +} + +// One UTF-8 sequence back to its CODE POINT, over bytes TableJsonScanUnit +// produced and therefore already well formed. It is the inverse of +// TableJsonEncodeUtf8 and nothing more. +inline uint32_t TableJsonDecodeUtf8( const char * unit, int32_t unit_length ) +{ + const unsigned char lead = (unsigned char) unit[0]; + if ( unit_length == 1 ) { return lead; } + uint32_t code = lead & ( unit_length == 2 ? 0x1fu : ( unit_length == 3 ? 0x0fu : 0x07u ) ); + for ( int32_t i = 1; i < unit_length; i++ ) + { + code = ( code << 6 ) | (uint32_t) ( (unsigned char) unit[i] & 0x3f ); + } + return code; +} + +// Scan one JSON string into a caller buffer of UTF-16 CODE UNITS: the wstring +// row of §16.2, the text TRANSCODED at the boundary. It shares +// TableJsonScanUnit with the narrow scan, so the escape grammar and the +// lone-surrogate rule are one grammar, and appends ONE CODE POINT AT A TIME — +// one unit below U+10000 and a surrogate PAIR above it. A string longer than +// the field is therefore clamped at N code units with a pair never split, and +// a high surrogate left without its low half is dropped with it, which is the +// same sentence the wire's clamp takes (§3). Clamping is counted, never fatal. +inline bool TableJsonScanWString( TableJsonIn & in, char16_t * out, int32_t capacity, int32_t * length ) +{ + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + int32_t placed = 0; + bool clamped = false; + for ( ;; ) + { + char unit[4]; + int32_t unit_length = 0; + if ( !TableJsonScanUnit( in, unit, &unit_length ) ) { return false; } + if ( unit_length == 0 ) { break; } + const uint32_t code = TableJsonDecodeUtf8( unit, unit_length ); + char16_t units[2]; + int32_t units_length = 1; + if ( code < 0x10000 ) + { + units[0] = (char16_t) code; + } + else + { + const uint32_t rest = code - 0x10000; + units[0] = (char16_t) ( 0xd800 + ( rest >> 10 ) ); + units[1] = (char16_t) ( 0xdc00 + ( rest & 0x3ff ) ); + units_length = 2; + } + if ( out == NULL ) + { + placed += units_length; + } + else if ( !clamped && placed + units_length <= capacity ) + { + for ( int32_t i = 0; i < units_length; i++ ) { out[placed + i] = units[i]; } + placed += units_length; + } + else + { + // A CLAMP IS A PREFIX, the narrow scan's own rule: once one code + // point does not fit, the scan stops placing. A pair is placed + // whole or not at all, so no clamp can leave an unpaired + // surrogate in storage. + clamped = true; + } + } + if ( clamped ) { in.report->clamped++; } + if ( length != NULL ) { *length = placed; } + return true; +} + + +// the numeric token at the cursor, copied out whole; false = not a number +// Scan one number, to JSON's OWN grammar (RFC 8259 §6) and not to a run of +// number-ish characters: +// +// number = [ "-" ] int [ frac ] [ exp ] +// int = "0" / ( digit1-9 *digit ) +// frac = "." 1*digit +// exp = ( "e" / "E" ) [ "-" / "+" ] 1*digit +// +// Scanning the production is what makes a typo in an authoring file a +// DIAGNOSTIC rather than a value: "1-2" scans as 1 and leaves "-2" where the +// object expects a comma, so the text is malformed — which is what §16.2 +// already promises. A permissive scan would hand "1-2" to a digit loop and +// report a clamp, and a config pipeline would never hear about it. Leading +// "+", leading zeros, ".5" and "3." are not JSON either. +inline bool TableJsonWalkNumber( TableJsonIn & in, bool * integral ) +{ + TableJsonSpace( in ); + bool whole = true; + if ( in.pos < in.size && in.text[in.pos] == '-' ) { in.pos++; } + // int: a lone zero, or a non-zero digit and any digits after it + if ( in.pos >= in.size ) { return false; } + if ( in.text[in.pos] == '0' ) + { + in.pos++; + } + else if ( in.text[in.pos] >= '1' && in.text[in.pos] <= '9' ) + { + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + } + else + { + return false; + } + // frac + if ( in.pos < in.size && in.text[in.pos] == '.' ) + { + in.pos++; + if ( in.pos >= in.size || in.text[in.pos] < '0' || in.text[in.pos] > '9' ) { return false; } + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + whole = false; + } + // exp + if ( in.pos < in.size && ( in.text[in.pos] == 'e' || in.text[in.pos] == 'E' ) ) + { + in.pos++; + if ( in.pos < in.size && ( in.text[in.pos] == '-' || in.text[in.pos] == '+' ) ) { in.pos++; } + if ( in.pos >= in.size || in.text[in.pos] < '0' || in.text[in.pos] > '9' ) { return false; } + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + whole = false; + } + *integral = whole; + return true; +} + +// the same production, with the token kept for conversion +inline bool TableJsonScanNumber( TableJsonIn & in, char * token, int32_t capacity, int32_t * length, bool * integral ) +{ + TableJsonSpace( in ); + int64_t start = in.pos; + if ( !TableJsonWalkNumber( in, integral ) ) { return false; } + int64_t count = in.pos - start; + if ( count <= 0 || count >= capacity ) { return false; } + memcpy( token, in.text + start, (size_t) count ); + token[count] = 0; + *length = (int32_t) count; + return true; +} + +// the token's exact double, through the runtime's own converter — which +// speaks the LOCALE's decimal point, so the token crosses back over that +// character on its way in +inline double TableJsonTokenDouble( const char * token, int32_t length, bool single ) +{ + char work[kTableJsonMaxNumber]; + memcpy( work, token, (size_t) length ); + work[length] = 0; + char point = TableJsonDecimalPoint(); + if ( point != '.' ) + { + for ( int32_t i = 0; i < length; i++ ) + { + if ( work[i] == '.' ) { work[i] = point; } + } + } + if ( single ) { return (double) strtof( work, NULL ); } + return strtod( work, NULL ); +} + +// ---- ONE CHECKED NUMERIC INTERPRETATION (docs/SPEC-TABLES.md §16.2) ---- +// +// JSON HAS ONE NUMBER TYPE, so every integer target reads a token the same way +// and the VALUE decides rather than the spelling: 2, 2.0 and 1e3 are the +// integers 2, 2 and 1000. What comes out of a token is a SIGN, a MAGNITUDE and +// a STATUS, and nothing on the way is cast through a type that cannot hold what +// it is handed. A uint64 magnitude past INT64_MAX is a magnitude and never a +// negative, and a double is consulted only for a spelling the digit path cannot +// read exactly. +// +// TWO POLICIES SIT ON TOP OF THE ONE VALUE and neither reinterprets the token: +// an ordinary FIELD clamps to its domain and counts, and a MAP KEY rejects the +// whole entry, because a key is an identity and a clamped one is two entries +// merged. That difference is the only difference between them. +// +// THE KEY READS ITS TOKEN EXACTLY AND A FIELD READS IT THROUGH THE DOUBLE, and +// that is the one place the two interpretations part. A key is an IDENTITY, so +// two spellings a 53-bit mantissa cannot tell apart are two keys and the key +// path carries TableJsonInterpretExact below. A field's value is a quantity +// under a clamp, and its interpretation is the one the C, Go and Rust ports +// read the same texts with, so it lives here and reads as they read. +struct TableJsonInteger +{ + uint64_t magnitude; // |value|, exact for every integral token 64 bits hold + bool negative; + bool fractional; // a genuinely fractional VALUE: the wrong shape for an integer + bool saturated; // a magnitude past what 64 bits hold, held at that edge + bool finite; // false: no integer target holds it at all +}; + +// THE FIELD'S INTERPRETATION: the token, parsed digit by digit so no width and +// no locale can move it, and through the runtime's converter only where the +// spelling carries a fraction or an exponent +inline TableJsonInteger TableJsonInterpret( const char * token, int32_t length, bool integral ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = false; + out.fractional = false; + out.saturated = false; + out.finite = true; + if ( integral ) + { + int32_t i = 0; + if ( i < length && token[i] == '-' ) // WalkNumber refuses a leading plus + { + out.negative = true; + i++; + } + for ( ; i < length; i++ ) + { + const uint64_t digit = (uint64_t) ( token[i] - '0' ); + if ( out.magnitude > ( UINT64_MAX - digit ) / 10 ) + { + out.magnitude = UINT64_MAX; + out.saturated = true; + break; + } + out.magnitude = out.magnitude * 10 + digit; + } + if ( out.magnitude == 0 ) { out.negative = false; } // -0 IS zero + return out; + } + const double d = TableJsonTokenDouble( token, length, false ); + if ( !TableJsonFinite( d ) ) { out.finite = false; return out; } + out.negative = d < 0; + const double whole = out.negative ? -d : d; + // THE DOMAIN IS ESTABLISHED BEFORE THE CAST: a magnitude past what sixty-four + // bits hold is answered here, so no value ever reaches a conversion that is + // undefined for it + if ( whole >= 18446744073709551616.0 ) + { + out.magnitude = UINT64_MAX; + out.saturated = true; + return out; + } + const uint64_t truncated = (uint64_t) whole; + if ( (double) truncated != whole ) { out.fractional = true; return out; } + out.magnitude = truncated; + if ( out.magnitude == 0 ) { out.negative = false; } + return out; +} + +// THE DECIMAL BAND a token is answered in without arithmetic: 10^20 is above +// UINT64_MAX whatever the digits are, so a point past it saturates and a token +// spelling 1e999999999 costs nothing to refuse. +const int64_t kTableJsonDecimalBand = 20; + +// THE MAP KEY'S INTERPRETATION, and no other path's: the token's own digits, +// read where they stand, so no 53-bit mantissa decides the identity of a 64-bit +// key. The int and frac runs are one digit string with the point after "point" +// of them, and the exponent moves the point rather than the digits, which is +// the normalization the wide kinds already use over one 64-bit lane. A zero +// fraction is the integer the token spells at every magnitude the kind holds, +// so 9007199254740993.0 is that key rather than the one a double rounds it to. +// No exact reader has an infinity, so finite is true here always. +inline TableJsonInteger TableJsonInterpretExact( const char * token, int32_t length ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = false; + out.fractional = false; + out.saturated = false; + out.finite = true; + int32_t i = 0; + if ( i < length && token[i] == '-' ) { out.negative = true; i++; } // WalkNumber refuses a leading plus + const char * int_digits = token + i; + int32_t int_len = 0; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { int_len++; i++; } + const char * frac_digits = token + i; + int32_t frac_len = 0; + if ( i < length && token[i] == '.' ) + { + i++; + frac_digits = token + i; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { frac_len++; i++; } + } + int64_t exp = 0; + if ( i < length && ( token[i] == 'e' || token[i] == 'E' ) ) + { + i++; + bool exp_negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { exp_negative = token[i] == '-'; i++; } + while ( i < length && token[i] >= '0' && token[i] <= '9' ) + { + if ( exp < 100000 ) { exp = exp * 10 + ( token[i] - '0' ); } + i++; + } + if ( exp_negative ) { exp = -exp; } + } + // leading and trailing zeros stripped, so the last digit kept is significant + int32_t start = 0, end = int_len + frac_len; + int64_t point = (int64_t) int_len + exp; + while ( start < end && ( start < int_len ? int_digits[start] : frac_digits[start - int_len] ) == '0' ) { start++; point--; } + while ( end > start && ( end - 1 < int_len ? int_digits[end - 1] : frac_digits[end - 1 - int_len] ) == '0' ) { end--; } + const int64_t digits = end - start; + if ( digits == 0 ) { out.negative = false; return out; } // the value is zero, and -0 IS zero + if ( point < digits ) { out.fractional = true; return out; } // a significant digit below the point + if ( point > kTableJsonDecimalBand ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + for ( int32_t k = start; k < end; k++ ) + { + const uint64_t digit = (uint64_t) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ); + if ( out.magnitude > ( UINT64_MAX - digit ) / 10 ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + out.magnitude = out.magnitude * 10 + digit; + } + for ( int64_t k = digits; k < point; k++ ) // the point's own zeros, which no digit spells + { + if ( out.magnitude > UINT64_MAX / 10 ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + out.magnitude *= 10; + } + return out; +} + +// a declared range bound as the same value. A bound is inside the field's own +// domain by construction, so nothing here saturates. +inline TableJsonInteger TableJsonIntegerOf( double bound ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = bound < 0; + out.fractional = false; + out.saturated = false; + out.finite = true; + const double whole = out.negative ? -bound : bound; + out.magnitude = whole >= 18446744073709551616.0 ? UINT64_MAX : (uint64_t) whole; + if ( out.magnitude == 0 ) { out.negative = false; } + return out; +} + +// THE TARGET DOMAIN, established before the value reaches storage: the bytes of +// storage, signed or not. Answers what the target holds and whether the domain +// MOVED it. An unsigned magnitude above INT64_MAX rides out as its bit pattern, +// which is the storage's own image of it and not a negative number. +inline int64_t TableJsonIntegerInDomain( const TableJsonInteger & number, bool is_signed, int32_t bytes, bool & moved ) +{ + moved = false; + uint64_t magnitude = number.magnitude; + if ( is_signed ) + { + const uint64_t high = bytes >= 8 ? (uint64_t) INT64_MAX : ( ( uint64_t( 1 ) << ( bytes * 8 - 1 ) ) - 1 ); + if ( number.negative ) + { + const uint64_t low = high + 1; // the floor's magnitude + if ( magnitude > low ) { magnitude = low; moved = true; } + return (int64_t) ( ~magnitude + 1 ); // two's complement, INT64_MIN included + } + if ( magnitude > high ) { magnitude = high; moved = true; } + return (int64_t) magnitude; + } + // A NEGATIVE TOKEN IN AN UNSIGNED FIELD CLAMPS TO ZERO, and -0 is zero, + // which is why the sign is dropped at a zero magnitude above + if ( number.negative ) { moved = true; return 0; } + const uint64_t high = bytes >= 8 ? UINT64_MAX : ( ( uint64_t( 1 ) << ( bytes * 8 ) ) - 1 ); + if ( magnitude > high ) { magnitude = high; moved = true; } + return (int64_t) magnitude; +} + +// A number token into a wide kind's raw storage (docs/SPEC-TABLES.md §16.2). A +// 128-bit integer takes any token whose VALUE is integral; a fixed field any +// token whose value is EXACTLY representable in its Q I.F — a finer fraction +// is the wrong shape for the field, counted as a kind mismatch and never +// rounded, the rule SPEC.md §4.6 gives a fixed default. A magnitude past 128 +// bits saturates and counts as a clamp, as an int64 field saturates at +// INT64_MAX; the declared range clamps after it, on the RAW scale, as it does +// for every bounded scalar. +// +// The token is normalized to its digits with the decimal point after "point" +// of them. An integer part past 40 digits is above 2^128 whatever the digits +// are, and a value below 10^-40 is finer than 2^-127, the finest fraction any +// F can spell — so outside that band the answer is known without the +// arithmetic, and a token spelling 1e999999999 costs nothing to refuse. +inline bool TableJsonReadWide( TableJsonIn & in, const char * token, int32_t length, void * storage, const TableFieldInfo * f ) +{ + bool is_signed = TableJsonKindWideSigned( f->kind ); + int frac = f->frac_bits; + int32_t i = 0; + bool negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { negative = token[i] == '-'; i++; } + const char * int_digits = token + i; + int32_t int_len = 0; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { int_len++; i++; } + const char * frac_digits = token + i; + int32_t frac_len = 0; + if ( i < length && token[i] == '.' ) + { + i++; + frac_digits = token + i; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { frac_len++; i++; } + } + int64_t exp = 0; + if ( i < length && ( token[i] == 'e' || token[i] == 'E' ) ) + { + i++; + bool exp_negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { exp_negative = token[i] == '-'; i++; } + while ( i < length && token[i] >= '0' && token[i] <= '9' ) + { + if ( exp < 100000 ) { exp = exp * 10 + ( token[i] - '0' ); } + i++; + } + if ( exp_negative ) { exp = -exp; } + } + // the digits, with the point after "point" of them; leading and trailing + // zeros stripped. digit( k ) reads the k-th of the int and frac runs. + int32_t start = 0, end = int_len + frac_len; + int64_t point = int_len + exp; + while ( start < end && ( start < int_len ? int_digits[start] : frac_digits[start - int_len] ) == '0' ) { start++; point--; } + while ( end > start && ( end - 1 < int_len ? int_digits[end - 1] : frac_digits[end - 1 - int_len] ) == '0' ) { end--; } + + TableJsonWide raw = { 0, 0 }; + bool saturated = false; + TableJsonWide signed_max = { ~uint64_t( 0 ), ~uint64_t( 0 ) >> 1 }; + TableJsonWide signed_min = { 0, uint64_t( 1 ) << 63 }; + TableJsonWide unsigned_max = { ~uint64_t( 0 ), ~uint64_t( 0 ) }; + if ( start == end ) + { + // zero, and -0 IS zero + } + else if ( point > 40 ) + { + saturated = true; + if ( !negative ) { raw = is_signed ? signed_max : unsigned_max; } + else if ( is_signed ) { raw = signed_min; } + } + else if ( point < -40 ) + { + in.report->kind_mismatch++; // finer than any F can spell + return true; + } + else + { + // the fraction FIRST, so an inexact value is the wrong shape whatever + // its magnitude: its digits, with the zeros a negative point puts in + // front, doubled F times; each doubling's carry is the next bit, and + // the value is exact iff nothing is left after the last one + char fd[kTableJsonMaxNumber + 48]; + int32_t fn = 0; + for ( int64_t z = point; z < 0; z++ ) { fd[fn++] = 0; } + for ( int32_t k = (int32_t) ( point > 0 ? point : 0 ) + start; k < end; k++ ) + { + fd[fn++] = (char) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ); + } + TableJsonWide fraction = { 0, 0 }; + for ( int b = 0; b < frac; b++ ) + { + int carry = 0; + for ( int32_t k = fn - 1; k >= 0; k-- ) + { + int d = fd[k] * 2 + carry; + fd[k] = (char) ( d % 10 ); + carry = d / 10; + } + fraction = TableJsonWideShl( fraction, 1 ); + fraction.lo |= (uint64_t) carry; + } + for ( int32_t k = 0; k < fn; k++ ) + { + if ( fd[k] != 0 ) + { + in.report->kind_mismatch++; + return true; + } + } + // then the whole part, saturating past 128 bits + TableJsonWide whole = { 0, 0 }; + for ( int64_t k = start; k < start + point && !saturated; k++ ) + { + uint32_t digit = k < end ? (uint32_t) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ) : 0; + if ( TableJsonWideMulAdd( &whole, 10, digit ) != 0 ) { saturated = true; } + } + if ( !saturated && frac > 0 && !TableJsonWideZero( TableJsonWideShr( whole, 128 - frac ) ) ) { saturated = true; } + if ( !saturated ) + { + raw = TableJsonWideShl( whole, frac ); + raw.lo |= fraction.lo; + raw.hi |= fraction.hi; + } + if ( is_signed ) + { + if ( !saturated && !negative && TableJsonWideNegative( raw ) ) { saturated = true; } + if ( !saturated && negative && TableJsonWideCompare( raw, signed_min, false ) > 0 ) { saturated = true; } + if ( saturated ) { raw = negative ? signed_min : signed_max; } + else if ( negative ) { raw = TableJsonWideNeg( raw ); } + } + else + { + if ( saturated ) { raw = unsigned_max; } + if ( negative && !TableJsonWideZero( raw ) ) { raw.lo = 0; raw.hi = 0; saturated = true; } + } + } + if ( saturated ) { in.report->clamped++; } + if ( f->wide != NULL ) + { + TableJsonWide lo = { f->wide->lo[0], f->wide->lo[1] }; + TableJsonWide hi = { f->wide->hi[0], f->wide->hi[1] }; + if ( TableJsonWideCompare( raw, lo, is_signed ) < 0 ) { raw = lo; in.report->clamped++; } + else if ( TableJsonWideCompare( raw, hi, is_signed ) > 0 ) { raw = hi; in.report->clamped++; } + } + TableJsonWideStore( storage, f->elem_size, raw ); + return true; +} + +inline bool TableJsonSkipValue( TableJsonIn & in, int32_t depth ); + +inline bool TableJsonSkipContainer( TableJsonIn & in, char close, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; // the opening bracket + bool first = true; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == close ) { in.pos++; return true; } + if ( c == 0 ) { in.bad = true; return false; } + if ( close == '}' ) + { + // the key is kept, because a skipped OBJECT may still be a + // pointer's: an `&node` opening it names a node the storage could + // not hold, and the numbering has to survive the drop (§16.7). + // Anywhere but first, the prefix is the reserved key out of place + // — in a pointered unit; a fixed unit skips the value whole. + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + if ( key[0] == '&' && in.graph != NULL ) + { + if ( !first ) { in.report->malformed = true; in.bad = true; return false; } + if ( !TableJsonSkippedAmpersand( in, key, depth ) ) { return false; } + first = false; + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == close ) { in.pos++; return true; } + in.bad = true; + return false; + } + } + first = false; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == close ) { in.pos++; return true; } + in.bad = true; + return false; + } +} + +inline bool TableJsonSkipValue( TableJsonIn & in, int32_t depth ) +{ + char c = TableJsonPeek( in ); + switch ( c ) + { + case '{': return TableJsonSkipContainer( in, '}', depth ); + case '[': return TableJsonSkipContainer( in, ']', depth ); + case '"': return TableJsonScanString( in, NULL, 0, NULL ); + case 't': return TableJsonLiteral( in, "true" ); + case 'f': return TableJsonLiteral( in, "false" ); + case 'n': return TableJsonLiteral( in, "null" ); + case 0: in.bad = true; return false; + default: + { + // consumed, never converted: skipping needs no buffer, and this + // is the one walk a hostile text drives to the depth cap. It is + // the SAME production the value path scans, so an unknown key + // cannot smuggle past a number a named key would refuse. + bool integral = false; + if ( !TableJsonWalkNumber( in, &integral ) ) { in.bad = true; return false; } + return true; + } + } +} + +inline bool TableJsonReadTable( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth ); +// a UNION ARM that names no declaration reads through the field walk one key +// down (docs/SPEC-TABLES.md §2.6, §16.2), which is defined below +inline bool TableJsonReadField( TableJsonIn & in, void * base, const TableFieldInfo * f, int32_t depth ); + +// place one scalar at one storage address +inline bool TableJsonReadScalar( TableJsonIn & in, void * storage, const TableFieldInfo * f, int32_t depth ) +{ + if ( f->arms != NULL ) + { + // a union is an object with ONE key, the arm's name; {} is None, and + // two keys is a text this walk will not guess at + const TableUnionInfo * arms = f->arms(); + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + TableJsonSetRaw( (uint8_t *) storage + arms->tag_offset, arms->tag_size, 0 ); + if ( TableJsonPeek( in ) == '}' ) { in.pos++; return true; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t tag = 0; + for ( int64_t t = 1; t <= f->enum_max; t++ ) + { + if ( strcmp( f->enum_name( (uint64_t) t ), key ) == 0 ) { tag = t; break; } + } + if ( tag == 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + void * payload = (uint8_t *) storage + arms->arms[tag].offset; + const TableFieldInfo * arm = arms->arms[tag].field; + bool placed = true; + if ( arm != NULL ) + { + // THE ARM'S VALUE TAKES THE ARM'S OWN ROW (§16.2). A value of + // the wrong shape for that row is a KIND MISMATCH: the union + // reads None, the event is counted, and the enclosing object + // continues — the rule a FIELD's value lives under, one key + // down. A pointer arm's null is a null pointer, not a shape + // error, exactly as a pointer field's is (§16.7). + char got = TableJsonValueShape( in ); + if ( arm->kind == 17 && !arm->is_array && got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + memset( payload, 0, (size_t) arms->arms[tag].size ); + } + else if ( got != TableJsonShape( arm ) ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else if ( arm->kind == 17 && !arm->is_array ) + { + // A POINTER ARM'S VALUE IS THE POINTEE IN PLACE, or a + // node reference to one (§16.7) — the read a pointer + // FIELD takes, which is not the scalar walk + memset( payload, 0, (size_t) arms->arms[tag].size ); + if ( !TableJsonReadPointer( in, payload, arm, depth + 1 ) ) { return false; } + } + else + { + // SELECTION ZERO-ESTABLISHES THE ARM (SPEC §5): an arm + // takes no specified default, so zero is the establish + memset( payload, 0, (size_t) arms->arms[tag].size ); + if ( !TableJsonReadField( in, storage, arm, depth + 1 ) ) { return false; } + } + } + else if ( arms->arms[tag].table != NULL ) + { + if ( TableJsonValueShape( in ) != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else + { + arms->arms[tag].table->reset( payload ); + if ( !TableJsonReadTable( in, payload, arms->arms[tag].table, depth + 1 ) ) { return false; } + } + } + else + { + // A PAYLOAD-FREE ARM'S VALUE IS null (§2.6): the arm name + // selects it and there is nothing to place + if ( TableJsonValueShape( in ) != 'z' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else if ( !TableJsonLiteral( in, "null" ) ) + { + return false; + } + } + if ( placed ) + { + TableJsonSetRaw( (uint8_t *) storage + arms->tag_offset, arms->tag_size, (uint64_t) tag ); + } + } + char c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; c = TableJsonPeek( in ); } + if ( c == '}' ) { in.pos++; return true; } + in.bad = true; // a second key: a one-of with two arms is not a value + return false; + } + if ( f->kind == 13 ) + { + f->table->reset( storage ); + return TableJsonReadTable( in, storage, f->table, depth + 1 ); + } + if ( TableJsonIsEnum( f ) ) + { + char name[kTableJsonMaxKey]; + int32_t name_length = 0; + if ( !TableJsonScanString( in, name, kTableJsonMaxKey - 1, &name_length ) ) { return false; } + name[name_length] = 0; + for ( int64_t v = 0; v <= f->enum_max; v++ ) + { + if ( strcmp( f->enum_name( (uint64_t) v ), name ) == 0 ) + { + TableJsonSetRaw( storage, f->elem_size, (uint64_t) v ); + return true; + } + } + // a name this build cannot name reads as None and counts as unknown, + // exactly as an unknown variant id does on the wire (§4) + TableJsonSetRaw( storage, f->elem_size, 0 ); + in.report->unknown++; + return true; + } + if ( TableJsonIsFlags( f ) ) + { + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + in.pos++; + uint64_t bits = 0; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + if ( c != '"' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + char name[kTableJsonMaxKey]; + int32_t name_length = 0; + if ( !TableJsonScanString( in, name, kTableJsonMaxKey - 1, &name_length ) ) { return false; } + name[name_length] = 0; + bool found = false; + for ( int64_t bit = 0; bit <= f->enum_max; bit++ ) + { + if ( strcmp( f->enum_name( (uint64_t) bit ), name ) == 0 ) + { + bits |= uint64_t( 1 ) << bit; + found = true; + break; + } + } + if ( !found ) { in.report->unknown++; } + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + TableJsonSetRaw( storage, f->elem_size, bits ); + return true; + } + if ( f->kind == 1 ) + { + char c = TableJsonPeek( in ); + if ( c == 't' ) { if ( !TableJsonLiteral( in, "true" ) ) { return false; } TableJsonSetRaw( storage, f->elem_size, 1 ); return true; } + if ( !TableJsonLiteral( in, "false" ) ) { return false; } + TableJsonSetRaw( storage, f->elem_size, 0 ); + return true; + } + char token[kTableJsonMaxNumber]; + int32_t length = 0; + bool integral = false; + if ( !TableJsonScanNumber( in, token, kTableJsonMaxNumber, &length, &integral ) ) + { + in.bad = true; + return false; + } + if ( TableJsonKindWide( f->kind ) ) + { + return TableJsonReadWide( in, token, length, storage, f ); + } + if ( f->kind == 10 || f->kind == 11 ) + { + bool single = f->kind == 10; + double value = TableJsonTokenDouble( token, length, single ); + // A magnitude the field's format cannot hold is the WRONG SHAPE for + // the kind, and it never reaches storage: 1e400 is not a float64 and + // 1e300 is not a float32. Storing the infinity the conversion + // produced would leave an instance this walk called CLEAN that + // ToJsonMeasure then refuses forever (a non-finite float has no JSON + // spelling), and §16.1's one invariant is that a text which reads + // clean writes back. + if ( !TableJsonFinite( value ) ) + { + in.report->kind_mismatch++; + return true; + } + if ( f->has_range ) + { + if ( value < f->range_min ) { value = f->range_min; in.report->clamped++; } + else if ( value > f->range_max ) { value = f->range_max; in.report->clamped++; } + } + if ( single ) + { + float narrow = (float) value; + if ( !TableJsonFinite( (double) narrow ) ) + { + in.report->kind_mismatch++; + return true; + } + memcpy( storage, &narrow, sizeof( narrow ) ); + } + else + { + memcpy( storage, &value, sizeof( value ) ); + } + return true; + } + // AN ORDINARY FIELD'S POLICY over the one interpreted value: it CLAMPS to + // its domain and counts. JSON has one number type, so 2.0 IS the integer 2 + // and 1e3 IS 1000. A library that round-trips numbers through a double + // emits them that way, and this walker's own float writer emits 1e+21. Only + // a genuinely fractional value is the wrong shape for the kind. + const bool is_signed = f->kind >= 2 && f->kind <= 5; + const TableJsonInteger number = TableJsonInterpret( token, length, integral ); + if ( !number.finite || number.fractional ) + { + in.report->kind_mismatch++; + return true; + } + if ( number.saturated ) { in.report->clamped++; } // past what sixty-four bits hold + // THE DECLARED RANGE FIRST, THEN THE STORAGE WIDTH, the wire's order (§4), + // so a text and a wire loaded from the same data land the same instance. + // The comparison is on the value's OWN scale, correctly signed past + // INT64_MAX, where the storage's bit pattern is not a number to compare. + TableJsonInteger bounded = number; + if ( f->has_range ) + { + const double scale = number.negative ? -(double) number.magnitude : (double) number.magnitude; + if ( scale < f->range_min ) { bounded = TableJsonIntegerOf( f->range_min ); in.report->clamped++; } + else if ( scale > f->range_max ) { bounded = TableJsonIntegerOf( f->range_max ); in.report->clamped++; } + } + bool moved = false; + const int64_t value = TableJsonIntegerInDomain( bounded, is_signed, (int32_t) f->elem_size, moved ); + if ( moved ) { in.report->clamped++; } + TableJsonSetRaw( storage, f->elem_size, (uint64_t) value ); + return true; +} + +inline bool TableJsonReadField( TableJsonIn & in, void * base, const TableFieldInfo * f, int32_t depth ) +{ + uint8_t * storage = (uint8_t *) base + f->offset; + if ( TableJsonIsMap( f ) ) + { + return TableJsonReadMap( in, (void *) storage, f, depth ); + } + if ( TableJsonIsList( f ) ) + { + return TableJsonReadList( in, (void *) storage, f, depth ); + } + + if ( f->kind == 12 ) + { + int32_t length = 0; + if ( !TableJsonScanString( in, (char *) storage, f->array_bound, &length ) ) { return false; } + storage[length] = 0; + TableJsonSetCount( base, f, length ); + return true; + } + if ( f->kind == 33 ) + { + char16_t * units = (char16_t *) (void *) storage; + int32_t length = 0; + if ( !TableJsonScanWString( in, units, (int32_t) f->array_bound, &length ) ) { return false; } + units[length] = 0; // the terminating zero UNIT at index length (§7.2, SPEC.md §4.12) + TableJsonSetCount( base, f, length ); + return true; + } + if ( TableJsonIsBytes( f ) ) + { + // base64 decodes STRAIGHT INTO the field's storage, six bits at a + // time — no window, no temporary, so a bytes(N) of any declared + // extent reads the same way. A base64 body carries no escapes, so a + // backslash in one is simply not an alphabet character. + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + memset( storage, 0, (size_t) f->array_bound ); + TableJsonSetCount( base, f, 0 ); + const char * alphabet = TableJsonBase64Alphabet(); + int32_t placed = 0; + uint32_t accumulator = 0; + int32_t held = 0; + bool clamped = false; + bool malformed = false; + for ( ;; ) + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos++]; + if ( c == '"' ) { break; } + if ( c == '=' || malformed ) { continue; } + const char * at = c != 0 ? strchr( alphabet, c ) : NULL; + if ( at == NULL ) { malformed = true; continue; } + accumulator = ( accumulator << 6 ) | (uint32_t) ( at - alphabet ); + held += 6; + if ( held >= 8 ) + { + held -= 8; + if ( placed < f->array_bound ) + { + storage[placed++] = (uint8_t) ( ( accumulator >> held ) & 0xff ); + } + else + { + clamped = true; + } + } + } + if ( malformed ) + { + // a body that is not base64 is the wrong shape for the kind: the + // field keeps its default and the event is counted + in.report->kind_mismatch++; + return true; + } + if ( clamped ) { in.report->clamped++; } + TableJsonSetCount( base, f, placed ); + return true; + } + if ( TableJsonIsKeyed( f ) ) + { + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + // every slot back to its declared defaults first, so a key the text + // omits keeps them and a repeated field key cannot leave an earlier + // occurrence's slots standing + for ( int32_t i = 0; i < f->array_bound; i++ ) + { + void * slot = storage + (int64_t) i * f->elem_size; + if ( f->kind == 13 ) { f->table->reset( slot ); } + else { memset( slot, 0, (size_t) f->elem_size ); } + } + char shape = TableJsonElementShape( f ); + // A KEYED OBJECT'S KEYS ARE KEYS: a variant named twice is a duplicate + // key like any other, last-wins and counted (§16.2). Tracked the way + // a table's own field keys are — a bounded, allocation-free bitmask; + // a vocabulary wider than this still reads, its repeats simply stop + // being counted. + uint64_t seen[8] = {}; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t slot = -1; + for ( int64_t v = 0; v < f->array_bound; v++ ) + { + // nothing is stored for None, so "None" finds no slot and is + // an unknown key like any other name this reader cannot place + if ( !TableJsonKeyedSlotValid( f, v ) ) { continue; } + if ( strcmp( f->key_name( TableJsonKeyedSlotKey( v ) ), key ) == 0 ) { slot = v; break; } + } + if ( slot >= 0 && slot < 512 ) + { + uint64_t bit = uint64_t( 1 ) << ( slot & 63 ); + if ( ( seen[slot >> 6] & bit ) != 0 ) { in.report->duplicate++; } + seen[slot >> 6] |= bit; + } + if ( slot < 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( TableJsonValueShape( in ) != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadScalar( in, storage + slot * f->elem_size, f, depth + 1 ) ) + { + return false; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; + } + if ( f->is_array ) + { + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + in.pos++; + // LAST WINS has to be true of a repeated ARRAY key too, and it is + // wire-visible: a fixed array writes every slot, so a second, shorter + // occurrence overlaying a prefix would leave the first occurrence's + // tail standing. The field goes back to its declared defaults before + // this occurrence's elements are placed — the re-establishment a nested + // table and a union arm already get. A table element's defaults are + // its own (the reset hook); every other element kind's storage + // default is zero, which is what the generated array declares. + if ( f->kind == 13 ) + { + for ( int32_t i = 0; i < f->array_bound; i++ ) + { + f->table->reset( storage + (int64_t) i * f->elem_size ); + } + } + else + { + memset( storage, 0, (size_t) f->array_bound * (size_t) f->elem_size ); + } + TableJsonSetCount( base, f, 0 ); + int32_t placed = 0; + char shape = TableJsonElementShape( f ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + if ( placed >= f->array_bound ) + { + // more elements than the reader's bound: the bounded prefix + // is kept and the excess counts, the wire's rule (§4) + in.report->clamped++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( f->kind == 17 ) + { + // an element of an ARRAY OF POINTERS (§2.1): null is a null slot, an + // object is the pointee in place or an `&node` reference (§16.7) + char got = TableJsonValueShape( in ); + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( storage + (int64_t) placed * f->elem_size, f->elem_size, 0 ); + } + else if ( got != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, storage + (int64_t) placed * f->elem_size, f, depth + 1 ) ) { return false; } + placed++; + } + else if ( TableJsonValueShape( in ) != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed++; + } + else + { + if ( !TableJsonReadScalar( in, storage + (int64_t) placed * f->elem_size, f, depth + 1 ) ) { return false; } + placed++; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + // a fixed array's tail keeps the defaults the prefill left there, + // exactly as a short wire count does + TableJsonSetCount( base, f, placed ); + return true; + } + return TableJsonReadScalar( in, storage, f, depth ); +} + +inline bool TableJsonReadTableKeys( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth, const char * first_key ); + +// ONE table object: keys are field keys, unknown ones are skipped and +// counted, a repeated key is last-wins and counted. The instance is already +// at its declared defaults when this is entered, so a key the text never +// mentions keeps the default an absent field takes on the wire (§4). +inline bool TableJsonReadTable( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + return TableJsonReadTableKeys( in, base, info, depth, NULL ); +} + +// The keys of an object whose brace is already consumed. A pointer's object +// opens the same way a table's does, but its FIRST key may be `&node` (§16.7) +// and the adapter that reads it has to scan the key to know — so it hands the +// key it scanned in as `first_key`, with the colon consumed, and this places +// it before scanning the rest. +inline bool TableJsonReadTableKeys( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth, const char * first_key ) +{ + // duplicate tracking, bounded and allocation-free: a table with more + // fields than this still reads, its repeats simply stop being counted + uint64_t seen[8] = {}; + for ( ;; ) + { + char key[kTableJsonMaxKey]; + char c = 0; + if ( first_key != NULL ) + { + memcpy( key, first_key, strlen( first_key ) + 1 ); // scanned into a buffer this size by the caller + first_key = NULL; + } + else + { + c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; return true; } + if ( c == 0 ) { in.bad = true; return false; } + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + } + int32_t index = -1; + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + if ( strcmp( info->fields[i].json, key ) == 0 ) { index = i; break; } + } + if ( key[0] == '&' ) + { + // THE AMPERSAND PREFIX IS RESERVED TO THE FORM (docs/SPEC-TABLES.md + // §16.7). No declaration may take a key beginning with it, so this + // is never a field this build lacks — it is the sharing construct + // somewhere it cannot stand: `&node` is the FIRST key of a pointer's + // object and nothing else, and the adapter that reads a pointer + // has consumed it before these keys are read. MALFORMED, refused + // and counted; never counted as unknown, never skipped. + in.report->malformed = true; + in.bad = true; + return false; + } + if ( index < 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + const TableFieldInfo * f = &info->fields[index]; + if ( index < 512 ) + { + uint64_t bit = uint64_t( 1 ) << ( index & 63 ); + if ( ( seen[index >> 6] & bit ) != 0 ) { in.report->duplicate++; } + seen[index >> 6] |= bit; + } + // PRESENCE OF THE KEY IS THE PRESENCE (§16.2): reaching this line + // is the key being present, so an optional is set present + // whatever its value — with one exception the page names: a JSON + // null, which reads as ABSENT rather than as a value. + char got = TableJsonValueShape( in ); + if ( f->kind == 17 && !f->is_array ) + { + // a pointer: null is a null pointer, an object is the pointee + // in place or an `&node` reference to one (§16.7), a string is + // a BYTE BUFFER's bytes (§2.5), and anything else is the wrong + // shape for the kind + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) base + f->offset, f->elem_size, 0 ); + } + else if ( got != TableJsonShape( f ) ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, (uint8_t *) base + f->offset, f, depth ) ) + { + return false; + } + } + else if ( f->optional && got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + // absent, and back at its defaults: a repeated key whose last + // occurrence is null must not leave an earlier value standing + if ( f->table != NULL ) { f->table->reset( (uint8_t *) base + f->offset ); } + else { memset( (uint8_t *) base + f->offset, 0, (size_t) f->elem_size ); } + TableJsonSetRaw( (uint8_t *) base + f->present_offset, 1, 0 ); + } + else + { + if ( got != TableJsonShape( f ) ) + { + // the wrong JSON type for the kind: skipped, never coerced + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadField( in, base, f, depth ) ) + { + return false; + } + if ( f->optional ) + { + TableJsonSetRaw( (uint8_t *) base + f->present_offset, 1, 1 ); + } + } + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; return true; } + in.bad = true; + return false; + } +} + +// ---- the two entry points the per-table wrappers name ---- + +inline bool TableJsonRead( void * value, const TableTypeInfo * info, const char * text, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableJsonIn in; + in.text = text; + in.size = bytes; + in.pos = 0; + in.report = report != NULL ? report : &ignored; + in.bad = false; + in.graph = NULL; + info->reset( value ); + if ( text == NULL || bytes < 0 ) + { + in.report->malformed = true; + return false; + } + bool ok = TableJsonReadTable( in, value, info, 0 ); + if ( ok ) + { + TableJsonSpace( in ); + if ( in.pos != in.size ) { in.bad = true; } // trailing rubbish is not one text + } + if ( in.bad || !ok ) + { + in.report->malformed = true; + return false; + } + return true; +} + +inline int64_t TableJsonWrite( const void * value, const TableTypeInfo * info, char * buffer, int64_t capacity ) +{ + TableJsonOut out; + out.buffer = buffer; + out.capacity = capacity; + out.offset = 0; + out.overflow = false; + out.graph = NULL; + if ( !TableJsonWriteValue( out, value, info, 0 ) ) { return -1; } + // THE CANONICAL TEXT ENDS WITH EXACTLY ONE NEWLINE (docs/SPEC-TABLES.md + // §16.1). Every writer emits it — this walk, the C# walk and + // "schema unpack" — and every reader accepts a text with or without one, + // because the trailing whitespace a read already skips is what makes the + // two the same text. It is a byte of the FORM rather than a file + // convention: a text that is written to a file, pasted into a diff and + // handed back through a pipe has to be one text in all three places, and a + // buffer whose last byte is a closing brace is the one shape that is not. + out.put( '\n' ); + if ( out.overflow ) { return -1; } + return out.offset; +} + +// ---- json walk: end ---- + +// ---- json graph walk: begin ---- +// +// THE VARIABLE CLASS's half of the text form (docs/SPEC-TABLES.md §16.7). The +// walk above places every kind but one; this defines the three adapters it +// calls for that one, and the two entry points a pointered table's wrappers +// name. The text is the fixed class's — a pointee is an object in place — and a +// node named more than once carries `&node`: defined once, with its fields, +// and referenced after by `{ "&node": N }` alone. + +// ---- the identity map ---- +// +// ONE map shape serves both directions. Writing keys it by a node's ADDRESS and +// counts the slots that name the node, so the second pass knows at a node's +// first occurrence whether it will be named again; reading keys it by the +// text's own label and answers the node it defined. Open addressing, a +// multiply-shift hash and quadrupling growth — TablePackMap's shape (§6.2), on +// the same terms: proportional to nodes, never to bytes, on the authoring +// side, and released before the call returns. + +struct TableJsonGraphEntry +{ + uint64_t key; // a node's address (write) or a label (read); 0 is an empty slot + int64_t count; // write: how many slots name this node + int64_t label; // write: the `&node` label assigned at its first write, 0 until then + uint8_t open; // the descent is still open: a reference here is a cycle (write), a self-reference (read) + uint32_t node; // read: the node's arena offset; 0 for a definition the reader dropped + const TableTypeInfo * type; // read: the node's table; NULL for a dropped one +}; + +struct TableJsonGraphMap +{ + TableJsonGraphEntry * entries; + int64_t capacity; // a power of two, or zero while empty + int64_t count; + TableAllocator allocator; // the caller's pair (§6.5): the builder's on read, the one handed to ToJson on write +}; + +inline void TableJsonGraphMapInit( TableJsonGraphMap & map, TableAllocator allocator ) +{ + map.entries = NULL; + map.capacity = 0; + map.count = 0; + map.allocator = allocator; +} + +inline void TableJsonGraphMapShutdown( TableJsonGraphMap & map ) +{ + map.allocator.free( map.allocator.context, map.entries ); + TableJsonGraphMapInit( map, map.allocator ); +} + +inline int64_t TableJsonGraphMapSlot( const TableJsonGraphMap & map, uint64_t key ) +{ + uint64_t hash = key * 0x9E3779B97F4A7C15ull; + hash ^= hash >> 29; + int64_t mask = map.capacity - 1; + int64_t at = (int64_t) ( hash & (uint64_t) mask ); + while ( map.entries[at].key != 0 && map.entries[at].key != key ) + { + at = ( at + 1 ) & mask; + } + return at; +} + +inline TableJsonGraphEntry * TableJsonGraphMapFind( TableJsonGraphMap & map, uint64_t key ) +{ + if ( map.capacity == 0 ) { return NULL; } + TableJsonGraphEntry * entry = &map.entries[ TableJsonGraphMapSlot( map, key ) ]; + return entry->key == key ? entry : NULL; +} + +inline bool TableJsonGraphMapGrow( TableJsonGraphMap & map ) +{ + TableJsonGraphMap grown; + grown.allocator = map.allocator; + grown.capacity = map.capacity != 0 ? map.capacity * 4 : 64; + grown.count = 0; + grown.entries = (TableJsonGraphEntry *) map.allocator.alloc( map.allocator.context, grown.capacity * (int64_t) sizeof( TableJsonGraphEntry ) ); // zeroed, by the pair's contract + if ( grown.entries == NULL ) { return false; } + for ( int64_t i = 0; i < map.capacity; i++ ) + { + if ( map.entries[i].key == 0 ) { continue; } + grown.entries[ TableJsonGraphMapSlot( grown, map.entries[i].key ) ] = map.entries[i]; + grown.count++; + } + map.allocator.free( map.allocator.context, map.entries ); + map = grown; + return true; +} + +// the entry for a key, made if it was not there; `taken` says which. NULL is the +// allocator refusing, and the walk refuses with it. +inline TableJsonGraphEntry * TableJsonGraphMapReach( TableJsonGraphMap & map, uint64_t key, bool & taken ) +{ + if ( ( map.count + 1 ) * 4 >= map.capacity * 3 ) // keep the load factor under three quarters + { + if ( !TableJsonGraphMapGrow( map ) ) { return NULL; } + } + TableJsonGraphEntry * entry = &map.entries[ TableJsonGraphMapSlot( map, key ) ]; + taken = entry->key != key; + if ( taken ) + { + entry->key = key; + map.count++; + } + return entry; +} + +// ---- reading: into a builder ---- + +struct TableJsonGraphIn +{ + TableWorker * worker; // where every node comes from + TableJsonGraphMap labels; // a label -> the node it defined +}; + +// `&node`'s value, the LABEL: a positive integer spelled as one — digits, no sign, no +// fraction, no exponent, no leading zero (§16.7). Anything else is malformed. +inline bool TableJsonScanLabel( TableJsonIn & in, uint64_t & label ) +{ + TableJsonSpace( in ); + if ( in.pos >= in.size || in.text[in.pos] < '1' || in.text[in.pos] > '9' ) + { + in.report->malformed = true; + in.bad = true; + return false; + } + uint64_t value = 0; + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) + { + uint64_t digit = (uint64_t) ( in.text[in.pos] - '0' ); + if ( value > ( UINT64_MAX - digit ) / 10 ) + { + in.report->malformed = true; + in.bad = true; + return false; + } + value = value * 10 + digit; + in.pos++; + } + label = value; + return true; +} + +// A BYTE BUFFER's text (docs/SPEC-TABLES.md §2.5, §16.2): a string. For a +// *string the string's bytes become the blob; for a *bytes the string is base64 +// and its decoded bytes do. The blob is allocated at EXACTLY the decoded +// length — the string is scanned once without keeping it to learn the length, +// and once into the node — so a blob of any size reads with no window and no +// bound to clamp against. A *bytes body that is not base64 is the wrong shape +// for the kind: the reference stays null and the event is counted. +inline bool TableJsonReadBlob( TableJsonIn & in, void * slot, const TableFieldInfo * f ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + TableRef * ref = (TableRef *) slot; + ref->value = 0; + if ( strcmp( f->type_name, "string" ) == 0 ) + { + const int64_t mark = in.pos; + int32_t length = 0; + if ( !TableJsonScanString( in, NULL, 0, &length ) ) { return false; } + in.pos = mark; + char * data = TableStringEmplace( *graph->worker, *ref, NULL, (int64_t) length ); + if ( data == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + int32_t placed = 0; + return TableJsonScanString( in, data, length, &placed ); + } + // base64: the alphabet characters decide the length, six bits apiece + const char * alphabet = TableJsonBase64Alphabet(); + const int64_t mark = in.pos + 1; + int64_t symbols = 0; + bool malformed = false; + in.pos++; + for ( ;; ) + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos++]; + if ( c == '"' ) { break; } + if ( c == '=' || malformed ) { continue; } + if ( c == 0 || strchr( alphabet, c ) == NULL ) { malformed = true; continue; } + symbols++; + } + if ( malformed ) + { + in.report->kind_mismatch++; + return true; + } + const int64_t length = ( symbols * 6 ) / 8; + uint8_t * data = TableBytesEmplace( *graph->worker, *ref, length ); + if ( data == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + int64_t placed = 0; + uint32_t accumulator = 0; + int32_t held = 0; + for ( int64_t at = mark; ; at++ ) + { + char c = in.text[at]; + if ( c == '"' ) { break; } + const char * symbol = c != '=' ? strchr( alphabet, c ) : NULL; + if ( symbol == NULL ) { continue; } + accumulator = ( accumulator << 6 ) | (uint32_t) ( symbol - alphabet ); + held += 6; + if ( held >= 8 ) + { + held -= 8; + if ( placed < length ) { data[placed++] = (uint8_t) ( ( accumulator >> held ) & 0xff ); } + } + } + return true; +} + +// A pointer's object. Its FIRST key decides what it is: `&node` naming a label not +// yet defined, with fields after it, is a DEFINITION; `&node` naming one already +// defined, alone, is a REFERENCE; any other key is a node named once, its +// object in place. The node comes from the +// builder's arena, and the slot holds its arena offset (§6.3). A pointer whose +// target is a BYTE BUFFER — no table — takes a string instead (§2.5). +inline bool TableJsonReadPointer( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( f->table == NULL ) { return TableJsonReadBlob( in, slot, f ); } + // the pointee nests one level down, exactly as a by-value table does, and + // takes the same cap: a chain nests as deep as it is long (§16.7) + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + char c = TableJsonPeek( in ); + if ( c == '}' ) + { + // an empty object: a node at its defaults, named once + in.pos++; + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + return true; + } + if ( c == 0 ) { in.bad = true; return false; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + if ( strcmp( key, "&node" ) != 0 ) + { + // a node named once: the pointee's object in place, and this key is + // its first field — unless it is the reserved prefix under a spelling + // this form does not have, which ReadTableKeys refuses + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } + return TableJsonReadTableKeys( in, node, f->table, depth + 1, key ); + } + uint64_t label = 0; + if ( !TableJsonScanLabel( in, label ) ) { return false; } + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph->labels, label, taken ); + if ( entry == NULL ) { in.report->malformed = true; in.bad = true; return false; } + // ONE SPELLING, and what follows the label says which half it is: fields + // after a label the text has not defined DEFINE it, and a label alone that + // the text has defined REFERS to it. The other two are malformed — a label + // alone that the text never defined, which would otherwise read as a default + // node under a silent report, and a field after a label already defined, + // which would be a second definition. That is what keeps a typo loud. + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; c = TableJsonPeek( in ); } + bool bare = c == '}'; + if ( bare == taken ) { in.report->malformed = true; in.bad = true; return false; } + if ( bare ) + { + // A REFERENCE. A label is defined when its object CLOSES, so a + // reference met inside its own definition — at any depth of by-value + // nesting — names a node whose descent is still open: the cycle the + // wire refuses (§3.1), refused here where it is written. A definition + // the reader dropped names no node, so the slot stays null with + // nothing more counted — the drop was counted where it happened. A + // node of another table than the slot declares is a kind mismatch, as + // on the wire. + in.pos++; + if ( entry->open != 0 ) { in.report->malformed = true; in.bad = true; return false; } + TableRef ref; + if ( entry->type == NULL ) + { + memcpy( slot, &ref, sizeof( ref ) ); + return true; + } + if ( entry->type != f->table ) + { + memcpy( slot, &ref, sizeof( ref ) ); + in.report->kind_mismatch++; + return true; + } + ref.value = (int64_t) entry->node; + memcpy( slot, &ref, sizeof( ref ) ); + return true; + } + // A DEFINITION: the node is allocated, the label is its, and the keys after + // `&node` are its fields. The entry is OPEN until the object closes, so a + // reference to the label from inside the node's own fields is refused as + // the cycle it is; the node and its table are filled in at the close. + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } + entry->open = 1; + if ( !TableJsonReadTableKeys( in, node, f->table, depth + 1, NULL ) ) { return false; } + entry = TableJsonGraphMapFind( graph->labels, label ); // the map may have grown under the descent + if ( entry == NULL ) { in.report->malformed = true; in.bad = true; return false; } + TableRef ref; + memcpy( &ref, slot, sizeof( ref ) ); + entry->node = (uint32_t) ref.value; + entry->type = f->table; + entry->open = 0; + return true; +} + +// An `&`-prefixed key opening an object the walk is SKIPPING — a value past an +// array's bound, an unknown key's value, a value of the wrong shape. A +// definition in there still takes its label, so the numbering survives whatever +// the storage could not hold (§16.7): the label is registered with no node, and a +// reference to it reads null. Any other prefixed key is the reserved prefix +// out of place. +inline bool TableJsonSkippedAmpersand( TableJsonIn & in, const char * key, int32_t ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL || strcmp( key, "&node" ) != 0 ) { in.report->malformed = true; in.bad = true; return false; } + uint64_t label = 0; + if ( !TableJsonScanLabel( in, label ) ) { return false; } + bool taken = false; + if ( TableJsonGraphMapReach( graph->labels, label, taken ) == NULL ) { in.report->malformed = true; in.bad = true; return false; } + return true; // a fresh entry is node 0, type NULL: a definition with no node +} + +// ---- writing: from a region's const root ---- + +struct TableJsonGraphOut +{ + TableJsonGraphMap nodes; // a node's address -> how many slots name it, and its `&node` once assigned + bool counting; // PASS ONE: count the references, refuse a cycle, emit nothing + int64_t next_label; +}; + +// The node a slot names: null as `null`, a node named once as its object in +// place, and a node named more than once under the construct. Which of the +// last two it is was learned in pass one; pass two spells it. +inline bool TableJsonWritePointer( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphOut * graph = (TableJsonGraphOut *) out.graph; + if ( graph == NULL ) { return false; } + const void * node = f->resolve( slot ); + if ( node == NULL ) + { + out.raw( "null", 4 ); + return true; + } + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph->nodes, (uint64_t) (uintptr_t) node, taken ); + if ( entry == NULL ) { return false; } + if ( f->table == NULL ) + { + // A BYTE BUFFER (§2.5, §16.7): its text is a string, which has no + // first key to carry `&node`, so a blob named from more than one + // slot has no spelling this form can carry and the graph is refused — + // as a shared node with nothing to write is. A blob named once is its + // bytes in place: base64 for a *bytes, the string itself for a *string. + if ( graph->counting ) { entry->count++; return true; } + if ( entry->count > 1 ) { return false; } + const TableBlob * blob = (const TableBlob *) node; + if ( blob->length > (uint32_t) 0x7fffffff ) { return false; } + if ( strcmp( f->type_name, "string" ) == 0 ) { TableJsonWriteString( out, (const char *) ( blob + 1 ), (int32_t) blob->length ); } + else { TableJsonWriteBase64( out, (const uint8_t *) ( blob + 1 ), (int32_t) blob->length ); } + return true; + } + if ( graph->counting ) + { + // PASS ONE: one visit per node, every slot that names it counted, and + // a reference to a node whose descent is still open is a cycle — + // refused here as the wire refuses it (§3.1) + entry->count++; + if ( !taken ) { return entry->open == 0; } + entry->open = 1; + if ( !TableJsonWriteValue( out, node, f->table, depth ) ) { return false; } + entry = TableJsonGraphMapFind( graph->nodes, (uint64_t) (uintptr_t) node ); // the map may have grown under the descent + if ( entry == NULL ) { return false; } + entry->open = 0; + return true; + } + // PASS TWO: a node named once is its object in place; a node named more + // than once is DEFINED at its first occurrence — `&node` first, then its + // fields — and REFERENCED by `&node` alone after that, spelled the same way at + // every site. Labels run from 1 in first-write order and are the text's own, + // so a stray number in a hand-edited text is most often one never defined. + if ( entry->count <= 1 ) + { + return TableJsonWriteValue( out, node, f->table, depth ); + } + if ( depth > kTableJsonMaxDepth ) { return false; } + if ( entry->label != 0 ) + { + out.put( '{' ); + out.line( depth + 1 ); + out.raw( "\"&node\": ", 9 ); + TableJsonWriteUnsigned( out, (uint64_t) entry->label ); + out.line( depth ); + out.put( '}' ); + return true; + } + entry->label = ++graph->next_label; + out.put( '{' ); + out.line( depth + 1 ); + out.raw( "\"&node\": ", 9 ); + TableJsonWriteUnsigned( out, (uint64_t) entry->label ); + bool any = true; + int64_t before = out.offset; + if ( !TableJsonWriteFields( out, node, f->table, depth, any ) ) { return false; } + // a definition carries at least one field, because a label alone is a + // reference: a shared node with nothing to write has no definition this + // form can spell, and the writer refuses it as it refuses any value it + // cannot spell (§16.3) + if ( out.offset == before ) { return false; } + out.line( depth ); + out.put( '}' ); + return true; +} + +// ---- the two entry points a pointered table's wrappers name ---- + +// The text into the builder's root. Every node the text names is allocated in +// the builder's arena through the field's own Emplace; the label map is the +// walk's, released before this returns. The root itself takes no label — nothing +// may name it (§16.7) — so an `&node` at the root is refused like any other key +// of the prefix. +inline bool TableJsonReadGraph( TableWorker & worker, void * root, const TableTypeInfo * info, const char * text, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + if ( worker.arena == NULL ) { if ( report != NULL ) { report->malformed = true; } return false; } + TableJsonGraphIn graph; + graph.worker = &worker; + TableJsonGraphMapInit( graph.labels, worker.arena->allocator ); + TableJsonIn in; + in.text = text; + in.size = bytes; + in.pos = 0; + in.report = report != NULL ? report : &ignored; + in.bad = false; + in.graph = &graph; + info->reset( root ); + if ( text == NULL || bytes < 0 ) + { + in.report->malformed = true; + return false; + } + bool ok = TableJsonReadTable( in, root, info, 0 ); + if ( ok ) + { + TableJsonSpace( in ); + if ( in.pos != in.size ) { in.bad = true; } // trailing rubbish is not one text + } + TableJsonGraphMapShutdown( graph.labels ); + if ( in.bad || !ok ) + { + in.report->malformed = true; + return false; + } + return true; +} + +// The text of a region's const root: measured when the buffer is NULL, written +// when it is not, over one code path. Two passes over one walk — the first +// counts how many slots name each node and refuses a cycle, the second writes +// — so a node's first occurrence knows whether it will be named again. The +// ROOT's entry is open for the whole first pass, so a reference back at it is +// the cycle it is (§3.1), and it takes no label. +inline int64_t TableJsonWriteGraph( const void * root, const TableTypeInfo * info, char * buffer, int64_t capacity, TableAllocator allocator ) +{ + if ( root == NULL ) { return -1; } + TableJsonGraphOut graph; + TableJsonGraphMapInit( graph.nodes, allocator ); + graph.counting = true; + graph.next_label = 0; + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph.nodes, (uint64_t) (uintptr_t) root, taken ); + if ( entry == NULL ) { TableJsonGraphMapShutdown( graph.nodes ); return -1; } + entry->open = 1; + TableJsonOut count; + count.buffer = NULL; + count.capacity = 0; + count.offset = 0; + count.overflow = false; + count.graph = &graph; + bool ok = TableJsonWriteValue( count, root, info, 0 ); + graph.counting = false; + TableJsonOut out; + out.buffer = buffer; + out.capacity = capacity; + out.offset = 0; + out.overflow = false; + out.graph = &graph; + if ( ok ) { ok = TableJsonWriteValue( out, root, info, 0 ); } + TableJsonGraphMapShutdown( graph.nodes ); + if ( !ok ) { return -1; } + out.put( '\n' ); // the canonical text ends with exactly one newline (§16.1) + if ( out.overflow ) { return -1; } + return out.offset; +} + +// ---- json graph walk: end ---- + +// ---- the out-of-line array's slot (docs/SPEC-TABLES.md §8.1) ---- + +inline int32_t TableJsonExtentCount( const void * slot ) +{ + int32_t count = 0; + memcpy( &count, (const uint8_t *) slot + 8, sizeof( count ) ); + return count < 0 ? 0 : count; +} + +inline const uint8_t * TableJsonExtentElements( const void * slot ) +{ + int64_t delta = 0; + memcpy( &delta, slot, sizeof( delta ) ); + return delta != 0 ? (const uint8_t *) slot + delta : NULL; +} + +// ---- json map walk: begin ---- + +inline bool TableJsonIsMap( const TableFieldInfo * f ) +{ + return f->is_array && f->array_bound == 0 && strncmp( f->type_name, "map[", 4 ) == 0; +} + +// the entry's two rows: fields[0] IS the key and fields[1] IS the value, which +// is what makes a user's own table of pairs the same bytes (§2.8) +inline const TableFieldInfo * TableJsonMapKeyField( const TableFieldInfo * f ) { return &f->table->fields[0]; } +inline const TableFieldInfo * TableJsonMapValueField( const TableFieldInfo * f ) { return &f->table->fields[1]; } + +inline bool TableJsonMapKeyIsString( const TableFieldInfo * key ) { return key->kind == 12; } +inline bool TableJsonMapKeySigned( const TableFieldInfo * key ) { return key->kind >= 2 && key->kind <= 5; } + +// AN INTEGER KEY IS THE INTEGER'S DECIMAL SPELLING, QUOTED, because a JSON +// object's keys are strings. Written digit by digit so no locale can move it. +inline void TableJsonWriteMapIntegerKey( TableJsonOut & out, const void * storage, const TableFieldInfo * key ) +{ + uint64_t magnitude = 0; + bool negative = false; + if ( TableJsonMapKeySigned( key ) ) + { + int64_t value = 0; + switch ( key->kind ) + { + case 2: value = (int64_t) *(const int8_t *) storage; break; + case 3: value = (int64_t) *(const int16_t *) storage; break; + case 4: value = (int64_t) *(const int32_t *) storage; break; + default: value = *(const int64_t *) storage; break; + } + negative = value < 0; + magnitude = negative ? ( ~(uint64_t) value ) + 1 : (uint64_t) value; + } + else + { + switch ( key->kind ) + { + case 6: magnitude = (uint64_t) *(const uint8_t *) storage; break; + case 7: magnitude = (uint64_t) *(const uint16_t *) storage; break; + case 8: magnitude = (uint64_t) *(const uint32_t *) storage; break; + default: magnitude = *(const uint64_t *) storage; break; + } + } + char digits[24]; + int32_t at = (int32_t) sizeof( digits ); + do { digits[--at] = (char) ( '0' + ( magnitude % 10 ) ); magnitude /= 10; } while ( magnitude != 0 ); + if ( negative ) { digits[--at] = '-'; } + TableJsonWriteString( out, digits + at, (int32_t) sizeof( digits ) - at ); +} + +inline void TableJsonWriteMapKey( TableJsonOut & out, const void * entry, const TableFieldInfo * key ) +{ + const uint8_t * storage = (const uint8_t *) entry + key->offset; + if ( TableJsonMapKeyIsString( key ) ) + { + // A STRING KEY IS THE STRING (§2.8): every JSON key of a map object is + // a KEY OF THE MAP and none is a field key, so the `&` prefix §16.7 + // reserves for field keys is ordinary data here. + TableJsonWriteString( out, (const char *) storage, *(const int32_t *) ( (const uint8_t *) entry + key->count_offset ) ); + return; + } + TableJsonWriteMapIntegerKey( out, (const void *) storage, key ); +} + +// ToJson WRITES ENTRIES IN ASCENDING KEY ORDER, so unpack then pack is +// byte-stable and a diff of two texts is a diff of two maps (§2.8, §17.2). +// A region holds them in that order already, so this is the array in place. +inline bool TableJsonWriteMap( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + const int32_t count = TableJsonExtentCount( slot ); + if ( count == 0 ) { out.raw( "{}", 2 ); return true; } + const TableFieldInfo * key = TableJsonMapKeyField( f ); + const TableFieldInfo * value = TableJsonMapValueField( f ); + const uint8_t * entries = TableJsonExtentElements( slot ); + out.put( '{' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + const void * entry = (const void *) ( entries + (int64_t) i * f->elem_size ); + TableJsonWriteMapKey( out, entry, key ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteField( out, entry, value, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( '}' ); + return true; +} + +// AN INTEGER KEY IS READ BY §16.2's INTEGER RULE AND BY NOTHING ELSE, so +// "2.0" and "1e3" are the integers 2 and 1000 and "-0" is zero. The token is +// walked as a JSON number over its own bytes; a token that rule calls +// malformed makes the KEY malformed, and a genuinely fractional value, or one +// outside the key kind's range, is kind_mismatch for that entry. +// +// THE KEY IS THE SPELLING AND NOTHING AROUND IT. The number walk steps over +// leading whitespace and comments, which is right BETWEEN tokens and wrong +// INSIDE one: a key is an identity, and a padded spelling that resolved to the +// same integer would be a second name for one entry. So the walk must begin at +// the token's first byte, and a token with anything before the number is not a +// JSON number at all, which is malformed on the terms "1-2" is. +inline bool TableJsonMapKeyValue( const char * token, int32_t length, const TableFieldInfo * key, + int64_t & value, bool & fits ) +{ + fits = false; + TableReport scratch; + TableJsonIn probe = { token, (int64_t) length, 0, &scratch, false, NULL }; + bool integral = false; + TableJsonSpace( probe ); + if ( probe.pos != 0 ) { return false; } // whitespace is never part of a key + if ( !TableJsonWalkNumber( probe, &integral ) ) { return false; } + if ( probe.pos != (int64_t) length ) { return false; } // trailing bytes: not a number + // A MAP KEY'S POLICY over the one interpreted value: it REJECTS THE WHOLE + // ENTRY. A key is an identity, so a clamped one is two entries merged, and + // a value the key kind does not hold is kind_mismatch for that entry, + // dropped and counted, never clamped. + const TableJsonInteger number = TableJsonInterpretExact( token, length ); + if ( number.fractional || number.saturated ) { return true; } + bool moved = false; + value = TableJsonIntegerInDomain( number, TableJsonMapKeySigned( key ), (int32_t) key->elem_size, moved ); + fits = !moved; + return true; +} + +// FromJson READS KEYS IN WHATEVER ORDER THE TEXT GIVES THEM. A repeated key is +// last-wins and counted duplicate, the object rule (§16.2) applied inside the +// map. An empty object is an empty map, and null is kind_mismatch. +inline bool TableJsonReadMap( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; + const TableFieldInfo * key = TableJsonMapKeyField( f ); + const TableFieldInfo * value = TableJsonMapValueField( f ); + const char shape = TableJsonShape( value ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + char token[kTableJsonMaxKey]; + int32_t token_length = 0; + bool key_over = false; // longer than THIS buffer: never truncated into a key + if ( !TableJsonScanString( in, token, kTableJsonMaxKey - 1, &token_length, &key_over ) ) { return false; } + token[token_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t key_value = 0; + bool place = true; + if ( !TableJsonMapKeyIsString( key ) ) + { + // AN INTEGER KEY PAST THIS SCAN'S BUFFER DROPS AS kind_mismatch, + // here and in the tool's walker. The bytes kept are a PREFIX, and a + // prefix is a different token, so the entry drops rather than a + // truncation being read as a value. The length alone does not + // settle it: a token that long can still spell a number an integer + // kind holds, "1" padded by an exponent of zeroes for one, and this + // read declines to find out. + bool fits = false; + if ( key_over ) { in.report->kind_mismatch++; place = false; } + else if ( !TableJsonMapKeyValue( token, token_length, key, key_value, fits ) ) + { + // A MALFORMED KEY STOPS THE READ where §16.1's rule stops it, + // with the instance holding what was placed before the stop. + in.report->malformed = true; + in.bad = true; + return false; + } + else if ( !fits ) { in.report->kind_mismatch++; place = false; } + } + else if ( key_over || token_length > key->array_bound ) + { + // A KEY LONGER THAN N DROPS ITS ENTRY AND COUNTS clamped, the + // wire's rule, because a clamped key is a merged entry (§2.8). The + // BOUND IS THE WALKER'S, tested here against the key field's own + // descriptor, so placement is left with one failure to report. A + // key past this scan's own buffer is the SAME event, because a + // truncated key is the merged entry the rule exists to prevent. + in.report->clamped++; + place = false; + } + const int32_t before = TableJsonExtentCount( (const void *) slot ); + void * entry = place ? f->place( *graph->worker, slot, token, token_length, key_value ) : NULL; + if ( place && entry == NULL ) + { + // AN ALLOCATION FAILURE IS NOT AN OVERSIZED KEY (§2.8, §16.1). The + // key was checked above, so the arena is what refused, and the read + // stops where the list, blob and pointer paths stop on one rather + // than handing back an instance short of entries the text spelled + // and calling itself clean. + in.report->malformed = true; + in.bad = true; + return false; + } + else if ( entry != NULL && TableJsonExtentCount( (const void *) slot ) == before ) + { + in.report->duplicate++; // last-wins, the object rule inside the map + } + const char got = TableJsonValueShape( in ); + if ( entry == NULL ) + { + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( value->kind == 17 && !value->is_array ) + { + // A POINTER VALUE IS SHARED EXACTLY AS A POINTER FIELD IS (§2.8): + // null is a null slot, an object is the pointee in place or an + // &node reference to one (§16.7), anything else is the wrong shape — + // the same three the field-key loop gives a pointer field, because + // an entry's value IS a field line. + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) entry + value->offset, value->elem_size, 0 ); + } + else if ( got != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, (uint8_t *) entry + value->offset, value, depth + 1 ) ) + { + return false; + } + } + else if ( got != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadField( in, entry, value, depth + 1 ) ) + { + return false; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; +} + +// ---- json map walk: end ---- + +// ---- json list walk: begin ---- + +// an unbounded array is the out-of-line array that is not a map (§8.1) +inline bool TableJsonIsList( const TableFieldInfo * f ) +{ + return f->is_array && f->array_bound == 0 && !TableJsonIsMap( f ); +} + +// ToJson WRITES THE ELEMENTS IN INDEX ORDER, which is the only order there is, +// so unpack then pack is byte-stable without a rule of its own (§2.9, §17.2). +// A region holds the array in place, so this steps it at the descriptor's pitch. +inline bool TableJsonWriteList( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + const int32_t count = TableJsonExtentCount( slot ); + if ( count == 0 ) { out.raw( "[]", 2 ); return true; } + const uint8_t * elements = TableJsonExtentElements( slot ); + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + const uint8_t * element = elements + (int64_t) i * f->elem_size; + if ( f->kind == 17 ) + { + // a []*T's elements take the pointer row (§16.7): the pointee's + // object in place, null, or `&node` for a shared one + if ( !TableJsonWritePointer( out, element, f, depth + 1 ) ) { return false; } + } + else if ( !TableJsonWriteScalar( out, element, f, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( ']' ); + return true; +} + +// FromJson READS EVERY ELEMENT THE TEXT CARRIES, appending each through the +// descriptor's place resolver: `[]` is an empty list, and null is +// kind_mismatch, the array row's own rule (§16.2). LAST WINS holds for a +// repeated key: the list goes back to EMPTY before this occurrence's elements +// land, the builder's storage being reclaimed at reset (§2.9). +inline bool TableJsonReadList( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; + TableJsonSetRaw( (uint8_t *) slot, 8, 0 ); + TableJsonSetRaw( (uint8_t *) slot + 8, 4, 0 ); + const char shape = TableJsonElementShape( f ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + void * element = f->place( *graph->worker, slot, NULL, 0, 0 ); + if ( element == NULL ) + { + // NOT ADDED: the arena could not carve another segment, or the + // count met the int32 cap. The text cannot be placed whole, and + // the read stops where §16.1's rule stops it. + in.report->malformed = true; + in.bad = true; + return false; + } + if ( f->kind == 17 ) + { + // an element of a []*T (§2.9): null is a null slot, an object is the + // pointee in place or an `&node` reference (§16.7) + char got = TableJsonValueShape( in ); + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) element, f->elem_size, 0 ); + } + else if ( got != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, element, f, depth + 1 ) ) { return false; } + } + else if ( TableJsonValueShape( in ) != shape ) + { + // the wrong shape for the element kind: the slot keeps its + // defaults and the event counts, the array row's rule (§16.2) + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadScalar( in, element, f, depth + 1 ) ) { return false; } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; +} + +// ---- json list walk: end ---- + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_JSON + +namespace mapdemo { + +bool PairsFromJson( PairsBuilder & builder, const char * text, int64_t bytes, TableReport * report ) +{ + Pairs * root = builder.GetRoot(); + if ( root == NULL ) { if ( report != NULL ) { report->malformed = true; } return false; } // locked, or the root allocation failed + return TableJsonReadGraph( builder.main, root, PairsTableType(), text, bytes, report ); +} + +int64_t PairsToJsonMeasure( const Pairs * root, TableAllocator allocator ) +{ + return TableJsonWriteGraph( root, PairsTableType(), NULL, 0, allocator ); +} + +int64_t PairsToJson( const Pairs * root, char * buffer, int64_t capacity, TableAllocator allocator ) +{ + return TableJsonWriteGraph( root, PairsTableType(), buffer, capacity, allocator ); +} + +} // namespace mapdemo diff --git a/testdata/golden/tables/maps/PairsTable.h b/testdata/golden/tables/maps/PairsTable.h new file mode 100644 index 000000000..b9b102c4d --- /dev/null +++ b/testdata/golden/tables/maps/PairsTable.h @@ -0,0 +1,9471 @@ +// Code generated by the schema compiler from Pairs.schema. DO NOT EDIT. +// SPDX-License-Identifier: NONE — this generated output is yours, under terms of +// your choice. See the LICENSE exception in the schema compiler; the compiler is +// AGPL-3.0, its output is not. +// package mapdemo — protocol id 0x1ac124decde5b2aa (packets only: tables version by field id, not by protocol id) +// The TABLE wire (evolution-tolerant, docs/SPEC-TABLES.md): no serialize +// dependency — includable from any TU. + +#pragma once + +#include +#include // the prefill's scalar-array fills +#include // offsetof, for the reflection descriptors + +// ---- the hooks (docs/USAGE.md, "the C++ table runtime's hooks") ---- +// +// schema_assert — the runtime's own assert, and the refusal a debugger reads. +// NDEBUG removes it, exactly as it removes assert. A caller who already routes +// serialize's asserts writes `#define schema_assert serialize_assert` before +// including this header and both halves land in one handler. +#ifndef schema_assert +#include +#define schema_assert assert +#endif // #ifndef schema_assert + +// schema_fatal — what stands after the assert on a path that cannot continue. +// NDEBUG does not remove it. Supply it and is never included. +#ifndef schema_fatal +#include // abort +#define schema_fatal abort +#endif // #ifndef schema_fatal + +// schema_allocate / schema_release — what "no allocator handed in" means for +// this program. schema_allocate hands back ZEROED bytes and NULL on failure: +// an arena segment is copied whole, padding included, so anything left +// uninitialized here would reach a packed region. Supply both and +// is never included; hand a TableAllocator to a builder to route one +// structure's allocations somewhere else again. +#ifndef schema_allocate +#include // calloc, free +#define schema_allocate( bytes ) calloc( (size_t) 1, (size_t) ( bytes ) ) +#define schema_release( pointer ) free( pointer ) +#endif // #ifndef schema_allocate +#include // a node's lifetime starts in arena storage (placement new) +#include // one atomic per slab: the arena is lock-free by ownership + +#include "Pairs.h" +#include "FleetTable.h" + +#ifndef MAPDEMO_SCHEMA_TABLE_PRIMITIVES +#define MAPDEMO_SCHEMA_TABLE_PRIMITIVES + +// THE CODEC DOES NOT DEPEND ON THE COMPILER'S INLINING BUDGET. A table of a +// realistic field count emits one large body per type, and the cursor a body +// writes through lives in the caller's `TableWriter`: across a call boundary +// that cursor round-trips through memory, and a `uint8_t *` store may alias the +// writer itself, so every put reloads it. When a budget runs out mid-body the +// codec silently degrades to that shape. Forcing the primitives and the +// fixed-class bodies inline is what keeps the cursor in registers and lets +// adjacent constant framing bytes merge into one store. +#if defined( _MSC_VER ) +#define MAPDEMO_TABLE_INLINE __forceinline +#elif defined( __GNUC__ ) || defined( __clang__ ) +#define MAPDEMO_TABLE_INLINE inline __attribute__(( always_inline )) +#else +#define MAPDEMO_TABLE_INLINE inline +#endif + +namespace mapdemo { + +// WHY A READ WAS REFUSED, by name (docs/SPEC-TABLES.md §3.3, §11). A REFUSAL +// is not one of §4's events: nothing is decoded, no counter moves and no +// damage is reported, so five zero counters and a false flag are what a clean +// read prints too and only the verdict tells them apart. The reason says which +// refusal it was. +// +// This is the MESSAGE PATH's vocabulary and not the cooked form's (§7.4): a +// caller meeting one of these has been refused a MESSAGE on a connection, +// which is a different recovery with a different owner than a file a header +// match turned down. +enum TableMessageReason +{ + newer_form, // a FORM BYTE this reader does not carry (§3) + no_vocabulary, // no table for this connection: the message arrived before the announcement, or after a refused one + second_announcement, // a second announcement on a connection: it sets nothing, amends nothing, and the connection closes + vocabulary_too_large, // an announcement above the receiver's declared bound, refused before an entry is touched + message_form_as_file, // a form 2 wire where a FILE was expected: its table is somewhere else + batch_too_large // a batch of more than 256 bodies on the write side, or of more than the caller has room for on the read side: nothing is written or decoded, and the count says what the wire carries +}; + +// The table-wire read report — the permissive contract's ledger. Silence +// (all zero) means the data matched this reader's schema exactly. +struct TableReport +{ + int32_t unknown = 0; // unknown field ids skipped (newer data) + int32_t kind_mismatch = 0; // known id, changed type — skipped, never misdecoded + // a kind that GREW since the writer (docs/SPEC-TABLES.md §4): an integer + // kind read into a wider one of the same signedness, or f32 into f64, + // decoded EXACTLY. One count per field or per map. It is the one counter + // that names no loss: the bytes were not the shape this reader declares, + // and the number survived. + int32_t widened = 0; + int32_t clamped = 0; // out-of-range values clamped to declared bounds + // a key the TEXT form saw twice: last wins, and the repeat is counted + // (docs/SPEC-TABLES.md §16.2). The wire never raises it — a body carrying an + // id twice is legal input whose last occurrence wins, silently (§3). + int32_t duplicate = 0; + bool malformed = false; // framing damage; decode stopped, partial result kept + // THE REFUSAL VERDICT, which is not one of §4's events and moves no counter + // (docs/SPEC-TABLES.md §3): a FORM BYTE this reader does not carry. Five + // zero counters and a false flag are what a clean read prints too, so the + // verdict is what tells the two apart. + bool refused = false; + // WHICH refusal, and it is read only when refused is set: a read that + // was not refused has no reason, and this member is the one the caller + // must not look at then (docs/SPEC-TABLES.md §3.3). + TableMessageReason reason = newer_form; + // RETAIN-UNKNOWN's pair (docs/SPEC-TABLES.md §6.6), on the same struct for + // the reason duplicate is: a caller has one report type and not two. Both + // are ZERO in every read that did not opt in, and retention moves no + // counter above. A retained field still counts unknown, because unknown + // says what a READER could not name and that stays true. + int32_t retained = 0; // unknown fields whose bytes were kept + int32_t retain_lost = 0; // every unknown this load or save could not keep +}; + + +// WHY A FILE WAS REFUSED, by name (docs/SPEC-TABLES.md §6.5, §7, §19.2): the +// one vocabulary a cook's Open, a block's BlockOpen and a load measure's -1 +// share, because a caller asking "why can I not have this file" is +// asking one question whichever call refused it. The FIRST failing clause names the +// reason, in the order §7 enumerates, so one file answers one value in every +// language. A refusal moves no counter, and a match writes nothing: the +// out-parameter is touched on the refusal path only. +// +// It is not the MESSAGE FORM's vocabulary (TableMessageReason, §3.3): a caller +// meeting one of these has been refused a FILE, by a header match or by a +// measure. +enum TableRefuseReason +{ + ok, // no clause failed: the only value beside a non-null root (§7) + not_a_cook, // the magic is neither this build's constant nor its byte reversal, or the byte-order word contradicts the magic + foreign_order, // the magic byte-reversed: a cook of the other byte order (§7.1) + wrong_build_version, // the build_version word is not this build's (§20) + reserved_not_zero, // a reserved header word is not zero (§7.1) + bad_alignment, // the alignment word is not a power of two, is below eight, is above sixty-four, or is not a multiple of the root's own alignof + truncated, // the part lengths against the caller's length, or a data part too short to hold the root + unaligned_base, // the pointer the caller passed is not aligned for the region: the caller's defect, not the file's + bad_layout, // BlockOpen (§19.2): a pitch, a count, an offset or an extent that disagrees with this build's or leaves the block + unknown_form, // at a MEASURE (§3, §6.5): a form byte this build does not carry, refused before any read + count_over_length, // an array or map count whose elements cannot fit the field's own L (§2.8, §2.9) + count_over_extent_cap, // a count above the int32 extent cap (§2.2), which no region can hold whatever its size + blob_over_size_cap, // a blob whose length is past the derived-size cap (§3.1, §11) + data_cycle // a data cycle reached from a builder: the AUTHORING side's -1 (§3.1, §7.6) +}; +// ---- reflection (tables only, docs/SPEC-TABLES.md) ---- +// +// Static field descriptors for every type in the table closure: name, wire +// id/kind, storage offset, bounds, ranges, enum names and branch guards — +// enough to walk, print, diff, edit or bind any table value at runtime with +// no RTTI and no schema files. TableType() returns X's descriptor. + +struct TableTypeInfo; + +// One arm of a union field: where its payload sits inside the union's storage +// and what its payload looks like. The arm's NAME and its table-wire id come +// from the field's enum_name/variant_id functions at the same tag, so nothing +// is spelled twice (docs/SPEC-TABLES.md §8). +struct TableFieldInfo; + +struct TableUnionArmInfo +{ + uint32_t offset; // offsetof the arm's payload within the union storage + const TableTypeInfo * table; // the arm payload's descriptor, or NULL + // AN ARM IS A FIELD LINE (docs/SPEC-TABLES.md §2.6): an arm that names no + // declared type or table carries the FIELD descriptor a field of that + // type would carry instead — offsets taken within the union storage — so + // a generic walk meets an arm's kind, width, bounds and companions where + // it meets a field's. Exactly one of the two is non-NULL on a set arm. + const TableFieldInfo * field; + uint32_t size; // the arm's whole storage, which selection zero-establishes +}; + +// A union field's shape: the tag, and the arms indexed by it. Arms run +// [0, enum_max]; index 0 is the EMPTY arm and carries no payload. +struct TableUnionInfo +{ + uint32_t tag_offset; // offsetof the tag within the union storage + uint32_t tag_size; // sizeof the tag + const TableUnionArmInfo * arms; +}; + +// The exact raw range of a wide-kind field (docs/SPEC-TABLES.md §8.2): two 128-bit +// values as 64-bit lanes, low lane first, two's complement for the signed kinds. +struct TableWideRange +{ + uint64_t lo[2]; + uint64_t hi[2]; +}; + +// THE SHARED EMPTY DOC (docs/SPEC-TABLES.md §8.1): a declaration with no /// +// block carries a doc column pointing at this one object, so absence costs a +// unit no string data and a printer concatenates doc columns with no null +// test. One definition for the whole unit: every absent doc compares equal by +// address. +inline const char TableDocNone[1] = ""; + +// the arena's allocation front, defined with the variable-length runtime +// below; a descriptor names it only through a pointer parameter. +struct TableWorker; + +struct TableFieldInfo +{ + const char * name; // schema field name, e.g. "health" + const char * json; // the TEXT form's key: the json = "key" attribute, else name (§16.3) + const char * type_name; // schema type name, e.g. "float32", "Grade" + uint64_t id; // table-wire field id: fnv1a64 of the name, of the was alias after a rename (§5) + uint8_t kind; // table-wire kind; for arrays/strings/bytes, the ELEMENT kind + bool is_array; // fixed or counted array (bytes included) + bool is_pointer; // a *T pointer field: storage is an 8-byte TableRef; the target is a table + // THE TWO THE TEXT FORM NEEDS (docs/SPEC-TABLES.md §16.7), and they + // are here for the same reason is_pointer is: the walk is ONE walk + // over descriptors and cannot spell a target's own At or + // Emplace. `resolve` reads a slot in a REGION and answers the + // node it names, or NULL; `emplace` allocates one in a BUILDER's + // arena and points the slot at it. NULL on every field that is not + // a pointer, and emitted only in a unit that declares one. + const void * (*resolve)( const void * slot ); + void * (*emplace)( TableWorker & worker, void * slot ); + bool counted; // a _count/_length int32 companion exists (counted arrays, strings, bytes) + bool optional; // a ?T field: a _present bool companion decides whether it rides + int32_t array_bound; // array capacity / string max length; 0 for plain scalars + uint32_t offset; // offsetof the storage member + uint32_t elem_size; // sizeof the member (element size for arrays) + uint32_t count_offset; // offsetof the _count/_length companion, or 0xffffffff + uint32_t present_offset; // offsetof the _present companion, or 0xffffffff + const TableTypeInfo * table; // nested table's descriptor, or NULL + bool has_range; // a declared [min, max] (int or float) + double range_min; // NOTE: int64 ranges beyond 2^53 lose precision here + double range_max; + // the WIDE kinds (18-29, docs/SPEC-TABLES.md §3, §8.2): frac_bits is a fixed + // field's F — its storage holds units × 2^F — and wide is the declared + // range on that RAW scale, exact, as two 128-bit two's-complement values + // in 64-bit lanes (low lane first). NULL where the declaration bounds + // nothing (a bare uint128) and for every other kind; frac_bits is 0 for + // every kind that is not fixed-point. range_min/range_max still carry + // the declared bounds as doubles — whole units for a fixed field — for + // a walker that only shows them. + uint8_t frac_bits; + const TableWideRange * wide; + int64_t enum_max; // enums: highest valid value (None = 0 always valid); + // unions: the arm count (tag range [0, enum_max]); + // flags: the highest declared BIT INDEX; else -1 + // the vocabulary's names, indexed the same way enum_max bounds: an enum's + // value -> name, a union's tag -> arm name, a FLAGS field's bit index -> + // variant name. NULL for every other kind. + const char * (*enum_name)( uint64_t value ); + // the TABLE-WIRE id of one variant (docs/SPEC-TABLES.md §5): for an enum, the + // hash of the variant's name; for a union, the hash of the arm's name. + // 0 is the reserved id — an enum's None, a union's empty. NULL for every + // other kind — a FLAGS field's variants have no per-variant wire id (§4), + // so a NULL here beside a non-NULL enum_name is what says "flags". + // Walk [0, enum_max] to enumerate a vocabulary and its ids. + uint64_t (*variant_id)( uint64_t value ); + // an ENUM-KEYED array (docs/SPEC-TABLES.md §2.4): the array has one slot per + // variant of key_type_name, indexed by the variant's value, and its slots + // ride under variant ids rather than positions. key_name and key_id are + // the key's vocabulary — walk [0, array_bound) to print slots by name. + // NULL on every other field. + const char * key_type_name; + const char * (*key_name)( uint64_t value ); + uint64_t (*key_id)( uint64_t value ); + // union fields: the tag and its arms, behind a function so the whole + // descriptor stays CONSTANT-INITIALISED (a captureless lambda converts to + // a function pointer at compile time; the arms themselves are a static + // inside it). NULL for every other kind. + const TableUnionInfo * (*arms)(); + // an OUT-OF-LINE array (docs/SPEC-TABLES.md §8.1): place one element and + // hand it back at its defaults. A MAP places BY KEY, a string key comes + // in as the bytes and the length, an integer key as the value, and NULL + // is NOT INSERTED: a key past the bound, or an arena that could not carve + // another segment. A LIST ignores the key and APPENDS, NULL at the arena + // or the int32 cap. NULL on every field that is neither. + void * ( * place )( TableWorker & worker, void * slot, const char * key, int32_t key_length, int64_t key_value ); + const char * guard; // branch guard, e.g. "at_rest" or "!at_rest"; "" if unguarded + // what a PERSON wrote about the field (docs/SPEC-TABLES.md §8.1): the /// + // block above it, verbatim (SPEC §4.1). It is TableDocNone when there is + // none, never NULL. Its tags (SPEC §4.2) follow in declared order, and an + // untagged field is 0 beside NULL. Static, constant-initialized, + // allocating nothing. + const char * doc; + int32_t num_tags; + const char * const * tags; +}; + +struct TableTypeInfo +{ + const char * name; // schema type name + uint32_t size; // sizeof the storage struct + int32_t num_fields; + const TableFieldInfo * fields; + // put one instance back at its declared defaults, in place. A generic + // walker that fills a value has to be able to establish the defaults an + // absent field takes, and it holds no type to spell — this is the one + // thing the descriptors could not express without it. Placement-new + // value-init, exactly what the wire's read path does, and no temporary. + void (*reset)( void * storage ); + // the DERIVED mode (docs/SPEC-TABLES.md): false = fixed-size, a plain + // relocatable struct; true = variable-length, built through a Builder + // and read through a region root. Nobody declares it; the compiler + // works it out. + bool variable; + // the declaration's own doc and tags, on the same terms as a field's + // (docs/SPEC-TABLES.md §8.1) + const char * doc; + int32_t num_tags; + const char * const * tags; +}; + +struct TableWriter +{ + uint8_t * buffer; + int64_t capacity; + int64_t offset = 0; + bool overflow = false; + + // the parameters do not repeat the member names: a parameter that hides a + // member is a warning the estate's compilers disagree about (gcc's + // -Wshadow and cl's C4458 refuse it, clang's -Wshadow does not), and this + // is a header a consumer compiles under its OWN flags + TableWriter( uint8_t * to_buffer, int64_t to_capacity ) : buffer( to_buffer ), capacity( to_capacity ) {} + + MAPDEMO_TABLE_INLINE void raw( const void * data, int64_t bytes ) + { + if ( offset + bytes > capacity ) { overflow = true; return; } + memcpy( buffer + offset, data, (size_t) bytes ); + offset += bytes; + } + MAPDEMO_TABLE_INLINE void put8( uint8_t v ) { raw( &v, 1 ); } + MAPDEMO_TABLE_INLINE void put16( uint16_t v ) { uint8_t b[2] = { uint8_t( v ), uint8_t( v >> 8 ) }; raw( b, 2 ); } + MAPDEMO_TABLE_INLINE void put32( uint32_t v ) { uint8_t b[4] = { uint8_t( v ), uint8_t( v >> 8 ), uint8_t( v >> 16 ), uint8_t( v >> 24 ) }; raw( b, 4 ); } + MAPDEMO_TABLE_INLINE void put64( uint64_t v ) { put32( uint32_t( v ) ); put32( uint32_t( v >> 32 ) ); } + // a 128-bit value as two lanes, the low half first (docs/SPEC-TABLES.md §3) + MAPDEMO_TABLE_INLINE void put128( uint64_t lo, uint64_t hi ) { put64( lo ); put64( hi ); } + // EVERY LENGTH, COUNT, INDEX AND ID REFERENCE IS ONE CANONICAL UNSIGNED + // LEB128 (docs/SPEC-TABLES.md §3): seven value bits a byte, the lowest + // group first, the high bit set on every byte but the last. One value has + // one spelling, so two conforming writers agree byte for byte. + MAPDEMO_TABLE_INLINE void putleb( uint64_t v ) + { + while ( v >= 0x80 ) { put8( uint8_t( v ) | 0x80 ); v >>= 7; } + put8( uint8_t( v ) ); + } +}; + +// TableLebBytes is one value's spelling length, which a MEASURE needs before +// the bytes exist — the length of a body has to be known before it is written, +// because a length whose own width moves cannot be patched in place. +inline int64_t TableLebBytes( uint64_t v ) +{ + int64_t n = 1; + while ( v >= 0x80 ) { v >>= 7; n++; } + return n; +} + +// THE ID TABLE, WRITER SIDE (docs/SPEC-TABLES.md §3). It holds every id the +// body used, once each, in FIRST-USE order over the whole wire, and the body +// names them by position: reference k is the kth entry, counted from 1, and +// reference 0 names NO ID. +// +// Its capacity is a COMPILE-TIME fact of the unit — the distinct names its +// table closure can spell — so a save allocates nothing: the table is a local +// of Measure and of Save. The bucket chain makes ref constant time and makes +// truncate constant time too, which is what an ELIDED field needs: a field +// that turns out not to ride costs nothing in the id table either, so the walk +// interns its id, builds the payload that decides, and undoes the entry when +// nothing rides. +struct TableIds +{ + static const int32_t kCapacity = 76; + static const int32_t kBuckets = 256; + + uint64_t ids[ kCapacity ]; + int32_t chain[ kCapacity ]; + int32_t head[ kBuckets ]; + int32_t count; + bool overflow; + + TableIds() : count( 0 ), overflow( false ) + { + for ( int32_t i = 0; i < kBuckets; i++ ) { head[i] = -1; } + } + + static MAPDEMO_TABLE_INLINE uint32_t bucket_of( uint64_t id ) + { + return uint32_t( ( id * 0x9E3779B97F4A7C15ull ) >> 56 ) & uint32_t( kBuckets - 1 ); + } + + // the reference an id takes: the file's own first-use entry, appended on + // first use. The MESSAGE form names no id at all: its references are + // compile-time slots of the announced vocabulary (docs/SPEC-TABLES.md §3.3). + MAPDEMO_TABLE_INLINE uint64_t ref( uint64_t id ) + { + const uint32_t b = bucket_of( id ); + for ( int32_t i = head[b]; i >= 0; i = chain[i] ) + { + if ( ids[i] == id ) { return uint64_t( i ) + 1; } + } + if ( count >= kCapacity ) { overflow = true; return 1; } + ids[count] = id; chain[count] = head[b]; head[b] = count; count++; + return uint64_t( count ); + } + + // undo every entry appended since mark. An entry removed is the most + // recent one in its bucket, so it sits at that bucket's head. + void truncate( int32_t mark ) + { + while ( count > mark ) + { + count--; + head[ bucket_of( ids[count] ) ] = chain[count]; + } + } +}; + +// TableIdsBytes is the trailer's own size: the entries, each a fixed +// little-endian u64, and the ENTRY COUNT, the one fixed-width number on the +// wire (docs/SPEC-TABLES.md §3). +inline int64_t TableIdsBytes( const TableIds & ids ) { return int64_t( ids.count ) * 8 + 8; } + +// TableIdsWrite puts the trailer where the walk ended: a writer never patches, +// because first-use order is known only when the walk ends. +inline void TableIdsWrite( TableWriter & w, const TableIds & ids ) +{ + for ( int32_t i = 0; i < ids.count; i++ ) { w.put64( ids.ids[i] ); } + w.put64( uint64_t( ids.count ) ); +} + +// THE ID TABLE, READER SIDE (docs/SPEC-TABLES.md §3). A reader locates it from +// the END of the wire and resolves it ONCE, at open: the entries are eight +// bytes each and a body names them by position, so every field dispatches +// through an index rather than through a search over hashes. +struct TableIdTable +{ + const uint8_t * entries = NULL; + int64_t count = 0; + + // the id a reference names. ref is 1-based and bounds-checked by the + // caller: a reference ABOVE the entry count is framing damage on the body + // that carries it, and 0 names no id at all. + uint64_t at( uint64_t ref ) const + { + const uint8_t * e = entries + ( ref - 1 ) * 8; + uint64_t lo = uint64_t( e[0] ) | uint64_t( e[1] ) << 8 | uint64_t( e[2] ) << 16 | uint64_t( e[3] ) << 24; + uint64_t hi = uint64_t( e[4] ) | uint64_t( e[5] ) << 8 | uint64_t( e[6] ) << 16 | uint64_t( e[7] ) << 24; + return lo | ( hi << 32 ); + } +}; + +struct TableReader +{ + const uint8_t * buffer; + int64_t size; + int64_t offset = 0; + TableReport * report; + const TableIdTable * ids = NULL; + // ONLY THE ROOT BODY CARRIES THE NODE TABLE (docs/SPEC-TABLES.md §3.1), so + // a body has to know which it is: the reserved id inside a NESTED body is + // malformed, because a second numbering cannot exist. Every reader made + // for a payload is nested; the two the wire surfaces make for a root say so. + bool nested = true; + + TableReader( const uint8_t * from_buffer, int64_t from_size, TableReport * to_report ) + : buffer( from_buffer ), size( from_size ), report( to_report ) {} + + TableReader( const uint8_t * from_buffer, int64_t from_size, TableReport * to_report, const TableIdTable * to_ids ) + : buffer( from_buffer ), size( from_size ), report( to_report ), ids( to_ids ) {} + + MAPDEMO_TABLE_INLINE bool has( int64_t bytes ) const { return offset + bytes <= size; } + // A LENGTH IS A 64-BIT NUMBER AND A BUFFER IS NOT (docs/SPEC-TABLES.md + // §3): every length, count and index on this wire has sixty-four bits of + // capability, so one past what remains must be compared UNSIGNED. Casting + // it to int64 first turns 0xFFFFFFFFFFFFFFFF into -1, and a negative + // length looks like room. + MAPDEMO_TABLE_INLINE bool room( uint64_t bytes ) const { return bytes <= (uint64_t) ( size - offset ); } + MAPDEMO_TABLE_INLINE uint8_t get8() { return buffer[offset++]; } + MAPDEMO_TABLE_INLINE uint16_t get16() { uint16_t v = uint16_t( buffer[offset] ) | uint16_t( buffer[offset+1] ) << 8; offset += 2; return v; } + MAPDEMO_TABLE_INLINE uint32_t get32() { uint32_t v = uint32_t( buffer[offset] ) | uint32_t( buffer[offset+1] ) << 8 | uint32_t( buffer[offset+2] ) << 16 | uint32_t( buffer[offset+3] ) << 24; offset += 4; return v; } + MAPDEMO_TABLE_INLINE uint64_t get64() { uint64_t lo = get32(); uint64_t hi = get32(); return lo | ( hi << 32 ); } + MAPDEMO_TABLE_INLINE void get128( uint64_t & lo, uint64_t & hi ) { lo = get64(); hi = get64(); } + + // ONE CANONICAL UNSIGNED LEB128 (docs/SPEC-TABLES.md §3), and a + // non-minimal spelling is MALFORMED: 0x80 0x00 and 0x00 both spell zero, + // and only the second is legal input. An encoding past ten bytes, or a + // tenth byte with a bit above the 64th value bit, is malformed on the same + // rule. false = framing damage on the body carrying it. + bool getleb( uint64_t & value ) + { + // A NUMBER THIS READER REFUSES LEAVES THE CURSOR WHERE IT WAS. The + // caller's next question is often "did this body end exactly at its + // L", and a rejected number that had moved the cursor would answer + // that question with the damage already stepped over. + const int64_t at = offset; + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( !has( 1 ) ) { offset = at; return false; } + const uint8_t b = get8(); + if ( i == 9 && b > 1 ) { offset = at; return false; } + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) + { + if ( i > 0 && b == 0 ) { offset = at; return false; } // a redundant continuation + return true; + } + shift += 7; + } + offset = at; + return false; + } + + // resolve one id reference against the file's table. false = a reference + // ABOVE the entry count, or a 0 where an id is required, both of which + // are framing damage on the body that carries it. + bool getid( uint64_t & id ) + { + uint64_t ref = 0; + if ( !getleb( ref ) ) { return false; } + if ( ref == 0 || ids == NULL || ref > (uint64_t) ids->count ) { return false; } + id = ids->at( ref ); + return true; + } + + // skip one payload by kind; false = framing damage. FOUR RULES COVER THE + // SET (docs/SPEC-TABLES.md §3), and a kind outside it is not skippable — + // which is why the set is closed and why kind 31 exists. + bool skip( uint8_t kind ) + { + switch ( kind ) + { + // the fixed-width kinds, each by its width: 18-29 are the 128-bit integers and + // the fixed-point family at every storage width (docs/SPEC-TABLES.md §3) + case 1: case 2: case 6: case 20: case 25: return has( 1 ) ? ( offset += 1, true ) : false; + case 3: case 7: case 21: case 26: return has( 2 ) ? ( offset += 2, true ) : false; + case 4: case 8: case 10: case 22: case 27: return has( 4 ) ? ( offset += 4, true ) : false; + case 5: case 9: case 11: case 23: case 28: return has( 8 ) ? ( offset += 8, true ) : false; + case 18: case 19: case 24: case 29: return has( 16 ) ? ( offset += 16, true ) : false; + case 17: case 30: // a NODE INDEX (§3.1) and an ENUM's variant reference: one LEB128 and stop + { + uint64_t ignored = 0; + return getleb( ignored ); + } + case 12: case 13: case 14: case 16: case 31: case 32: case 33: // 31 is the ESCAPE, 32 the payload-free kind, 33 wide text + { + uint64_t n = 0; + if ( !getleb( n ) ) return false; + return room( n ) ? ( offset += (int64_t) n, true ) : false; + } + case 15: // union: the arm id reference, then its kind, its L and its payload (reference 0 = empty) + { + uint64_t arm = 0; + if ( !getleb( arm ) ) return false; + if ( arm == 0 ) return true; + if ( !has( 1 ) ) return false; + offset += 1; // the arm's kind byte + uint64_t n = 0; + if ( !getleb( n ) ) return false; + return room( n ) ? ( offset += (int64_t) n, true ) : false; + } + // KIND 34 IS RESERVED FOR float16 AND IS NOT PART OF THIS MAJOR (§3): + // no writer emits it and no reader has a rule for it, so a reader + // meets it only as DAMAGE, exactly as it meets 35 or 200. A bare 34 + // is a writer that ignored the escape kind 31. + case 34: return false; + } + return false; + } +}; + + +// WIDENING (docs/SPEC-TABLES.md §4): a payload under a kind BELOW the reader's +// on the same ladder decodes exactly. The signed ladder is kinds 2, 3, 4, 5, +// 18, the unsigned one 6, 7, 8, 9, 19, and 10 into 11 is the float rung. Every +// other pair is a kind mismatch. The declared kind is a constant at every call +// site, so this folds to one or two comparisons on the mismatch path and to +// nothing on the matching one. +inline bool TableKindWidens( uint8_t kind, uint8_t declared ) +{ + switch ( declared ) + { + case 3: case 4: case 5: return kind >= 2 && kind < declared; + case 18: return kind >= 2 && kind <= 5; + case 7: case 8: case 9: return kind >= 6 && kind < declared; + case 19: return kind >= 6 && kind <= 9; + case 11: return kind == 10; + } + return false; +} + +// a fixed-width kind's payload width, for the one place the width is a +// runtime fact: an arm whose kind byte the reader widens, whose L must be the +// wire kind's own width (§3) +inline int64_t TableKindWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + } + return 0; +} + +// the payload of a kind on the SIGNED ladder (2 to 5), sign-extended to +// sixty-four bits; false = the body cannot cover it, which is framing damage +inline bool TableReadSignedAt( TableReader & r, uint8_t kind, int64_t & out ) +{ + switch ( kind ) + { + case 2: if ( !r.has( 1 ) ) { return false; } out = (int8_t) r.get8(); return true; + case 3: if ( !r.has( 2 ) ) { return false; } out = (int16_t) r.get16(); return true; + case 4: if ( !r.has( 4 ) ) { return false; } out = (int32_t) r.get32(); return true; + default: if ( !r.has( 8 ) ) { return false; } out = (int64_t) r.get64(); return true; + } +} + +// the payload of a kind on the UNSIGNED ladder (6 to 9), zero-extended +inline bool TableReadUnsignedAt( TableReader & r, uint8_t kind, uint64_t & out ) +{ + switch ( kind ) + { + case 6: if ( !r.has( 1 ) ) { return false; } out = r.get8(); return true; + case 7: if ( !r.has( 2 ) ) { return false; } out = r.get16(); return true; + case 8: if ( !r.has( 4 ) ) { return false; } out = r.get32(); return true; + default: if ( !r.has( 8 ) ) { return false; } out = r.get64(); return true; + } +} + +// f32 into f64, exact: a NaN's payload is data and rides on the bits, since +// the hardware conversion would set the quiet bit (§4) +inline double TableWidenF32( uint32_t bits ) +{ + if ( ( bits & 0x7F800000u ) == 0x7F800000u && ( bits & 0x007FFFFFu ) != 0 ) + { + const uint64_t sign = (uint64_t) ( bits >> 31 ) << 63; + const uint64_t payload = (uint64_t) ( bits & 0x007FFFFFu ) << 29; + const uint64_t nan_bits = sign | 0x7FF0000000000000ull | payload; + double d; memcpy( &d, &nan_bits, 8 ); return d; + } + float f; memcpy( &f, &bits, 4 ); return (double) f; +} + +// ILL-FORMED TEXT IS DAMAGE (docs/SPEC-TABLES.md §3, §4): a kind 12 payload is +// well-formed UTF-8 with no zero byte among its bytes, checked AS IT ARRIVES +// and before the reader's own bound, because a payload that is not text is not +// text at whatever length the reader would have kept. Rejects a zero byte, a +// truncated sequence, a bare continuation, an overlong encoding, a surrogate +// and a code point past U+10FFFF, which is SPEC.md §4.7's rule in this wire's +// idiom: the field reads its declared default, one malformed counts, and the +// parent reads on past L. +// +// A LENGTH IS A 64-BIT NUMBER (§3), so it arrives as one: a payload length is +// whatever the wire spelled, and narrowing it to a signed count would read +// 0xFFFFFFFFFFFFFFFF as an empty payload. +inline bool TableUtf8Valid( const uint8_t * bytes, uint64_t length ) +{ + uint64_t i = 0; + while ( i < length ) + { + const uint8_t lead = bytes[i]; + uint64_t continuations; + uint32_t code_point; + if ( lead == 0 ) { return false; } + if ( lead < 0x80 ) { i++; continue; } + else if ( ( lead & 0xE0 ) == 0xC0 ) { continuations = 1; code_point = lead & 0x1F; } + else if ( ( lead & 0xF0 ) == 0xE0 ) { continuations = 2; code_point = lead & 0x0F; } + else if ( ( lead & 0xF8 ) == 0xF0 ) { continuations = 3; code_point = lead & 0x07; } + else { return false; } + if ( i + continuations >= length ) { return false; } + for ( uint64_t k = 1; k <= continuations; k++ ) + { + if ( ( bytes[i + k] & 0xC0 ) != 0x80 ) { return false; } + code_point = ( code_point << 6 ) | uint32_t( bytes[i + k] & 0x3F ); + } + if ( continuations == 1 && code_point < 0x80 ) { return false; } + if ( continuations == 2 && ( code_point < 0x800 || ( code_point >= 0xD800 && code_point <= 0xDFFF ) ) ) { return false; } + if ( continuations == 3 && ( code_point < 0x10000 || code_point > 0x10FFFF ) ) { return false; } + i += 1 + continuations; + } + return true; +} + +// A CLAMP CUTS AT A CODE POINT BOUNDARY (§3, §16.2): the last whole code point +// that fits within the bound, over a payload the check above already accepted, +// so a clamp can never invent ill-formed storage. +// +// THE ANSWER IS NEVER ABOVE THE BOUND. The length arrives as the wire's own +// 64-bit number and the caller turns the answer back into the size of a copy, +// so a length no reader could have bounded has to leave here bounded: taken as +// a signed count, 0xFFFFFFFFFFFFFFFF is -1, -1 is under every bound, and the +// copy would run at SIZE_MAX. +inline int64_t TableUtf8Clamp( const uint8_t * bytes, uint64_t length, int64_t bound ) +{ + if ( length <= (uint64_t) bound ) { return (int64_t) length; } + int64_t cut = bound; + while ( cut > 0 && ( bytes[cut] & 0xC0 ) == 0x80 ) { cut--; } + return cut; +} + +// ONE CODE UNIT off the wire: two bytes LITTLE-ENDIAN, this wire's order for +// every fixed-width number (docs/SPEC-TABLES.md §3). No unit can exceed +// 0xFFFF, because two bytes cannot spell one. +inline uint16_t TableUtf16Unit( const uint8_t * bytes, int64_t index ) +{ + return uint16_t( uint16_t( bytes[index * 2] ) | ( uint16_t( bytes[index * 2 + 1] ) << 8 ) ); +} + +// ILL-FORMED WIDE TEXT IS DAMAGE (docs/SPEC-TABLES.md §3, §4): a kind 33 +// payload carrying an UNPAIRED SURROGATE or a ZERO CODE UNIT among its units, +// checked AS IT ARRIVES and before the reader's own bound, on the rule kind 12 +// takes for UTF-8. An ODD L is framing damage and the caller rejects it ahead +// of this, because units is L / 2. SPEC.md §4.12 refuses the same content +// TERMINALLY on the packet wire; here the field reads its declared default, +// one malformed counts, and the parent reads on past L. +inline bool TableUtf16Valid( const uint8_t * bytes, int64_t units ) +{ + int64_t i = 0; + while ( i < units ) + { + const uint16_t unit = TableUtf16Unit( bytes, i ); + if ( unit == 0 ) { return false; } + if ( unit >= 0xD800 && unit <= 0xDBFF ) + { + if ( i + 1 >= units ) { return false; } // a high surrogate with no low half + const uint16_t low = TableUtf16Unit( bytes, i + 1 ); + if ( low < 0xDC00 || low > 0xDFFF ) { return false; } + i += 2; + continue; + } + if ( unit >= 0xDC00 && unit <= 0xDFFF ) { return false; } // a low surrogate first + i++; + } + return true; +} + +// A CLAMP CUTS AT A CODE UNIT BOUNDARY AND NEVER SPLITS A PAIR (§3, §16.2): +// the first bound units of a payload the check above already accepted, and +// where the last kept unit is a HIGH SURROGATE whose low half did not fit, +// that unit is dropped with it. So a clamp can never invent an unpaired +// surrogate, exactly as kind 12's clamp can never invent a broken sequence. +inline int64_t TableUtf16Clamp( const uint8_t * bytes, int64_t units, int64_t bound ) +{ + if ( units <= bound ) { return units; } + int64_t cut = bound; + if ( cut > 0 ) + { + const uint16_t last = TableUtf16Unit( bytes, cut - 1 ); + if ( last >= 0xD800 && last <= 0xDBFF ) { cut--; } + } + return cut; +} + +// The RESERVED node-table id, the one id the language holds back +// (docs/SPEC-TABLES.md §3.1, §5). It rides in every unit, pointered or not, +// because every body has to know that a NESTED body claiming one is damaged. +static const uint64_t kTableNodeTableFieldId = 0xFFFFFFFFFFFFFFFFull; + +// TableWireForm is the FORM BYTE, and it is the whole header +// (docs/SPEC-TABLES.md §3). A reader that meets a byte it does not know +// refuses the wire by name and never reports damage. +const uint8_t kTableWireForm = 1; + +// TableOpen reads the form byte and the trailer, in that order, and hands back +// the ROOT BODY. It answers one of three verdicts, because five zero counters +// and a false flag are what a clean read prints too: +// +// TableOpenOk the form is known and the table read whole +// TableOpenRefused a FORM BYTE this reader does not carry: nothing is +// decoded, nothing is counted, and no damage is reported +// TableOpenDamaged a table that cannot be read whole — fewer than eight +// bytes, a count whose entries run past the front of the +// file, a count that leaves no room for the form byte, or +// ONE ID IN TWO ENTRIES. The whole wire is malformed, +// nothing is decoded, and one event is counted. +// TableOpenBodyStopped the form and the table were good and the ROOT BODY +// could not be walked to its own terminator. What it +// decoded before that is kept, as everywhere on this wire. +enum TableOpenVerdict { TableOpenOk, TableOpenRefused, TableOpenDamaged, TableOpenBodyStopped }; + +inline TableOpenVerdict TableOpen( const uint8_t * buffer, int64_t bytes, TableIdTable & table, int64_t & body_bytes ) +{ + if ( bytes < 1 ) { return TableOpenDamaged; } + if ( buffer[0] != kTableWireForm ) { return TableOpenRefused; } + if ( bytes < 9 ) { return TableOpenDamaged; } + const uint8_t * tail = buffer + bytes - 8; + uint64_t lo = uint64_t( tail[0] ) | uint64_t( tail[1] ) << 8 | uint64_t( tail[2] ) << 16 | uint64_t( tail[3] ) << 24; + uint64_t hi = uint64_t( tail[4] ) | uint64_t( tail[5] ) << 8 | uint64_t( tail[6] ) << 16 | uint64_t( tail[7] ) << 24; + uint64_t count = lo | ( hi << 32 ); + if ( count > (uint64_t) ( bytes / 8 ) ) { return TableOpenDamaged; } + const int64_t span = (int64_t) count * 8 + 8; + if ( span + 1 > bytes ) { return TableOpenDamaged; } + table.entries = buffer + bytes - span; + table.count = (int64_t) count; + // THE ENTRIES ARE DISTINCT: a table that carries one id twice is malformed + // for the whole wire, because no wire this schema writes carries a repeat + // and it would leave one more shape of table for a hostile writer to aim + // at (docs/SPEC-TABLES.md §3). + for ( int64_t i = 1; i < table.count; i++ ) + { + const uint64_t id = table.at( uint64_t( i ) + 1 ); + for ( int64_t j = 0; j < i; j++ ) + { + if ( table.at( uint64_t( j ) + 1 ) == id ) { return TableOpenDamaged; } + } + } + body_bytes = bytes - span - 1; + return TableOpenOk; +} + +// TableBodyExtent walks a body's framing to the zero reference that ends it, +// so a reader can tell a body that ENDED EARLY — leaving bytes no field claims +// — from one that is merely damaged. ANY BYTE BETWEEN THE ROOT'S TERMINATOR +// AND THE TABLE'S FIRST ENTRY IS MALFORMED, because no field claims it and the +// two ends of the file have met (docs/SPEC-TABLES.md §3). +inline bool TableBodyEndsEarly( const uint8_t * body, int64_t bytes, const TableIdTable & table ) +{ + TableReport ignored; + TableReader r( body, bytes, &ignored, &table ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.getleb( ref ) ) { return false; } + if ( ref == 0 ) { return r.offset != bytes; } + if ( ref > (uint64_t) table.count ) { return false; } + if ( !r.has( 1 ) ) { return false; } + if ( !r.skip( r.get8() ) ) { return false; } + } +} + +// THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3): a batch of BITPACKED bodies +// under one announced vocabulary. +// +// A form 2 wire is THREE PARTS: the form byte, the body count, and the bodies +// as one continuous bit stream, zero-padded to the next byte at the end and +// nowhere else. A body is a sequence of fields, each a REFERENCE followed by a +// PAYLOAD and nothing else: no kind byte and no length, because the +// announcement carries the kind and the shape of every entry. +const uint8_t kTableWireMessageForm = 2; + +// THE COUNT IS A RANGED INTEGER OVER [1, 256], eight bits carrying M - 1. 256 +// is a WIRE CONSTANT of this form rather than a receiver's policy, because the +// count's WIDTH depends on it and two peers that disagreed on the width would +// not be reading the same wire. A batch of zero is not spellable. +static const int64_t kTableMessageBatchMax = 256; + +// The RESERVED ids of the announcement's own two fields (§5, §11), beside the +// node table's. They are the announcement's transport, they never appear in a +// body, and they take no slot in the vocabulary. +static const uint64_t kTableBuildVersionFieldId = 0xFFFFFFFFFFFFFFFEull; +static const uint64_t kTableMessageVocabularyFieldId = 0xFFFFFFFFFFFFFFFDull; + +// THE WIDEST COUNT THIS FORM SPELLS, which is the count an UNBOUNDED array +// announces (§2.9): an unbounded array states no bound, so the announcement +// states the widest one a batch could carry. It is the ceiling an array's or a +// keyed entry's announced min and max are checked against. +static const uint64_t kTableMessageListMax = 0xFFFFFFFFull; + +// THIS UNIT'S OWN REFERENCE WIDTH: the bits a writer spends on every reference +// of every body it writes, which is a compile-time constant because the +// vocabulary is. A READER spends the width the SENDER's vocabulary settles. +static const int64_t kTableMessageRefBitsHere = 7; + +// THIS UNIT'S OWN ENTRY COUNT, which is the CAPACITY a receiver declares for +// its resolved vocabulary when it talks only to peers of this schema (§3.3). +// The vocabulary is a pure function of the build version, so a peer at this +// build announces exactly this many entries; a receiver that means to meet +// OTHER builds declares more, and an announcement above whatever it declared +// is refused as vocabulary_too_large. +static const int64_t kTableMessageEntriesHere = 75; + +// The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A +// pointered body names the node table through it, and the node table is the +// ROOT body's FIRST field because a pointer index's width is settled by the +// node count it carries. +static const uint64_t kTableNodeTableFieldSlot = 54; + +// THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own +// layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, +// so a value written here and a value written by a generated packet writer are +// the same bits in the same places. + +// EIGHT BYTES OF THE STREAM AS ONE WORD, and the word is LITTLE-END-FIRST +// whatever order this host is in, because the stream's own definition puts +// bit i in byte i/8: byte 0 of the run holds the word's low eight bits. That +// is what lets one value of any width move in one unaligned load or store +// instead of one touch a byte, and the BITS ON THE WIRE do not move. +inline uint64_t table_message_byteswap64( uint64_t v ) +{ + return ( v >> 56 ) | ( ( v >> 40 ) & 0xff00ull ) | ( ( v >> 24 ) & 0xff0000ull ) | ( ( v >> 8 ) & 0xff000000ull ) + | ( ( v << 8 ) & 0xff00000000ull ) | ( ( v << 24 ) & 0xff0000000000ull ) | ( ( v << 40 ) & 0xff000000000000ull ) + | ( v << 56 ); +} + +inline uint64_t table_message_load64( const uint8_t * p ) +{ + uint64_t v = 0; + memcpy( &v, p, 8 ); +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + v = table_message_byteswap64( v ); +#endif + return v; +} + +inline void table_message_store64( uint8_t * p, uint64_t v ) +{ +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + v = table_message_byteswap64( v ); +#endif + memcpy( p, &v, 8 ); +} + +struct TableBitWriter +{ + uint8_t * buffer; + int64_t capacity; // bytes + int64_t bits; + bool overflow; + + TableBitWriter() : buffer( NULL ), capacity( 0 ), bits( 0 ), overflow( false ) {} + TableBitWriter( uint8_t * to_buffer, int64_t to_capacity ) : buffer( to_buffer ), capacity( to_capacity ), bits( 0 ), overflow( false ) {} + + // ONE WORD AT A TIME, never one bit and never one byte of arithmetic: the + // value is shifted into place in a REGISTER once, and the bytes it + // occupies are stored from that register with no read back. A sixty-four + // bit field costs one shift rather than nine masked read-modify-writes. + // The word is assembled little-end-first, so the BITS ON THE WIRE are the + // same bits in the same places, bit i in byte i/8 at position i%8 with the + // low bit first, which is what the pinned goldens hold. IT WRITES EXACTLY + // THE BYTES THE VALUE OCCUPIES and never one past them, so a caller's + // buffer beyond the batch is its own. + void put( uint64_t value, int64_t n ) + { + if ( n <= 0 ) { return; } + if ( ( bits + n + 7 ) / 8 > capacity ) { overflow = true; bits += n; return; } + if ( n < 64 ) { value &= ( uint64_t( 1 ) << n ) - 1; } // a caller's high bits never leak + const int64_t index = bits >> 3; + const int64_t bit = bits & 7; + // the byte the write STARTS in keeps the bits already written to it, + // and every byte after it is this value's own + const uint64_t head = bit != 0 ? ( uint64_t( buffer[index] ) & ( ( uint64_t( 1 ) << bit ) - 1 ) ) : 0; + const uint64_t word = head | ( value << bit ); + const int64_t need = ( bit + n + 7 ) >> 3; // 1 to 9 bytes + if ( need >= 8 ) + { + table_message_store64( buffer + index, word ); + if ( need > 8 ) { buffer[index + 8] = uint8_t( value >> ( 64 - bit ) ); } + } + else + { + for ( int64_t i = 0; i < need; i++ ) { buffer[index + i] = uint8_t( word >> ( 8 * i ) ); } + } + bits += n; + } + + // THE ALIGN IS WHAT BUYS THIS (docs/SPEC-TABLES.md §3.3): a string(N), a + // bytes(N) and a blob record align before their bytes precisely so the + // largest payload on the wire moves as ONE memcpy. Off a boundary there is + // nothing to memcpy and the bytes go through put. + void putbytes( const uint8_t * data, int64_t n ) + { + if ( n <= 0 ) { return; } + if ( ( bits & 7 ) == 0 ) + { + if ( ( bits >> 3 ) + n > capacity ) { overflow = true; bits += n * 8; return; } + memcpy( buffer + ( bits >> 3 ), data, (size_t) n ); + bits += n * 8; + return; + } + for ( int64_t i = 0; i < n; i++ ) { put( (uint64_t) data[i], 8 ); } + } + + // a string's or a bytes' payload ALIGNS before its bytes, and a batch + // aligns once at its end. Both are zero fill, spent in one call. + void align() { put( 0, ( 8 - ( bits & 7 ) ) & 7 ); } +}; + +// TableAlignBits is what an align costs from a bit position, which a measure +// spends exactly where a save does. +inline int64_t TableAlignBits( int64_t bits ) { return ( 8 - ( bits % 8 ) ) % 8; } + +struct TableBitReader +{ + const uint8_t * buffer; + int64_t bits; // the stream's extent, in bits + int64_t offset; // bits consumed + + TableBitReader() : buffer( NULL ), bits( 0 ), offset( 0 ) {} + TableBitReader( const uint8_t * from_buffer, int64_t from_bytes ) : buffer( from_buffer ), bits( from_bytes * 8 ), offset( 0 ) {} + + bool has( int64_t n ) const { return n >= 0 && offset + n <= bits; } + + // the primitive is sixty-four bits, and a width above it is refused + // here as well as at the announcement: no field on any body can ask this + // reader to move more bits than it holds + // ONE WORD OF THE BUFFER AT A TIME, the mirror of the writer's put: the + // eight bytes the value starts in load as one little-end-first word and a + // ninth byte carries the spill a value that straddles the word needs. + // Within nine bytes of the stream's end there is no room for a word load + // and the bytes come one at a time, by the same arithmetic. + bool get( uint64_t & value, int64_t n ) + { + if ( n > 64 || !has( n ) ) { return false; } + value = 0; + if ( n == 0 ) { return true; } + const int64_t index = offset >> 3; + const int64_t bit = offset & 7; + const int64_t bytes = ( bits + 7 ) >> 3; + if ( index + 9 <= bytes ) + { + uint64_t v = table_message_load64( buffer + index ) >> bit; + if ( bit != 0 && bit + n > 64 ) { v |= uint64_t( buffer[index + 8] ) << ( 64 - bit ); } + value = n == 64 ? v : ( v & ( ( uint64_t( 1 ) << n ) - 1 ) ); + offset += n; + return true; + } + int64_t got = 0; + while ( got < n ) + { + const int64_t byte = offset >> 3; + const int64_t off = offset & 7; + const int64_t room = 8 - off; + const int64_t take = ( n - got ) < room ? ( n - got ) : room; + const uint64_t chunk = ( uint64_t( buffer[byte] ) >> off ) & ( ( uint64_t( 1 ) << take ) - 1 ); + value |= chunk << got; + offset += take; + got += take; + } + return true; + } + + // the bytes of an ALIGNED payload, which is the read side of the memcpy + // the align buys (docs/SPEC-TABLES.md §3.3) + bool getbytes( uint8_t * out, int64_t n ) + { + if ( n < 0 || !has( n * 8 ) ) { return false; } + if ( ( offset & 7 ) == 0 ) + { + memcpy( out, buffer + ( offset >> 3 ), (size_t) n ); + offset += n * 8; + return true; + } + for ( int64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !get( by, 8 ) ) { return false; } + out[i] = (uint8_t) by; + } + return true; + } + + bool skip( int64_t n ) { if ( !has( n ) ) { return false; } offset += n; return true; } + + // the pad to the next byte boundary is VERIFIED ZERO, which is the packet + // wire's rule for the same reason (SPEC.md §4.3) + bool align() + { + const int64_t pad = ( 8 - ( offset & 7 ) ) & 7; + if ( pad == 0 ) { return true; } + uint64_t bits_read = 0; + return get( bits_read, pad ) && bits_read == 0; + } +}; + +// TableBitsRequired is bits_required( min, max ): the bit length of max - min, +// and zero where the two are equal, which is a value that spends no bit at all. +inline int64_t TableBitsRequired( int64_t min, int64_t max ) +{ + if ( max <= min ) { return 0; } + uint64_t span = (uint64_t) ( max - min ); + int64_t n = 0; + while ( span > 0 ) { n++; span >>= 1; } + return n; +} + +// THE ANNOUNCED ENTRY (§3.3): an id, a kind, and a SHAPE, which is the width +// and range facts a reader needs to SKIP a field exactly and to DECODE one +// whose own declaration has moved. One name may take TWO entries, at two kinds or two +// shapes, and a body names the one it means. +// +// The ELEMENT's own facts ride beside the field's because an array's element +// is the one nesting this wire has: an array of arrays is not a table-wire +// construct, so one level is every level. +// +// IT IS THE RESOLVED ENTRY AND THE CALLER SIZES AN ARRAY OF THEM, so it +// carries what a DECODE takes and nothing a decode does not: qmin, qdelta and +// qcount are what SPEC.md §4.3's rule leaves behind, and the qmax and qres +// that rule CONSUMES are locals of the parse. The widths are int16 because a +// width is bounded by the kind it came under and no kind holds more than 128 +// bits. +struct TableMessageEntry +{ + uint64_t id = 0; + int64_t min = 0; // an array's minimum count + int64_t max = 0; // an array's maximum count, a string's capacity, a keyed array's slots + int64_t base_lo = 0; // the ranged base, low half: a signed kind's sign-extends, an unsigned kind's is whole + int64_t base_hi = 0; // its high half, for a 128-bit kind + int64_t elem_max = 0; + int64_t elem_base_lo = 0; + int64_t elem_base_hi = 0; + // what SPEC.md §4.3's derivation leaves: the base, the step and the count + float qmin = 0.0f; + float qdelta = 0.0f; + uint32_t qcount = 0; + float elem_qmin = 0.0f; + float elem_qdelta = 0.0f; + uint32_t elem_qcount = 0; + // THE PAYLOAD'S WIDTH, RESOLVED: what the kind, the packing and the + // announced bits together say, computed once at AnnounceRead, and -1 + // where the payload is not a fixed-width value at all + int16_t value_bits = -1; + int16_t elem_value_bits = -1; + uint8_t kind = 0; + uint8_t packing = 0; + uint8_t elem_kind = 0; + uint8_t elem_packing = 0; +}; + +// TableMessageEntrySame reports whether two RESOLVED entries carry the same +// shape, which is every fact of the entry but its id and its kind. It is what +// the announcement's duplicate rule is asked in: two entries that agree on all +// three parts are malformed (§3.3). +inline bool TableMessageEntrySame( const TableMessageEntry & a, const TableMessageEntry & b ) +{ + return a.min == b.min && a.max == b.max && a.base_lo == b.base_lo && a.base_hi == b.base_hi + && a.elem_max == b.elem_max && a.elem_base_lo == b.elem_base_lo && a.elem_base_hi == b.elem_base_hi + && a.qmin == b.qmin && a.qdelta == b.qdelta && a.qcount == b.qcount + && a.elem_qmin == b.elem_qmin && a.elem_qdelta == b.elem_qdelta && a.elem_qcount == b.elem_qcount + && a.value_bits == b.value_bits && a.elem_value_bits == b.elem_value_bits + && a.packing == b.packing && a.elem_kind == b.elem_kind && a.elem_packing == b.elem_packing; +} + +// TableMessageKindBits is the widest RANGED value a kind can carry, its own +// storage width: a width above it is a hostile width on the announcement. +inline int64_t TableMessageKindBits( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 6: case 20: case 25: return 8; + case 3: case 7: case 21: case 26: return 16; + case 4: case 8: case 22: case 27: return 32; + case 5: case 9: case 23: case 28: return 64; + default: return 128; + } +} + +// TableMessageQuantization is SPEC.md §4.3's derivation over an announced +// triple, in float32 and by nothing else: delta, the step count and the +// width. False is a triple SPEC.md calls non-conforming, which on the +// announcement is a hostile width like any other (§3.3). +inline bool TableMessageQuantization( float qmin, float qmax, float qres, float & delta, uint32_t & count, int64_t & bits ) +{ + if ( !( qmin < qmax ) || !( qres > 0.0f ) ) { return false; } + delta = qmax - qmin; + float values = delta / qres; + if ( !( delta - delta == 0.0f ) || !( values - values == 0.0f ) ) { return false; } // Inf - Inf is NaN + if ( !( values >= 1.0f ) ) { values = 1.0f; } + else if ( values > 4294967040.0f ) { values = 4294967040.0f; } // the largest float below 2^32 + count = (uint32_t) values; + if ( (float) count < values ) { count++; } // ceil, on a value the cast holds exactly + bits = TableBitsRequired( 0, (int64_t) count ); + return true; +} + +// The two roundings on each side of the rule (SPEC.md §7.2): the product +// rounds to float32 BEFORE the add, which a compiler permitted to contract +// would otherwise fuse into one rounding and move the wire. +#if ( defined( __GNUC__ ) || defined( __clang__ ) ) && ( defined( __aarch64__ ) || defined( _M_ARM64 ) ) +#define TABLE_FLOAT_FORCE_ROUND( x ) __asm__ ( "" : "+w" ( x ) ) +#elif ( defined( __GNUC__ ) || defined( __clang__ ) ) && ( defined( __x86_64__ ) || defined( __i386__ ) ) +#define TABLE_FLOAT_FORCE_ROUND( x ) __asm__ ( "" : "+x" ( x ) ) +#else +#define TABLE_FLOAT_FORCE_ROUND( x ) do { volatile float table_float_force_round_slot = ( x ); ( x ) = table_float_force_round_slot; } while ( 0 ) +#endif + +// TableMessageQuantize is the writer's half: the index a value takes. +inline uint32_t TableMessageQuantize( float value, float qmin, float delta, uint32_t count ) +{ + float normalized = ( value - qmin ) / delta; + if ( !( normalized >= 0.0f ) ) { normalized = 0.0f; } + else if ( !( normalized <= 1.0f ) ) { normalized = 1.0f; } + float scaled = normalized * (float) count; + TABLE_FLOAT_FORCE_ROUND( scaled ); + uint32_t index = (uint32_t) ( scaled + 0.5f ); // floor of a non-negative value + if ( index > count ) { index = count; } + return index; +} + +// TableMessageDequantize is the reader's half: the float an index names. +inline float TableMessageDequantize( uint32_t index, float qmin, float delta, uint32_t count ) +{ + if ( index > count ) { index = count; } + const float normalized = index / (float) count; + float scaled = normalized * delta; + TABLE_FLOAT_FORCE_ROUND( scaled ); + return scaled + qmin; +} + +inline bool TableMessageIntegerKind( uint8_t kind ) +{ + return ( kind >= 2 && kind <= 9 ) || kind == 18 || kind == 19; +} + +inline bool TableMessageFixedKind( uint8_t kind ) { return kind >= 20 && kind <= 29; } + +inline bool TableMessageKnownKind( uint8_t kind ) +{ + return kind == 0 || ( kind >= 1 && kind <= 17 ) || ( kind >= 18 && kind <= 29 ) || ( kind >= 30 && kind <= 33 ); +} + +// A CANONICAL LEB128, which is the announcement's own integer: the +// announcement is a form 1 FILE and takes §3's rule. +inline bool TableMessageLeb( const uint8_t * in, int64_t size, int64_t & at, uint64_t & value ) +{ + value = 0; + for ( int64_t shift = 0; at < size; shift += 7 ) + { + if ( shift >= 64 ) { return false; } + const uint8_t by = in[ at++ ]; + value |= uint64_t( by & 0x7F ) << shift; + if ( ( by & 0x80 ) == 0 ) { return !( shift > 0 && by == 0 ); } + } + return false; +} + +// TableMessageShapeFacts is where one shape's facts land: the field's own, +// or its element's, which is the one nesting this wire has. +struct TableMessageShapeFacts +{ + uint8_t & packing; int64_t & value_bits; int64_t & base_lo; int64_t & base_hi; + float & qmin; float & qmax; float & qres; float & qdelta; uint32_t & qcount; + int64_t & min; int64_t & max; uint8_t & elem_kind; +}; + +inline bool TableMessageShapeRead( const uint8_t * in, int64_t size, int64_t & at, uint8_t kind, TableMessageShapeFacts f ); +inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t value_bits ); + +// TableMessageEntryRead parses ONE entry, and answers false for a HOSTILE +// SHAPE: bits above the kind's own domain, an array whose min exceeds its +// max, an element kind outside the closed set, a quantized triple SPEC.md +// calls non-conforming, or a shape running past the vocabulary's own bytes. +inline bool TableMessageEntryRead( const uint8_t * in, int64_t size, int64_t & at, TableMessageEntry & entry ) +{ + if ( at + 9 > size ) { return false; } + entry = TableMessageEntry(); + for ( int i = 0; i < 8; i++ ) { entry.id |= uint64_t( in[ at + i ] ) << ( 8 * i ); } + entry.kind = in[ at + 8 ]; + at += 9; + if ( !TableMessageKnownKind( entry.kind ) ) { return false; } + // The parse lands in LOCALS and the entry keeps what a decode reads: the + // quantized max and res are the derivation's inputs and never a field's. + uint8_t packing = 0, elem_kind = 0; + int64_t bits = 0, base_lo = 0, base_hi = 0, min = 0, max = 0; + float qmin = 0.0f, qmax = 0.0f, qres = 0.0f, qdelta = 0.0f; + uint32_t qcount = 0; + TableMessageShapeFacts own = { packing, bits, base_lo, base_hi, + qmin, qmax, qres, qdelta, qcount, + min, max, elem_kind }; + if ( !TableMessageShapeRead( in, size, at, entry.kind, own ) ) { return false; } + entry.packing = packing; + entry.value_bits = (int16_t) TableMessageValueBits( entry.kind, packing, bits ); + entry.base_lo = base_lo; + entry.base_hi = base_hi; + entry.qmin = qmin; + entry.qdelta = qdelta; + entry.qcount = qcount; + entry.min = min; + entry.max = max; + entry.elem_kind = elem_kind; + if ( entry.kind == 14 || entry.kind == 16 ) + { + uint8_t elem_packing = 0, inner_kind = 0; + int64_t elem_bits = 0, elem_base_lo = 0, elem_base_hi = 0, elem_min = 0, elem_max = 0; + float elem_qmin = 0.0f, elem_qmax = 0.0f, elem_qres = 0.0f, elem_qdelta = 0.0f; + uint32_t elem_qcount = 0; + TableMessageShapeFacts elem = { elem_packing, elem_bits, elem_base_lo, elem_base_hi, + elem_qmin, elem_qmax, elem_qres, elem_qdelta, elem_qcount, + elem_min, elem_max, inner_kind }; + if ( !TableMessageShapeRead( in, size, at, entry.elem_kind, elem ) ) { return false; } + entry.elem_packing = elem_packing; + entry.elem_value_bits = (int16_t) TableMessageValueBits( entry.elem_kind, elem_packing, elem_bits ); + entry.elem_base_lo = elem_base_lo; + entry.elem_base_hi = elem_base_hi; + entry.elem_qmin = elem_qmin; + entry.elem_qdelta = elem_qdelta; + entry.elem_qcount = elem_qcount; + entry.elem_max = elem_max; + } + return true; +} + +// TableMessageShapeRead is one shape, by the kind that names it (§3.3's shape +// table). Every number in it is a canonical LEB128 except where the row says +// otherwise: a RANGED BASE IS ENCODED BY ITS KIND'S SIGNEDNESS, zigzag for the +// signed kinds, unsigned for the unsigned kinds and sixteen bytes for the +// 128-bit and fixed-point kinds, and a QUANTIZED f32 carries min, max and res +// as float32, from which the step count and the width derive by SPEC.md +// §4.3's rule and by nothing else. +inline bool TableMessageShapeRead( const uint8_t * in, int64_t size, int64_t & at, uint8_t kind, TableMessageShapeFacts f ) +{ + uint64_t v = 0; + if ( TableMessageIntegerKind( kind ) || TableMessageFixedKind( kind ) || kind == 10 ) + { + if ( at >= size ) { return false; } + f.packing = in[ at++ ]; + if ( f.packing == 0 ) { return true; } + if ( f.packing == 1 && kind != 10 ) + { + if ( !TableMessageLeb( in, size, at, v ) || (int64_t) v > TableMessageKindBits( kind ) ) { return false; } + f.value_bits = (int64_t) v; + if ( kind == 18 || kind == 19 || TableMessageFixedKind( kind ) ) + { + if ( at + 16 > size ) { return false; } + uint64_t lo = 0, hi = 0; + for ( int i = 0; i < 8; i++ ) { lo |= uint64_t( in[ at + i ] ) << ( 8 * i ); } + for ( int i = 0; i < 8; i++ ) { hi |= uint64_t( in[ at + 8 + i ] ) << ( 8 * i ); } + f.base_lo = (int64_t) lo; f.base_hi = (int64_t) hi; + at += 16; + return true; + } + if ( !TableMessageLeb( in, size, at, v ) ) { return false; } + if ( kind >= 2 && kind <= 5 ) { f.base_lo = (int64_t) ( v >> 1 ) ^ -(int64_t) ( v & 1 ); } // zigzag + else { f.base_lo = (int64_t) v; } // the unsigned domain, whole + return true; + } + if ( f.packing == 2 && kind == 10 ) + { + if ( at + 12 > size ) { return false; } + uint32_t raw[3] = { 0, 0, 0 }; + for ( int k = 0; k < 3; k++ ) { for ( int i = 0; i < 4; i++ ) { raw[k] |= uint32_t( in[ at + 4 * k + i ] ) << ( 8 * i ); } } + at += 12; + memcpy( &f.qmin, &raw[0], 4 ); + memcpy( &f.qmax, &raw[1], 4 ); + memcpy( &f.qres, &raw[2], 4 ); + return TableMessageQuantization( f.qmin, f.qmax, f.qres, f.qdelta, f.qcount, f.value_bits ); + } + return false; // a packing outside the closed set + } + // A MAX ABOVE WHAT THE KIND CAN HOLD IS A HOSTILE WIDTH (§3.3). A string + // and a wide string are bounded by the int32 storage cap the checker + // applies to every N (SPEC §4.3, §6.1), and an array and a keyed entry by + // the 32-bit count an unbounded array announces (§2.9), which is the + // widest count this form spells. A larger bound is a shape no conforming + // declaration can produce, and a reader that carried it would do its + // length arithmetic in a range that overflows. + if ( kind == 12 || kind == 33 ) + { + if ( !TableMessageLeb( in, size, at, v ) || v > (uint64_t) INT32_MAX ) { return false; } + f.max = (int64_t) v; + return true; + } + if ( kind == 14 || kind == 16 ) + { + if ( kind == 14 ) + { + if ( !TableMessageLeb( in, size, at, v ) || v > kTableMessageListMax ) { return false; } + f.min = (int64_t) v; + } + if ( !TableMessageLeb( in, size, at, v ) || v > kTableMessageListMax ) { return false; } + if ( (int64_t) v < f.min ) { return false; } + f.max = (int64_t) v; + if ( at >= size ) { return false; } + f.elem_kind = in[ at++ ]; + if ( !TableMessageKnownKind( f.elem_kind ) ) { return false; } + // AND AN ELEMENT KIND OF 12 OR 33 IS REFUSED HERE, at the + // announcement, rather than at the skip that would meet it (§3.3): no + // declaration this language accepts is an array of string(N) or of + // wstring(N), so a shape announcing one is one rule's business and not + // two. + if ( f.elem_kind == 12 || f.elem_kind == 33 ) { return false; } + return true; + } + return true; +} + +// TableMessageValueBits is one value's width under a shape, and -1 where the +// kind's payload is not a fixed-width value at all. +inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t value_bits ) +{ + if ( kind == 1 ) { return 1; } + if ( kind == 11 ) { return 64; } + if ( kind == 10 ) { return packing == 2 ? value_bits : 32; } + if ( TableMessageIntegerKind( kind ) || TableMessageFixedKind( kind ) ) + { + if ( packing == 1 ) { return value_bits; } + switch ( kind ) + { + case 2: case 6: case 20: case 25: return 8; + case 3: case 7: case 21: case 26: return 16; + case 4: case 8: case 22: case 27: return 32; + case 5: case 9: case 23: case 28: return 64; + default: return 128; + } + } + return -1; +} + +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an +// ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under +// the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 +// over element kind 6, and a trailer of those two reserved ids. +// +// THE VOCABULARY IS A FIELD AND NOT THE TRAILER, and that buys three things: +// §3's writer rule that an id no body references is never written is restored +// unbroken, an entry can carry a KIND and a SHAPE which a trailer of bare ids +// cannot, and one NAME can appear at two shapes. +// +// The order is the COOK PROJECTION's (§20.2): each record in the order the +// projection renders it and each record's fields in the order the projection +// renders them, then each enum's variants and each union's arms. Then comes +// the tail the projection does not name: the reserved node-table id, the three +// blob type ids as bytes, string and wstring, and every table's own name id in +// the projection's sorted record order. The tail is UNCONDITIONAL, so an +// ordinary edit only ever grows it at its end and never moves a slot a +// generated field header carries as a literal. +static const int64_t kTableAnnounceBytes = 901; +static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, + 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, + 0xe4, 0x7c, 0x11, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, + 0x20, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0d, 0xec, 0x10, + 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, + 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, + 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, + 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, + 0x07, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x21, 0x06, + 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0c, 0x10, 0x38, 0x81, + 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, + 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, +}; + +// TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries +// an announcement carried, RESOLVED ONCE, under one numbering. +// +// THE RECEIVER RESOLVES ONCE (§3.3), so this holds the entries themselves and +// not the announcement's bytes: every entry is parsed at AnnounceRead, and +// every body after it dispatches through ONE ARRAY INDEX with nothing to +// re-read and nothing to decide. The announcement is free the moment +// AnnounceRead returns. +// +// THE STORAGE IS THE CALLER'S and this library never allocates. The caller +// declares an array of entries wherever it wants it, static, on a heap, in an +// arena or beside its connection, and hands it here with its CAPACITY. The +// announcement holds for the life of the connection (§3.3), so the array does +// too, and a peer holds TWO for a connection, the one it writes with and the +// one it reads with. A restart opens a fresh connection with an empty +// vocabulary and nothing is cached across connections. +// +// kTableMessageEntriesHere is the capacity a receiver that talks only to peers +// of THIS schema declares, and a receiver meeting other builds declares more. +struct TableVocabulary +{ + // THE CONFORMING DEFAULT BYTE BOUND (§3.3). The ENTRY bound has no default + // because it IS the caller's capacity: an announcement naming more entries + // than the caller made room for is refused as vocabulary_too_large before + // an entry is touched, and the byte bound is read off the vocabulary + // field's own length before that. + static const int64_t kDefaultMaxBytes = 64 * 1024; + + TableVocabulary( TableMessageEntry * storage, int64_t capacity ) + : entries( storage ), max_entries( capacity ) {} + + TableMessageEntry * entries; // THE CALLER'S, capacity max_entries + int64_t max_entries; + int64_t count = 0; + int64_t ref_bits = 0; + uint64_t build_version = 0; + bool announced = false; + // REFUSAL IS TERMINAL (§3.3): a connection whose first announcement was + // refused, for any reason, carries no vocabulary for its life, and every + // announcement after it is refused as second_announcement + bool refused = false; + int64_t max_bytes = kDefaultMaxBytes; +}; + +// TableVocabularyEntryAt is the entry a reference names, counted from 1: ONE +// ARRAY INDEX into the caller's resolved storage, no parse and no branch. +inline const TableMessageEntry & TableVocabularyEntryAt( const TableVocabulary & vocabulary, uint64_t slot ) +{ + return vocabulary.entries[ slot - 1 ]; +} + +// AnnounceRead reads an announcement into one direction's vocabulary (§3.3). +// +// The announcement IS a file, so every malformed rule of §3 already covers it. +// Over its body there are EXACTLY TWO STRICT CHECKS: the BUILD VERSION +// present, exactly once, under kind 9, eight bytes wide, and the VOCABULARY +// present, exactly once, under kind 14 over element kind 6. Everything else is +// ordinary and tolerant, so an unknown field is skipped and counted and the +// announcement can GAIN a field in a later minor without a lockstep redeploy. +// +// The FIRST announcement sets the vocabulary and it is the only one that can. +// A SECOND is refused by name: it does not replace it, does not amend it and +// changes nothing. A refused announcement sets NO VOCABULARY, and the refusal +// is TERMINAL: every announcement after it, whether or not the first set +// anything, is second_announcement, so a peer holds no retry on the +// connection and cannot buy a second resolve by having its first refused. +inline bool AnnounceReadOnce( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * to ); + +inline bool AnnounceRead( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * to = report != NULL ? report : &ignored; + if ( vocabulary.announced || vocabulary.refused ) + { + to->refused = true; + to->reason = second_announcement; + return false; + } + const bool set = AnnounceReadOnce( vocabulary, buffer, bytes, to ); + if ( !set ) { vocabulary.refused = true; } + return set; +} + +inline bool AnnounceReadOnce( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * to ) +{ + if ( bytes < 1 ) { to->malformed = true; return false; } + if ( buffer[0] != kTableWireForm ) + { + to->refused = true; + to->reason = buffer[0] == kTableWireMessageForm ? message_form_as_file : newer_form; + return false; + } + if ( bytes < 9 ) { to->malformed = true; return false; } + TableIdTable table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( buffer, bytes, table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { to->malformed = true; } + else { to->refused = true; to->reason = newer_form; } + return false; + } + if ( TableBodyEndsEarly( buffer + 1, body_bytes, table ) ) { to->malformed = true; return false; } + TableReader r( buffer + 1, body_bytes, to, &table ); + uint64_t version = 0; + const uint8_t * words = NULL; + int64_t words_bytes = 0; + int32_t seen_version = 0, seen_vocabulary = 0; + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.getleb( ref ) ) { to->malformed = true; return false; } + if ( ref == 0 ) { break; } + if ( ref > (uint64_t) table.count || !r.has( 1 ) ) { to->malformed = true; return false; } + const uint64_t id = table.at( ref ); + const uint8_t kind = r.get8(); + if ( id == kTableBuildVersionFieldId ) + { + if ( kind != 9 || !r.has( 8 ) ) { to->malformed = true; return false; } + version = r.get64(); + // THE BUILD VERSION IS KEPT THE MOMENT IT IS READ, refusal or not, so + // that a refusal on this connection NAMES IT (§3.3). It is not the + // vocabulary, and a refused announcement still sets none. + vocabulary.build_version = version; + seen_version++; + continue; + } + if ( id == kTableMessageVocabularyFieldId ) + { + // kind 14 over element kind 6, which is §3's spelling for an + // opaque run of bytes + uint64_t framed = 0; + if ( kind != 14 || !r.getleb( framed ) || !r.has( (int64_t) framed ) ) { to->malformed = true; return false; } + const int64_t begin = r.offset, end = r.offset + (int64_t) framed; + r.offset = end; + if ( begin >= end || r.buffer[ begin ] != 6 ) { to->malformed = true; return false; } + int64_t at = begin + 1; + uint64_t length = 0; + if ( !TableMessageLeb( r.buffer, end, at, length ) || at + (int64_t) length != end ) { to->malformed = true; return false; } + if ( (int64_t) length > vocabulary.max_bytes ) { to->refused = true; to->reason = vocabulary_too_large; return false; } + words = r.buffer + at; + words_bytes = (int64_t) length; + seen_vocabulary++; + continue; + } + to->unknown++; + if ( !r.skip( kind ) ) { to->malformed = true; return false; } + } + if ( seen_version != 1 || seen_vocabulary != 1 ) { to->malformed = true; return false; } + + // THE ENTRIES, RESOLVED ONCE into the caller's storage (§3.3): every width + // is checked here and never again, and no body after this parses a byte of + // an announcement. An entry count above the caller's CAPACITY is refused + // by name before the entry is touched. + int64_t at = 0, count = 0, node_table_slots = 0; + while ( at < words_bytes ) + { + if ( count >= vocabulary.max_entries ) { to->refused = true; to->reason = vocabulary_too_large; return false; } + TableMessageEntry & parsed = vocabulary.entries[ count ]; + if ( !TableMessageEntryRead( words, words_bytes, at, parsed ) ) { to->malformed = true; return false; } + // THE RESERVED IDS WHERE THEY DO NOT BELONG (§3.3): the announcement's + // own two never take a slot, and the node-table id takes exactly one, + // so a vocabulary carrying either of the first or a SECOND node-table + // id is malformed whole and sets nothing + if ( parsed.id == kTableBuildVersionFieldId || parsed.id == kTableMessageVocabularyFieldId ) { to->malformed = true; return false; } + if ( parsed.id == kTableNodeTableFieldId ) { if ( node_table_slots++ > 0 ) { to->malformed = true; return false; } } + // A TRIPLE ALREADY PLACED IS NEVER PLACED TWICE, so two entries that + // agree on the id, the kind and every fact of the shape are malformed + // (§3.3): no writer this wire has produces one, and a reader that took + // it would carry two slots naming one thing. The scan is quadratic in + // the entry count, and the entry count is bounded above at 4096, so it + // is at most eight million compares on a path that runs ONCE a + // connection and never again. + for ( int64_t seen = 0; seen < count; seen++ ) + { + const TableMessageEntry & other = vocabulary.entries[ seen ]; + if ( other.id == parsed.id && other.kind == parsed.kind && TableMessageEntrySame( other, parsed ) ) { to->malformed = true; return false; } + } + count++; + } + vocabulary.count = count; + vocabulary.ref_bits = TableBitsRequired( 0, count ); + vocabulary.build_version = version; + vocabulary.announced = true; + return true; +} + +// TableMessageReserved is one of the three ids the language holds back (§3.1, +// §3.3, §5): each is malformed anywhere but its own transport, and the rule +// OUTRANKS the wrong-sort rule below. +inline bool TableMessageReserved( uint64_t id ) +{ + // THE THREE ARE THE TOP THREE VALUES a uint64 holds, so the test is ONE + // comparison: 0xFFFFFFFFFFFFFFFD, FE and FF and nothing else is at or + // above the vocabulary's own id, and a declaration hashing to any of them + // is refused by name (§11) + return id >= kTableMessageVocabularyFieldId; +} + +// TableMessageNameEntry resolves a reference used as a VALUE, which is an +// enum's variant, a keyed array's slot key or a node record's type id, and +// which must name a kind-0 entry (§3.3). A reference of 0 where an entry is required, one +// above E, one naming a reserved id and one naming an entry that carries a +// payload are each damage: the reader RESOLVED the entry and it contradicts +// the position it was used in, so the next bit's meaning is what is in doubt. +inline bool TableMessageNameEntry( const TableVocabulary & vocabulary, uint64_t ref, TableMessageEntry & entry ) +{ + if ( ref == 0 || ref > (uint64_t) vocabulary.count ) { return false; } + entry = TableVocabularyEntryAt( vocabulary, ref ); + return !TableMessageReserved( entry.id ) && entry.kind == 0; +} + +// TableMessageArmEntry resolves a UNION's arm reference, which must name an +// entry carrying the arm's own kind and shape: a kind-0 entry frames nothing, +// and a reserved id belongs to no arm (§3.3). +inline bool TableMessageArmEntry( const TableVocabulary & vocabulary, uint64_t ref, TableMessageEntry & entry ) +{ + if ( ref == 0 || ref > (uint64_t) vocabulary.count ) { return false; } + entry = TableVocabularyEntryAt( vocabulary, ref ); + return !TableMessageReserved( entry.id ) && entry.kind != 0; +} + +// TableMessageSkipVariant steps over an ENUM's variant reference on a SKIP +// path and RESOLVES it while it is there: 0 is None and the whole payload, and +// every other reference must name a kind-0 entry, because every reference +// above E is damage and one naming an entry that carries a payload +// contradicts the position it was used in, whether or not this reader was +// going to keep the value (§3.3). +inline bool TableMessageSkipVariant( TableBitReader & r, const TableVocabulary & vocabulary ) +{ + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + TableMessageEntry named; + return TableMessageNameEntry( vocabulary, ref, named ); +} +// TableMessageSkip steps over one field's payload without decoding it, using +// the announced ENTRY alone (§3.3). It is what makes an unknown entry +// skippable on a body with no kind byte, and it is ONE function over every +// table, because a shape says everything a skipper needs. +inline bool TableMessageSkipBody( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits ); +inline bool TableMessageSkip( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ); + +// TableMessageSkipElement steps over ONE element of an array or keyed entry +// by the element's own announced shape: a nested body to its zero reference, +// a variant or a node index at its reference width, a union arm by its own +// entry, and a fixed-width value at its bits. +inline bool TableMessageSkipElement( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ) +{ + switch ( entry.elem_kind ) + { + case 13: return TableMessageSkipBody( r, vocabulary, index_bits ); + case 30: return TableMessageSkipVariant( r, vocabulary ); + case 17: return index_bits > 0 && r.skip( index_bits ); + case 15: + { + TableMessageEntry inner; + inner.kind = 15; + return TableMessageSkip( r, vocabulary, index_bits, inner ); + } + default: + { + const int64_t elem = entry.elem_value_bits; + return elem >= 0 && r.skip( elem ); + } + } +} + +// TableMessageElementRunBits is the bits ONE element of an array or a keyed +// entry occupies on the SKIP path, where nothing is resolved and a run of them +// is one multiplication, and -1 where the element's width is its own +// content's. A ZERO is a real answer, and it is why this exists: a ranged +// element whose min equals its max rides no bits at all (§3.3). +inline int64_t TableMessageElementRunBits( const TableVocabulary & vocabulary, const TableMessageEntry & entry ) +{ + int64_t elem = 0; + switch ( entry.elem_kind ) + { + // a nested body, a union arm, an enum's variant and a node index each + // RESOLVE something, and a resolve that contradicts its position is + // damage this reader must still find, so they are walked + case 13: case 15: case 30: case 17: return -1; + default: elem = entry.elem_value_bits; break; + } + if ( elem < 0 ) { return -1; } + if ( entry.kind == 16 ) { elem += vocabulary.ref_bits; } // a keyed slot's own key reference + return elem; +} + +// TableMessageSkipRun steps over n elements of one fixed width in a single +// arithmetic step. A FIXED-WIDTH ELEMENT IS ARITHMETIC (§3.3), and a loop here +// would be the one superlinear thing in this form: a zero-width element under +// a count of 2^31 is six bytes of wire. +inline bool TableMessageSkipRun( TableBitReader & r, uint64_t n, int64_t width ) +{ + if ( width < 0 ) { return false; } + if ( width == 0 ) { return true; } + if ( n > (uint64_t) ( INT64_MAX / width ) ) { return false; } + return r.skip( (int64_t) ( n * (uint64_t) width ) ); +} + +inline bool TableMessageSkip( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ) +{ + switch ( entry.kind ) + { + case 0: case 32: return true; // a name, and a payload-free arm + case 30: return TableMessageSkipVariant( r, vocabulary ); + case 13: return TableMessageSkipBody( r, vocabulary, index_bits ); + case 17: return index_bits > 0 && r.skip( index_bits ); // a node index, at the width the body's node count settled + case 15: + { + uint64_t arm = 0; + if ( !r.get( arm, vocabulary.ref_bits ) ) { return false; } + if ( arm == 0 ) { return true; } + TableMessageEntry arm_entry; + if ( !TableMessageArmEntry( vocabulary, arm, arm_entry ) ) { return false; } + return TableMessageSkip( r, vocabulary, index_bits, arm_entry ); + } + case 12: + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() ) { return false; } + return r.skip( (int64_t) n * 8 ); + } + case 33: + { + // the length, NO align, then SIXTEEN bits a code unit (§3.3) + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + return r.skip( (int64_t) n * 16 ); + } + case 31: + { + // THE ESCAPE: align, a thirty-two bit L, then L bytes, opaque. It is + // the one path a later-major writer has on this form (§3.3) + uint64_t n = 0; + if ( !r.align() || !r.get( n, 32 ) ) { return false; } + return r.skip( (int64_t) n * 8 ); + } + case 14: case 16: + { + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, width ) ) { return false; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !r.align() ) { return false; } + // A RUN OF FIXED-WIDTH ELEMENTS IS ONE MULTIPLICATION (§3.3), and + // only an element whose width is its own content's is walked + const int64_t run = TableMessageElementRunBits( vocabulary, entry ); + if ( run >= 0 ) { return TableMessageSkipRun( r, n, run ); } + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 && !r.skip( vocabulary.ref_bits ) ) { return false; } + if ( !TableMessageSkipElement( r, vocabulary, index_bits, entry ) ) { return false; } + } + return true; + } + } + const int64_t width = entry.value_bits; + return width >= 0 && r.skip( width ); +} + +inline bool TableMessageSkipBody( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + if ( !TableMessageSkip( r, vocabulary, index_bits, TableVocabularyEntryAt( vocabulary, ref ) ) ) { return false; } + } +} + +// TableMessageNodeTableOpen reads the node table's opening when a body has +// one: the reserved id's reference and the count at thirty-two raw bits. A +// body whose first reference is anything else has no node table, and the +// reader is left where it was. False is damage: a reference past E, or bits +// that run out. +inline bool TableMessageNodeTableOpen( TableBitReader & r, const TableVocabulary & vocabulary, int64_t & count ) +{ + count = 0; + const int64_t at = r.offset; + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { r.offset = at; return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + if ( TableVocabularyEntryAt( vocabulary, ref ).id != kTableNodeTableFieldId ) { r.offset = at; return true; } + uint64_t n = 0; + if ( !r.get( n, 32 ) ) { return false; } + count = (int64_t) n; + return true; +} +// AnnounceMeasure is the announcement's byte count, which is a constant of the +// unit and not a walk. +inline int64_t AnnounceMeasure() { return kTableAnnounceBytes; } + +// Announce writes the announcement into the caller's buffer and answers the +// bytes written, which is exactly AnnounceMeasure's answer, or -1 when the +// buffer is too small. It allocates nothing and walks nothing. +inline int64_t Announce( uint8_t * buffer, int64_t capacity ) +{ + if ( buffer == NULL || capacity < kTableAnnounceBytes ) { return -1; } + memcpy( buffer, kTableAnnounce, (size_t) kTableAnnounceBytes ); + return kTableAnnounceBytes; +} + +// THE PRIMITIVE IS A BATCH (§3.3): a number of bodies of ONE ROOT in one +// buffer, one count and one continuous bit stream with no alignment between +// them. A single message is the batch of one. +// +// The count rides ahead of the bodies, so a writer declares it at Begin and +// End refuses a batch that wrote a different number: a count the bodies do not +// match is not a wire this writer will hand anyone. +struct TableMessageBatch +{ + TableBitWriter w; + int64_t declared = 0; + int64_t written = 0; +}; + +inline bool TableMessageBatchBegin( TableMessageBatch & batch, uint8_t * buffer, int64_t capacity, int64_t bodies ) +{ + if ( buffer == NULL || capacity < 1 || bodies < 1 || bodies > kTableMessageBatchMax ) { return false; } + buffer[0] = kTableWireMessageForm; // the FORM BYTE is read first, always + batch.w = TableBitWriter( buffer + 1, capacity - 1 ); + batch.declared = bodies; + batch.written = 0; + batch.w.put( (uint64_t) ( bodies - 1 ), 8 ); // a ranged integer over [1, 256] + return true; +} + +// TableMessageBatchEnd zero-fills to the next byte, the one alignment a batch +// spends at its end, and answers the whole batch's byte count, or -1. +inline int64_t TableMessageBatchEnd( TableMessageBatch & batch ) +{ + if ( batch.written != batch.declared || batch.w.overflow ) { return -1; } + batch.w.align(); + if ( batch.w.overflow ) { return -1; } + return 1 + batch.w.bits / 8; +} + +// TableMessageBatchBytes is a batch's byte count from its bodies' BIT count, +// which is what every MeasureMessages answers. +inline int64_t TableMessageBatchBytes( int64_t body_bits ) +{ + if ( body_bits < 0 ) { return -1; } + return 1 + ( 8 + body_bits + 7 ) / 8; +} + +// The reading half. A batch is opened once and its bodies are then read in +// order into the storage the caller sized for them: which root a batch carries +// is the APPLICATION's and never this wire's. +struct TableMessageBatchReader +{ + TableBitReader r; + const TableVocabulary * vocabulary = NULL; + TableReport * report = NULL; + int64_t remaining = 0; + // THE SINK A CALLER THAT PASSED NO REPORT WRITES INTO IS THE READER'S OWN, + // not a static: a static is shared mutable state, and two threads reading + // two batches without reports would be writing one object. LoadMessages + // already keeps its sink locally, for the same reason. + TableReport ignored; +}; + +// TableMessageBatchOpen answers the batch's body count, or -1 with the refusal +// on the report: a form byte this reader does not carry, or a body from a peer +// that never announced. +inline int64_t TableMessageBatchOpen( TableMessageBatchReader & br, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + br.report = report != NULL ? report : &br.ignored; + br.vocabulary = &vocabulary; + if ( bytes < 1 ) { br.report->malformed = true; return -1; } + if ( buffer[0] != kTableWireMessageForm ) { br.report->refused = true; br.report->reason = newer_form; return -1; } + if ( !vocabulary.announced ) { br.report->refused = true; br.report->reason = no_vocabulary; return -1; } + br.r = TableBitReader( buffer + 1, bytes - 1 ); + uint64_t count = 0; + if ( !br.r.get( count, 8 ) ) { br.report->malformed = true; return -1; } + br.remaining = (int64_t) count + 1; + return br.remaining; +} + +// TableMessageRefuseBatch is the batch's own refusal (§3.3): M above 256 on the +// write side, or above the caller's capacity on the read side. Nothing is +// written or decoded, no counter moves, and the reason names it. +inline void TableMessageRefuseBatch( TableReport * report ) +{ + if ( report == NULL ) { return; } + report->refused = true; + report->reason = batch_too_large; +} + +// TableMessageBatchClose verifies the trailing pad, and that NOTHING FOLLOWS +// IT: the batch ends at the pad to the byte boundary, and a buffer with bytes +// left over describes no batch this reader can name (§3.3). +inline bool TableMessageBatchClose( TableMessageBatchReader & br ) +{ + if ( br.remaining != 0 || !br.r.align() || br.r.offset != br.r.bits ) { br.report->malformed = true; return false; } + return true; +} + + +// An ENUM-KEYED array's storage: E.Max slots, ONE PER NAMED VARIANT, with the +// key k at index k-1 — the storage SHIFTS LEFT and nothing is stored for None. +// +// NOTHING OUTSIDE THE ARRAY NAMES ITS SIZE: the extent is derived from E::Max +// here and nowhere else, so there is no size parameter to spell and no count a +// consumer could put one out of step with. +// +// NONE IS THE NULL KEY: it names no slot, it never rides on the wire, a stored +// key of 0 is malformed, and INDEXING BY IT IS A PROGRAM ERROR IN EVERY +// CONFIGURATION — caught by operator[], which cannot see a runtime key any +// earlier, and REFUSED UNCONDITIONALLY. A KEY PAST Max IS THE SAME ERROR for +// the same reason — it names a variant this enum does not have — so the +// accessor refuses BOTH ENDS. NDEBUG does not remove the compare: +// there is NO UB PATH here in any build. ITERATION is still the surface a +// consumer of the whole array wants: begin()/end() walk every stored slot and +// yield the KEY, 1..E.Max, so a call site writes no bound, no cast, no shift +// and no None question. +template +struct TableKeyed +{ + // the extent is the enum's, derived here and named nowhere else + static constexpr int32_t kSlots = (int32_t) E::Max; + + T slots[kSlots] = {}; + + T & operator[]( E key ) + { + RefuseKey( key ); + return slots[ (int32_t) key - 1 ]; + } + const T & operator[]( E key ) const + { + RefuseKey( key ); + return slots[ (int32_t) key - 1 ]; + } + + // THE REFUSAL, and it stands in EVERY BUILD, AT BOTH ENDS. The storage + // holds one slot per NAMED variant: nothing for None below it and nothing + // above Max, so a build that skipped this compare would index one element + // BEFORE the array or past its end — undefined behavior in the + // configuration a game ships. Either key is a program error, so the + // accessor ends the program rather than reading something. The assert + // carries the message where a debugger can read it and NDEBUG removes + // that; the fatal is what stands after it. BOTH GO THROUGH THE HOOKS — + // define schema_assert and schema_fatal and this refusal lands in your + // own handler. + // + // ONE UNSIGNED COMPARE COVERS BOTH ENDS: the storage index is key - 1, and + // None's is -1, which wraps above kSlots unsigned. The cost is one + // perfectly-predicted compare, on a path that reads config. + static void RefuseKey( E key ) + { + if ( (uint32_t) ( (int32_t) key - 1 ) >= (uint32_t) kSlots ) + { + schema_assert( false && "an enum-keyed array holds one slot per named variant: None keys none, and neither does a key past Max" ); + schema_fatal(); + } + } + + // ---- iteration: keys 1..E.Max over storage 0..E.Max-1, key beside element ---- + // + // The entry is a key and a REFERENCE, handed out BY VALUE the way any + // proxy is: for ( auto [ key, element ] : keyed ) binds element to the + // reference member, so iterating fills the array as well as reads it. + // auto & [ key, element ] does NOT compile, and that is by design — a + // non-const lvalue reference cannot bind to the proxy. Write + // auto [ ... ], or auto && [ ... ] if you prefer the reference form. + // + // THE ITERATORS CARRY NO iterator_traits TYPEDEFS. They bought std::distance + // and the forward-pass algorithms for an audience that does not call them, + // and the they need is the single most expensive include the + // generated corpus had: 536 headers and 986 KB, in a header whose whole + // remaining set is 123. begin(), end() and size() need none of it. + + struct Entry { E key; T & element; }; + struct ConstEntry { E key; const T & element; }; + + struct Iterator + { + T * slots; + int32_t index; // the STORAGE index; the key it holds is index + 1 + Entry operator*() const { return Entry{ (E) ( index + 1 ), slots[index] }; } + Iterator & operator++() { index++; return *this; } + bool operator==( const Iterator & other ) const { return index == other.index; } + bool operator!=( const Iterator & other ) const { return index != other.index; } + }; + + struct ConstIterator + { + const T * slots; + int32_t index; // the STORAGE index; the key it holds is index + 1 + ConstEntry operator*() const { return ConstEntry{ (E) ( index + 1 ), slots[index] }; } + ConstIterator & operator++() { index++; return *this; } + bool operator==( const ConstIterator & other ) const { return index == other.index; } + bool operator!=( const ConstIterator & other ) const { return index != other.index; } + }; + + Iterator begin() { return Iterator{ slots, 0 }; } + Iterator end() { return Iterator{ slots, kSlots }; } + ConstIterator begin() const { return ConstIterator{ slots, 0 }; } + ConstIterator end() const { return ConstIterator{ slots, kSlots }; } +}; + +inline float table_bits_to_float( uint32_t bits ) { float f; memcpy( &f, &bits, 4 ); return f; } +inline uint32_t table_float_to_bits( float f ) { uint32_t b; memcpy( &b, &f, 4 ); return b; } +inline double table_bits_to_double( uint64_t bits ) { double d; memcpy( &d, &bits, 8 ); return d; } +inline uint64_t table_double_to_bits( double d ) { uint64_t b; memcpy( &b, &d, 8 ); return b; } + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_PRIMITIVES + +#ifndef MAPDEMO_SCHEMA_TABLE_ARENA +#define MAPDEMO_SCHEMA_TABLE_ARENA + +namespace mapdemo { + +// ---- variable-length tables: tuning constants (docs/SPEC-TABLES.md) ---- +// +// The segment size and the count multiply to exactly 2^32: the u32 reference +// is the arena's hard ceiling, and these constants saturate it rather than +// leaving address space unreachable. Slab handout costs one atomic per slab, +// so per-node allocation costs no synchronization at all. + +static const uint32_t kTableSegmentBits = 22; // 4 MiB segments +static const uint32_t kTableSegmentSize = 1u << kTableSegmentBits; +static const uint32_t kTableSegmentMask = kTableSegmentSize - 1u; +static const uint32_t kTableMaxSegments = 1u << ( 32 - kTableSegmentBits ); // 1024 -> 4 GiB +static const uint32_t kTableSlabBytes = 64u * 1024u; // one atomic per slab +static const uint32_t kTableAlign = 8; // every node starts 8-aligned +static const uint32_t kTableAllocFailed = 0xFFFFFFFFu; + +// ---- THE CALLER'S ALLOCATOR (docs/SPEC-TABLES.md §6.5) ---- +// +// Every allocation the variable-length runtime makes goes through one of +// these — the arena's segments, the pack walk's identity map, the numbering's +// entry array, the packed region, and the tool path's node directory. There is +// no other call to the C library on this path, so a counting allocator sees +// every byte and a game's own heap can own all of it. +// +// It is the shape TableBlockAllocator already has (§19.1): two function +// pointers and a context the caller carries. What it adds is a CONTRACT ON +// alloc — the bytes come back ZEROED. Lock copies whole nodes, PADDING +// INCLUDED, so anything left uninitialized reaches a packed region; the default +// pair reaches that through calloc, which costs nothing measurable because a +// fresh segment is untouched pages either way. +struct TableAllocator +{ + void * ( *alloc )( void * context, int64_t bytes ); // ZEROED bytes, NULL on failure + void ( *free )( void * context, void * pointer ); + void * context; +}; + +// The default pair, and it is the one every entry point takes when the caller +// names none. It calls schema_allocate / schema_release, so a program with its +// own C-library replacement can move the floor without writing a struct at all. +inline void * table_default_alloc( void * context, int64_t bytes ) { (void) context; return schema_allocate( bytes ); } +inline void table_default_free( void * context, void * pointer ) { (void) context; schema_release( pointer ); } + +inline TableAllocator TableDefaultAllocator() +{ + TableAllocator allocator; + allocator.alloc = table_default_alloc; + allocator.free = table_default_free; + allocator.context = NULL; + return allocator; +} + +// ---- TableRef: a relocatable reference (never a machine pointer) ---- +// +// Two encodings, one slot, and the FORM says which is in force: +// +// in the arena — the node's arena offset (segment index in the high bits) +// in a region — the SELF-RELATIVE byte delta from this slot's own address, +// so a deref is one add, needs no base pointer, and a whole +// region relocates by memcpy with zero fix-up +// +// 0 is null in both, and a slot can never name the node that contains it, so +// zero names nothing real in either form. +// +// A REGION DELTA HAS NO REQUIRED SIGN (§6.3). A region is packed depth-first, +// so a node's FIRST reference points forward; every LATER reference to that +// same node points BACK at the one body it already has, which is exactly what +// makes one node one node in a region. Sharing and a back-reference are the +// same fact, and nothing validates a reference by its sign. +// +// IT IS EIGHT BYTES, SIGNED, so ONE REGION REACHES EVERYTHING (§6.3, §7): a +// four-byte slot bounded a region at 2 GiB, and the scale a cook exists for is +// *"100mbs or many gigabytes of data in Assets.bin"*. +struct TableRef +{ + int64_t value = 0; + bool null() const { return value == 0; } +}; + +// TableSlot is what Alloc hands back: usable as the node pointer (write +// fields through it) AND as the reference to store in a pointer field. +template struct TableSlot +{ + T * ptr = NULL; + TableRef ref; + T * operator->() const { return ptr; } + T & operator*() const { return *ptr; } + operator T *() const { return ptr; } + operator TableRef() const { return ref; } + bool null() const { return ptr == NULL; } +}; + +inline uint32_t TableAlignUp( uint32_t bytes ) { return ( bytes + kTableAlign - 1 ) & ~( kTableAlign - 1 ); } +inline int64_t TableAlignUp64( int64_t bytes ) { return ( bytes + kTableAlign - 1 ) & ~( int64_t( kTableAlign ) - 1 ); } + +// ---- a BYTE BUFFER's node (docs/SPEC-TABLES.md §2.5, §6.3) ---- +// +// A *bytes or *string slot is a TableRef like every pointer slot, and it names +// a BLOB NODE: this eight-byte header and then the bytes, at offset eight so +// the data is eight-aligned. A *string blob carries one more zero byte after +// its data, so a region hands back a C string with no copy. The node's extent +// is the header plus its bytes, rounded to the arena's alignment like every +// node's; on the wire it is a record whose body is the bytes (§3.1). +struct TableBlob +{ + uint32_t length; + uint32_t zero; +}; + +static const int64_t kTableBlobHeader = 8; // length (u32), then four zero bytes +static const int64_t kTableBlobMaxLength = 0xFFFFFFFF; // a record's length is a u32 (§3.1) + +// the node's storage: the header, the bytes, a string's terminator, rounded +// to the arena's alignment like every node +inline int64_t TableBlobStorage( int64_t length, bool terminated ) +{ + return TableAlignUp64( kTableBlobHeader + length + ( terminated ? 1 : 0 ) ); +} + +// What a read answers: a pointer INTO the region and the length, NULL and +// zero for a null slot. Off a locked region, a loaded one or an opened cook +// the pointer is one add from the slot, and nothing is copied. +struct TableBytesView +{ + const uint8_t * data; + int64_t length; +}; + +struct TableStringView +{ + const char * data; // zero-terminated + int64_t length; +}; + +// What AllocBytes and AllocString hand back: the bytes to write through, the +// length asked for, and the reference to store in the slot — the three +// answers TableSlot gives for a table node. +struct TableBytesSlot +{ + uint8_t * data = NULL; + int64_t length = 0; + TableRef ref; + bool null() const { return data == NULL; } + operator TableRef() const { return ref; } +}; + +struct TableStringSlot +{ + char * data = NULL; // room for length bytes and the terminator, already zero + int64_t length = 0; + TableRef ref; + bool null() const { return data == NULL; } + operator TableRef() const { return ref; } +}; + +// ---- the arena: segmented, slab-handed, lock-free by ownership ---- +// +// Allocation is thread-local inside a worker's slab — no atomics on the node +// path. A worker takes its next slab with ONE compare-exchange, and a new +// segment is published with one more. Nothing ever moves: a segment, once +// allocated, lives untouched until the arena is torn down, so a T* obtained +// from Alloc stays valid while other workers allocate, and an offset stays +// correct while the arena grows. +// +// The model this DELIBERATELY refuses: one buffer under a lock, grown by +// realloc. A realloc moves the buffer under workers mid-write; offsets fix +// identity but not the raw references already resolved from them, and the +// resulting corruption is invisible until much later. Segments never move, so +// that bug class cannot be written here. +// +// Slack: at most one slab tail per worker plus one slab per segment (a slab +// that will not fit is skipped rather than split), i.e. under 2% of a segment +// plus threads x 64 KiB. That is the price of never synchronizing per node. +struct TableArena +{ + std::atomic segments[ kTableMaxSegments ]; + std::atomic cursor; // (segment << kTableSegmentBits) | bytes handed out + bool locked = false; // MONOTONIC: Lock() is one-way, there is no unlock + // THE ARENA CARRIES ITS OWN, so everything downstream of a builder — + // segments, pack map, numbering, region, node directory — allocates through + // the one pair the caller named, with nothing to thread by hand. + TableAllocator allocator; +}; + +inline void TableArenaInit( TableArena & arena, TableAllocator allocator ) +{ + for ( uint32_t i = 0; i < kTableMaxSegments; i++ ) + { + arena.segments[i].store( NULL, std::memory_order_relaxed ); + } + arena.cursor.store( 0, std::memory_order_relaxed ); + arena.locked = false; + arena.allocator = allocator; +} + +inline void TableArenaShutdown( TableArena & arena ) +{ + for ( uint32_t i = 0; i < kTableMaxSegments; i++ ) + { + uint8_t * segment = arena.segments[i].exchange( NULL, std::memory_order_acq_rel ); + if ( segment != NULL ) { arena.allocator.free( arena.allocator.context, segment ); } + } + arena.cursor.store( 0, std::memory_order_relaxed ); +} + +// one L1 load plus an add: the segment table is 8 KiB and stays hot +inline uint8_t * TableArenaAt( const TableArena & arena, uint32_t offset ) +{ + return arena.segments[ offset >> kTableSegmentBits ].load( std::memory_order_relaxed ) + ( offset & kTableSegmentMask ); +} + +// TableArenaGrabSlab hands one worker its next private slab. Returns +// kTableAllocFailed when the arena's address space or the allocator is +// exhausted — a loud refusal, never a silent smaller slab. +inline uint32_t TableArenaGrabSlab( TableArena & arena ) +{ + for ( ;; ) + { + uint32_t cursor = arena.cursor.load( std::memory_order_acquire ); + uint32_t segment = cursor >> kTableSegmentBits; + uint32_t used = cursor & kTableSegmentMask; + // strictly less: a slab is never split across segments, and the tail + // is the documented slack + if ( used + kTableSlabBytes < kTableSegmentSize ) + { + if ( arena.segments[segment].load( std::memory_order_acquire ) == NULL ) + { + // THE SEGMENT COMES BACK ZEROED, which is the allocator's + // contract and not an extra pass here: Lock copies whole nodes, + // PADDING INCLUDED, so anything uninitialized reaches a packed + // region. Value-initializing a node with placement new zeroes + // its MEMBERS and not its padding, so the zeroing has to happen + // at the segment or not at all. It costs nothing measurable: a + // fresh segment is untouched pages either way, and the default + // pair's calloc has the kernel hand them over zeroed. + uint8_t * memory = (uint8_t *) arena.allocator.alloc( arena.allocator.context, (int64_t) kTableSegmentSize ); + if ( memory == NULL ) { return kTableAllocFailed; } + uint8_t * expected = NULL; + if ( !arena.segments[segment].compare_exchange_strong( expected, memory, std::memory_order_acq_rel ) ) + { + // another worker published this segment first + arena.allocator.free( arena.allocator.context, memory ); + } + } + if ( arena.cursor.compare_exchange_weak( cursor, cursor + kTableSlabBytes, std::memory_order_acq_rel ) ) + { + return ( segment << kTableSegmentBits ) | used; + } + continue; + } + uint32_t next_segment = segment + 1; + if ( next_segment >= kTableMaxSegments ) { return kTableAllocFailed; } // 4 GiB: the u32 reference's ceiling + arena.cursor.compare_exchange_weak( cursor, next_segment << kTableSegmentBits, std::memory_order_acq_rel ); + } +} + +// TableArenaGrabSpan reserves a SPAN of the arena's address space for one node +// larger than a slab — a BYTE BUFFER of any size (docs/SPEC-TABLES.md §2.5) — +// and allocates it as one contiguous block. It takes whole segment indices +// from the cursor, starting at the index after the cursor's so nothing else +// is ever handed out inside the span, and publishes the block under the first +// of them; the indices the span covers past that one stay NULL, which is +// enough, because only a node's START is ever resolved through the segment +// table and a blob's bytes follow its header inside the one allocation. The +// unused tail of the segment the cursor was in is slack, like a slab tail. +// Returns kTableAllocFailed when the address space or the allocator is +// exhausted — a loud refusal, never a smaller blob. +inline uint32_t TableArenaGrabSpan( TableArena & arena, int64_t bytes ) +{ + if ( bytes <= 0 || bytes > ( (int64_t) kTableMaxSegments - 2 ) * (int64_t) kTableSegmentSize ) { return kTableAllocFailed; } + const uint32_t spanned = (uint32_t) ( ( bytes + kTableSegmentSize - 1 ) >> kTableSegmentBits ); + for ( ;; ) + { + uint32_t cursor = arena.cursor.load( std::memory_order_acquire ); + uint32_t start = ( cursor >> kTableSegmentBits ) + 1; + if ( start + spanned >= kTableMaxSegments ) { return kTableAllocFailed; } // 4 GiB: the u32 reference's ceiling + uint32_t next = ( start + spanned ) << kTableSegmentBits; + if ( !arena.cursor.compare_exchange_weak( cursor, next, std::memory_order_acq_rel ) ) { continue; } + // the span is this worker's now: nothing else can publish under its + // first index, so a plain store suffices, and the block comes back + // ZEROED like every segment — the blob's bytes and its tail are zeros + // until written + uint8_t * memory = (uint8_t *) arena.allocator.alloc( arena.allocator.context, bytes ); + if ( memory == NULL ) { return kTableAllocFailed; } + arena.segments[start].store( memory, std::memory_order_release ); + return start << kTableSegmentBits; + } +} + +// ---- TableWorker: one thread's allocation front ---- +// +// The threading contract, stated plainly: +// * Alloc on YOUR OWN worker is safe concurrently with any other worker's. +// No locks, no atomics per node. +// * Writing fields of a node ANOTHER worker allocated is your own +// synchronization problem — this runtime does not arbitrate it. +// * Lock and Save are single-threaded: call them after the workers have +// joined. +struct TableWorker +{ + TableArena * arena = NULL; + uint32_t next = 0; + uint32_t end = 0; + + template TableSlot Alloc() + { + static_assert( alignof( T ) <= kTableAlign, "a table node's alignment must fit the arena's" ); + TableSlot slot; + if ( arena == NULL || arena->locked ) { return slot; } + uint32_t bytes = TableAlignUp( (uint32_t) sizeof( T ) ); + if ( bytes > kTableSlabBytes ) { return slot; } // a node larger than a slab: refused, never split + if ( end == 0 || next + bytes > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return slot; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + uint32_t at = next; + next += bytes; + // A NODE IS BORN IN TWO HALVES: start its lifetime in the raw + // storage, then write the declared defaults ONE MEMBER AT A TIME. + // + // It is "T", not "T{}". Value-initialising the whole aggregate says + // the same thing and costs cl O(BYTES) TO COMPILE — it expands element + // by element in its front end — while both halves here cost + // O(declarations). The slab cap below refuses a large node at RUN + // TIME and bounds nothing at compile time: the cost is paid by + // whatever T a caller instantiates this with. + // Padding is not the difference: value-initialisation zeroes MEMBERS + // and not padding either way, which is why the segment is calloc'd. + // + // TableReset is an OVERLOAD SET, one per closure member, reached from + // this template by argument-dependent lookup on T's own namespace — + // Alloc is a template and cannot spell Reset. + // + // The reset is here because ONE DEFINITION SAYS WHAT THE DECLARED + // DEFAULTS ARE, and it is Reset. Default-initialisation lands on + // the same values today, because a member with a non-zero default + // carries a member initializer that says so — but that is the class + // definition agreeing with Reset, not the arena reading it, and #320's + // fix was itself a pass that MOVED initialisation between the two. + // The arena reads the definition. + slot.ptr = new ( TableArenaAt( *arena, at ) ) T; + TableReset( *slot.ptr ); + slot.ref.value = at; + return slot; + } + + // Alloc a BYTE BUFFER's node of exactly length bytes (docs/SPEC-TABLES.md + // §2.5): the blob header and its bytes, zeroed, in this thread's slab when + // it fits and in a span of the arena's own when it does not. NULL is the + // arena locked, a length below zero or past a record's u32, or the + // allocator refusing. The offset comes back for the reference. + TableBlob * AllocBlob( int64_t length, bool terminated, uint32_t & at ) + { + at = 0; + if ( arena == NULL || arena->locked ) { return NULL; } + if ( length < 0 || length > kTableBlobMaxLength ) { return NULL; } + const int64_t bytes = TableBlobStorage( length, terminated ); + if ( bytes > (int64_t) kTableSlabBytes ) + { + at = TableArenaGrabSpan( *arena, bytes ); + if ( at == kTableAllocFailed ) { at = 0; return NULL; } + } + else + { + if ( end == 0 || next + (uint32_t) bytes > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return NULL; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + at = next; + next += (uint32_t) bytes; + } + TableBlob * blob = (TableBlob *) TableArenaAt( *arena, at ); + blob->length = (uint32_t) length; // the bytes after it are the segment's zeros + blob->zero = 0; + return blob; + } + + // RAW, ZEROED storage of the bytes asked for, at the alignment asked for: a MAP's or a LIST's builder + // head and its segments (docs/SPEC-TABLES.md §2.8, §2.9). It is not a node: it carries + // no type id, takes no index and has no Reset, so it goes through the same + // slab and span the blob path uses rather than through Alloc. + uint8_t * AllocRaw( int64_t bytes, int64_t align, uint32_t & at ) + { + at = 0; + if ( arena == NULL || arena->locked ) { return NULL; } + if ( bytes <= 0 || align > (int64_t) kTableAlign ) { return NULL; } + const int64_t rounded = TableAlignUp64( bytes ); + if ( rounded > (int64_t) kTableSlabBytes ) + { + at = TableArenaGrabSpan( *arena, rounded ); + if ( at == kTableAllocFailed ) { at = 0; return NULL; } + return TableArenaAt( *arena, at ); + } + if ( end == 0 || next + (uint32_t) rounded > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return NULL; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + at = next; + next += (uint32_t) rounded; + return TableArenaAt( *arena, at ); // the segment came back zeroed + } + // a *bytes node: the bytes to write through, and the reference to store + TableBytesSlot AllocBytes( int64_t length ) + { + TableBytesSlot slot; + uint32_t at = 0; + TableBlob * blob = AllocBlob( length, false, at ); + if ( blob == NULL ) { return slot; } + slot.data = (uint8_t *) ( blob + 1 ); + slot.length = length; + slot.ref.value = at; + return slot; + } + + // a *string node: room for length bytes and the zero byte after them + TableStringSlot AllocString( int64_t length ) + { + TableStringSlot slot; + uint32_t at = 0; + TableBlob * blob = AllocBlob( length, true, at ); + if ( blob == NULL ) { return slot; } + slot.data = (char *) ( blob + 1 ); + slot.length = length; + slot.ref.value = at; + return slot; + } +}; + +// ---- TablePackMap: the pack walk's identity map (docs/SPEC-TABLES.md §3.1, §6.2) ---- +// +// ONE ENTRY PER REACHABLE NODE, and that map IS identity: a node must know +// where it landed to be named a second time, so Lock packs a shared node ONCE +// and every later reference resolves to the one body it already has. That is +// the same first-visit numbering the wire uses, so the pack order and the node +// order are one order. +// +// COLOURING AN ENTRY WHILE ITS DESCENT IS OPEN COSTS ONE BIT, and it is what +// makes a data cycle free to refuse: a reference to an entry still open is a +// cycle, and Lock returns failure rather than recursing away. The ROOT's entry +// is open for the whole walk. +// +// The map is proportional to NODES, never to bytes, and it lives on the +// AUTHORING side, where §6.5 licenses allocation. Nothing on the reading path +// ever builds one. +struct TablePackEntry +{ + const void * key; // the node's address in the graph being packed + int64_t offset; // where that node landed in the region + uint8_t open; // its descent is still open: a reference here is a cycle +}; + +struct TablePackMap +{ + TablePackEntry * entries = NULL; + int64_t capacity = 0; // a power of two, or zero while empty + int64_t count = 0; + TableAllocator allocator; // the caller's, carried from the walk that built it +}; + +inline void TablePackMapInit( TablePackMap & map, TableAllocator allocator ) +{ + map.entries = NULL; + map.capacity = 0; + map.count = 0; + map.allocator = allocator; +} + +inline void TablePackMapShutdown( TablePackMap & map ) +{ + map.allocator.free( map.allocator.context, map.entries ); + TablePackMapInit( map, map.allocator ); +} + +// The two walks behind Lock re-derive the SAME map from the same graph — the +// numbering is never carried between them (§3.1) — so the second starts from +// an empty map and keeps the capacity the first paid for. +inline void TablePackMapReset( TablePackMap & map ) +{ + if ( map.entries != NULL ) { memset( map.entries, 0, (size_t) map.capacity * sizeof( TablePackEntry ) ); } + map.count = 0; +} + +// open addressing, linear probing, a multiply-shift hash over the address: a +// node key is a pointer and its low bits are alignment, so the low bits alone +// would collide on every node of one type +inline int64_t TablePackMapSlot( const TablePackMap & map, const void * key ) +{ + uint64_t hash = (uint64_t) (uintptr_t) key; + hash *= 0x9E3779B97F4A7C15ull; + hash ^= hash >> 29; + int64_t mask = map.capacity - 1; + int64_t at = (int64_t) ( hash & (uint64_t) mask ); + while ( map.entries[at].key != NULL && map.entries[at].key != key ) + { + at = ( at + 1 ) & mask; + } + return at; +} + +inline TablePackEntry * TablePackMapFind( TablePackMap & map, const void * key ) +{ + if ( map.capacity == 0 ) { return NULL; } + TablePackEntry * entry = &map.entries[ TablePackMapSlot( map, key ) ]; + return entry->key == key ? entry : NULL; +} + +// QUADRUPLING, not doubling, and the reason is measured: growth rehashes every +// entry, and on a graph of 131,071 nodes the doubling schedule spent 45% of +// Lock in rehashing alone. Quadrupling from 1024 buys 1.35x on that graph and +// keeps the map NODE-proportional (§6.2) — under 128 bytes a node at its +// worst, right after a grow, and about 64 on average. +inline bool TablePackMapGrow( TablePackMap & map ) +{ + TablePackMap grown; + grown.allocator = map.allocator; + grown.capacity = map.capacity != 0 ? map.capacity * 4 : 1024; + grown.entries = (TablePackEntry *) map.allocator.alloc( map.allocator.context, grown.capacity * (int64_t) sizeof( TablePackEntry ) ); + if ( grown.entries == NULL ) { return false; } + for ( int64_t i = 0; i < map.capacity; i++ ) + { + if ( map.entries[i].key == NULL ) { continue; } + grown.entries[ TablePackMapSlot( grown, map.entries[i].key ) ] = map.entries[i]; + grown.count++; + } + map.allocator.free( map.allocator.context, map.entries ); + map = grown; + return true; +} + +// REACH a node: one probe answers both questions the walk has. A true "taken" +// says this is a FIRST visit, and the entry is now the node's, coloured open +// at "offset"; otherwise the entry is the one the node already has, and its +// open bit says cycle or sharing. NULL is an allocation failure, and it is a +// refusal like any other: Lock fails rather than packing a graph it cannot +// track. +// +// It is one call and not a find followed by an insert because the walk asks +// this question twice per node — once to measure, once to pack — and every +// probe is a miss into a table larger than L2. +inline TablePackEntry * TablePackMapReach( TablePackMap & map, const void * key, int64_t offset, bool & taken, int64_t & slot ) +{ + if ( ( map.count + 1 ) * 4 >= map.capacity * 3 ) // keep the load factor under three quarters + { + if ( !TablePackMapGrow( map ) ) { return NULL; } + } + slot = TablePackMapSlot( map, key ); + TablePackEntry * entry = &map.entries[slot]; + taken = entry->key != key; // an empty slot is a first visit; the key is never NULL + if ( taken ) + { + entry->key = key; + entry->offset = offset; + entry->open = 1; + map.count++; + } + return entry; +} + +// The descent finished: the node keeps its entry — identity outlives the +// descent — and stops being a cycle. The "hint" is the slot Reach returned, and it +// is checked against the key rather than trusted, so a rehash between the two +// costs a second probe instead of correctness. +inline void TablePackMapClose( TablePackMap & map, const void * key, int64_t hint ) +{ + if ( hint >= 0 && hint < map.capacity && map.entries[hint].key == key ) + { + map.entries[hint].open = 0; + return; + } + TablePackEntry * entry = TablePackMapFind( map, key ); + if ( entry != NULL ) { entry->open = 0; } +} + +// ---- resolution contexts: which encoding a walk is reading ---- + +struct TableArenaCtx { const TableArena * arena; }; +struct TableRegionCtx {}; + +// ---- a BYTE BUFFER's resolution (docs/SPEC-TABLES.md §2.5, §6.3) ---- +// +// The same two encodings a table pointer has, resolved the same way: a +// self-relative delta in a region — one add, no base — and an arena offset +// while the builder is mutable. The blob is reached through its header, and a +// view is the header plus eight and the header's first word. Nothing here +// allocates and nothing copies: off a locked region, a loaded one or an +// opened cook the view points INTO the region. +inline const TableBlob * TableBlobAt( const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) ( (const uint8_t *) &ref + ref.value ) : NULL; +} +inline const TableBlob * TableBlobAt( const TableRegionCtx &, const TableRef & ref ) { return TableBlobAt( ref ); } +inline const TableBlob * TableBlobAt( const TableArenaCtx & ctx, const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) TableArenaAt( *ctx.arena, (uint32_t) ref.value ) : NULL; +} +inline const TableBlob * TableBlobAt( const TableArena & arena, const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) TableArenaAt( arena, (uint32_t) ref.value ) : NULL; +} + +inline TableBytesView TableBytesViewOf( const TableBlob * blob ) +{ + TableBytesView view = { NULL, 0 }; + if ( blob != NULL ) { view.data = (const uint8_t *) ( blob + 1 ); view.length = (int64_t) blob->length; } + return view; +} +inline TableStringView TableStringViewOf( const TableBlob * blob ) +{ + TableStringView view = { NULL, 0 }; + if ( blob != NULL ) { view.data = (const char *) ( blob + 1 ); view.length = (int64_t) blob->length; } + return view; +} + +// the const form's hot path: one add, no base +inline TableBytesView TableBytesAt( const TableRef & ref ) { return TableBytesViewOf( TableBlobAt( ref ) ); } +inline TableStringView TableStringAt( const TableRef & ref ) { return TableStringViewOf( TableBlobAt( ref ) ); } +// and the context forms a walk uses: a region context, an arena context, or +// the arena itself while the builder is mutable +template inline TableBytesView TableBytesAt( const Ctx & ctx, const TableRef & ref ) { return TableBytesViewOf( TableBlobAt( ctx, ref ) ); } +template inline TableStringView TableStringAt( const Ctx & ctx, const TableRef & ref ) { return TableStringViewOf( TableBlobAt( ctx, ref ) ); } + +// allocate a blob in the arena and point the slot at it; the slot holds the +// arena offset, as every slot does while the builder is mutable +inline uint8_t * TableBytesEmplace( TableWorker & worker, TableRef & slot, int64_t length ) +{ + TableBytesSlot allocated = worker.AllocBytes( length ); + slot = allocated.ref; + return allocated.data; +} +// the text is copied in when one is given; a NULL text leaves the zeros for +// the caller to fill +inline char * TableStringEmplace( TableWorker & worker, TableRef & slot, const char * text, int64_t length ) +{ + TableStringSlot allocated = worker.AllocString( length ); + slot = allocated.ref; + if ( allocated.data != NULL && text != NULL && length > 0 ) { memcpy( allocated.data, text, (size_t) length ); } + return allocated.data; +} + +// ---- the FLAT NODE TABLE (docs/SPEC-TABLES.md §3.1) ---- +// +// A pointered save writes every reachable node ONCE, into a node table, and a +// pointer field rides as an INDEX into it under kind 17. The encoding is +// flat: no pointer edge is a nesting level, so a chain's length is not a depth, +// and two references to one node are one node. +// +// THE FIELD RIDES ONCE: an L with sixty-four bits of capability frames a +// numbering of any size, so the whole numbering is one contiguous payload and a +// save's node bodies have no aggregate ceiling. + +static const uint64_t kTableNodeIndexNull = 0; // absence and null are one value +static const uint64_t kTableNodeIndexRoot = 1; // the body that hosts the table + +// The not-materialized sentinel (§6.3): a record whose type id this build could +// not name. Distinct from every real offset including the root's 0, so an index +// resolving through it yields NULL and can never fabricate the root. +static const uint64_t kTableNodeAbsent = 0xFFFFFFFFFFFFFFFFull; + +// What a node's storage answers when the FRAMING ITSELF is refused rather than +// merely unnameable: a count its L cannot carry, one above the int32 cap, or a +// blob past the size cap (docs/SPEC-TABLES.md §3.1, §6.5). An unnameable type +// id commands no storage and keeps its index. This one makes the whole measure +// answer -1 with its reason. +static const int64_t kTableNodeRefused = -2; + +// ---- the numbering, on the SAVE side ---- +// +// One entry per reachable node in FIRST-VISIT order, so entry k is node index +// k + 2. The two thunks are what let one loop write a table of mixed types: the +// numbering walk knows each target's type STATICALLY at the site it numbers it, +// so it stores the instantiation there and the loop never asks what a node is. +struct TableNumbering; + +struct TableNodeEntry +{ + const void * node; + uint64_t type_id; + // the type id's MESSAGE-FORM SLOT (docs/SPEC-TABLES.md §3.3), stored where + // the numbering walk stores the id itself and for the same reason: the + // target's type is known STATICALLY at the site that numbers it, so a + // form 2 save reads the slot out of the entry instead of looking an id up. + // Every pointer target's type id is an entry of the announcement, which is + // what makes the slot a compile-time fact of a POINTERED message too. + uint64_t type_slot; + int64_t ( * measure )( const void * ctx, const TableNumbering & numbering, TableIds & ids, const void * node ); + bool ( * save )( const void * ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const void * node ); + // the same two over the MESSAGE FORM (docs/SPEC-TABLES.md §3.3): a bitpacked + // body at a bit position, its pointer indices at the width the node count + // settled + int64_t ( * message_measure )( const void * ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const void * node ); + bool ( * message_save )( const void * ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const void * node ); +}; + +struct TableNumbering +{ + TablePackMap seen; // node -> index; the ROOT is index 1, open for the whole walk + TableNodeEntry * entries = NULL; + int64_t count = 0; + int64_t capacity = 0; +}; + +// The numbering allocates through the map's pair rather than carrying a second +// copy of it: one numbering is one walk, and a walk has one allocator. +inline void TableNumberingInit( TableNumbering & n, TableAllocator allocator ) +{ + TablePackMapInit( n.seen, allocator ); + n.entries = NULL; + n.count = 0; + n.capacity = 0; +} + +inline void TableNumberingShutdown( TableNumbering & n ) +{ + TableAllocator allocator = n.seen.allocator; + TablePackMapShutdown( n.seen ); + allocator.free( allocator.context, n.entries ); + n.entries = NULL; + n.count = 0; + n.capacity = 0; +} + +// The index a numbered node was given, for the save that writes it into a +// pointer slot. False means the two walks disagree about the graph, which is a +// refusal and never a guess. +inline bool TableNumberingIndex( const TableNumbering & n, const void * node, uint64_t & index ) +{ + if ( n.seen.capacity == 0 ) { return false; } + const TablePackEntry & entry = n.seen.entries[ TablePackMapSlot( n.seen, node ) ]; + if ( entry.key != node ) { return false; } + index = (uint64_t) entry.offset; + return true; +} + +inline bool TableNumberingAppend( TableNumbering & n, const TableNodeEntry & entry ) +{ + if ( n.count == n.capacity ) + { + // GROW BY COPY, never by realloc: the allocator hook is a PAIR, and a + // game's heap is not required to have a resize primitive at all. The + // schedule quadruples, so the copying is amortized to a constant per + // entry and the growth is the same growth it always was. + int64_t capacity = n.capacity != 0 ? n.capacity * 4 : 256; + TableAllocator allocator = n.seen.allocator; + TableNodeEntry * grown = (TableNodeEntry *) allocator.alloc( allocator.context, capacity * (int64_t) sizeof( TableNodeEntry ) ); + if ( grown == NULL ) { return false; } + if ( n.entries != NULL ) + { + memcpy( grown, n.entries, (size_t) n.count * sizeof( TableNodeEntry ) ); + allocator.free( allocator.context, n.entries ); + } + n.entries = grown; + n.capacity = capacity; + } + n.entries[n.count++] = entry; + return true; +} + +// The thunks the numbering stores. Each resolves to the closure member's own +// MeasureBody / SaveBodyFields through an overload set in the member's DECLARING +// file, reached by argument-dependent lookup at instantiation — the same bridge +// the arena's TableReset uses, and the reason a numbering may span the files of +// one unit without any file naming another's members. +template +inline int64_t TableNodeMeasureThunk( const void * ctx, const TableNumbering & numbering, TableIds & ids, const void * node ) +{ + return TableNodeMeasure( *(const Ctx *) ctx, numbering, ids, *(const T *) node ); +} + +template +inline bool TableNodeSaveThunk( const void * ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const void * node ) +{ + return TableNodeSave( *(const Ctx *) ctx, numbering, w, ids, *(const T *) node ); +} + +template +inline int64_t TableNodeMessageMeasureThunk( const void * ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const void * node ) +{ + return TableNodeMessageMeasure( *(const Ctx *) ctx, numbering, index_bits, at, *(const T *) node ); +} + +template +inline bool TableNodeMessageSaveThunk( const void * ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const void * node ) +{ + return TableNodeMessageSave( *(const Ctx *) ctx, numbering, index_bits, w, *(const T *) node ); +} +// ---- a BYTE BUFFER's record (docs/SPEC-TABLES.md §2.5, §3.1) ---- +// +// A blob rides as a node record under one of two RESERVED type ids — the fold +// a table's name takes, over the keywords "bytes" and "string", which no table +// can be named — with the bytes as its body and nothing framed inside. These +// two thunks are what the numbering stores for a blob, as it stores a +// member's codec for a table: the length, and the bytes verbatim. +static const uint64_t kTableBytesTypeId = 0x2f2ec0474f1c4fe4ull; // fnv1a64( "bytes" ) +static const uint64_t kTableStringTypeId = 0x704be0d8faaffc58ull; // fnv1a64( "string" ) + +template +inline int64_t TableBlobMeasureThunk( const void *, const TableNumbering &, TableIds &, const void * node ) +{ + return (int64_t) ( (const TableBlob *) node )->length; +} + +template +inline bool TableBlobSaveThunk( const void *, const TableNumbering &, TableWriter & w, TableIds &, const void * node ) +{ + const TableBlob * blob = (const TableBlob *) node; + w.raw( (const void *) ( blob + 1 ), (int64_t) blob->length ); + return true; +} + +// and the same two on the MESSAGE FORM (§3.3): a blob record is its length at +// thirty-two raw bits, an ALIGN, then the bytes verbatim +template +inline int64_t TableBlobMessageMeasureThunk( const void *, const TableNumbering &, int64_t, int64_t at, const void * node ) +{ + const int64_t length = (int64_t) ( (const TableBlob *) node )->length; + return 32 + TableAlignBits( at + 32 ) + length * 8; +} + +template +inline bool TableBlobMessageSaveThunk( const void *, const TableNumbering &, int64_t, TableBitWriter & w, const void * node ) +{ + const TableBlob * blob = (const TableBlob *) node; + w.put( (uint64_t) blob->length, 32 ); + w.align(); + w.putbytes( (const uint8_t *) ( blob + 1 ), (int64_t) blob->length ); + return !w.overflow; +} +// TableNodeTableMeasure and TableNodeTableSave are the framing, and they are +// ONE fill rule written twice — measure derives it from the graph and save +// derives the same one, which is what makes measure == save hold across a +// pointer graph (§3.1). +// +// The field rides ONCE, under the reserved id, kind 12: the payload opens with +// the count and then carries the records back to back, each a type id +// REFERENCE, a length and a body. The reserved id is interned BEFORE the +// records, and a record's type id before its body, which is the first-use order +// the trailer is written in (§3). +template +inline int64_t TableNodeTablePayload( const Ctx & ctx, TableIds & ids, const TableNumbering & n ) +{ + int64_t payload = TableLebBytes( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + payload += TableLebBytes( ids.ref( n.entries[k].type_id ) ); + const int64_t body = n.entries[k].measure( (const void *) &ctx, n, ids, n.entries[k].node ); + if ( body < 0 ) { return -1; } + payload += TableLebBytes( (uint64_t) body ) + body; + } + return payload; +} + +template +inline int64_t TableNodeTableMeasure( const Ctx & ctx, TableIds & ids, const TableNumbering & n ) +{ + if ( n.count == 0 ) { return 0; } // a root that reaches no nodes writes none of them + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayload( ctx, ids, n ); + if ( payload < 0 ) { return -1; } + return TableLebBytes( ref ) + 1 + TableLebBytes( (uint64_t) payload ) + payload; +} + +template +inline bool TableNodeTableSave( const Ctx & ctx, TableWriter & w, TableIds & ids, const TableNumbering & n ) +{ + if ( n.count == 0 ) { return true; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayload( ctx, ids, n ); + if ( payload < 0 ) { return false; } + w.putleb( ref ); + w.put8( 12 ); // kind 12 is the opaque byte payload: a reader that cannot name the id skips by L + w.putleb( (uint64_t) payload ); + w.putleb( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.putleb( ids.ref( n.entries[k].type_id ) ); + const int64_t body = n.entries[k].measure( (const void *) &ctx, n, ids, n.entries[k].node ); + if ( body < 0 ) { return false; } + w.putleb( (uint64_t) body ); + if ( !n.entries[k].save( (const void *) &ctx, n, w, ids, n.entries[k].node ) ) { return false; } + } + return true; +} + +// ---- the numbering, on the LOAD side: a region's NODE DIRECTORY (§6.3) ---- +// +// The wire's numbering made resident: one entry per numbered node, in index +// order, position i describing node index i + 1 — so position 0 is the ROOT at +// offset 0. It is ATTRIBUTION, and attribution is separable: nothing that reads +// a structure touches it, a deref is one add on a self-relative offset, and a +// caller may release it once Load returns. +struct TableNodeDirEntry +{ + uint64_t offset; + uint64_t type_id; +}; + +// the node's extent cursor, defined with the extent runtime (docs/SPEC-TABLES.md +// §2.8, §2.9); the node map names it only through a pointer. +struct TableExtentCarve; + +// TableNodeMap is what a pointer slot resolves through while a body decodes. +struct TableNodeMap +{ + uint8_t * base = NULL; + const TableNodeDirEntry * entries = NULL; + int64_t count = 0; // the ROOT's entry included, so it is records + 1 + bool good = false; // the node table read whole; a numbering that failed resolves nothing + // WHERE THE NODES LIVE, and therefore what a resolved slot holds: a region + // takes the SELF-RELATIVE delta so a deref is one add, and the tool's + // builder path takes the node's ARENA OFFSET (§6.3). + bool arena = false; + // WHERE A MAP'S ENTRIES AND A LIST'S ELEMENTS LAND while this node's body + // decodes (docs/SPEC-TABLES.md §2.8, §2.9): the node's own extent on the + // region path and the builder's arena on the tool's. It is MUTABLE + // because the cursor belongs to ONE node's decode and the dispatch that + // owns that node holds the map by const reference, exactly as it did + // before either construct existed. The decoder's signature does not + // move for a construct it may not carry. + mutable TableExtentCarve * carve = NULL; + // and the TOOL's path's allocation front, set once: there the arrays + // are the builder's arena's rather than a node's extent. + TableWorker * worker = NULL; + // THE TOOL PATH'S REFUSAL (docs/SPEC-TABLES.md §2.9): a count above the + // int32 cap met while a body decoded. LoadBuilder answers NULL for it + // and moves no counter; mutable for the reason the cursor is. + mutable bool refused = false; +}; + +// TableNodeResolve places one node index in a pointer slot, and every failure +// is one of §4's events with the pointer left null. The declared TARGET type id +// is checked at every index, the root's included: the root carries no record +// and therefore no wire type id, so the READER'S OWN root type is what the +// claim is checked against. +inline void TableNodeResolve( const TableNodeMap & map, TableRef & slot, uint64_t index, uint64_t target, TableReport * report ) +{ + slot.value = 0; + if ( index == kTableNodeIndexNull || !map.good ) { return; } + if ( index - 1 >= (uint64_t) map.count ) + { + report->malformed = true; // an index above node_count + 1 + return; + } + const TableNodeDirEntry & entry = map.entries[index - 1]; + if ( entry.offset == kTableNodeAbsent ) + { + // a node whose type id this build could not name KEEPS ITS INDEX, and + // every pointer naming it reads null. The unknown was counted once, at + // the node, not once per pointer. + return; + } + if ( entry.type_id != target ) + { + report->kind_mismatch++; + return; + } + slot.value = map.arena ? (int64_t) entry.offset + : (int64_t) ( ( map.base + entry.offset ) - (const uint8_t *) &slot ); +} + +// ---- the record SCAN, and it is the whole of load's bound (§3.1) ---- +// +// Reading follows no reference. The scan walks the root body's top-level fields, +// finds the ONE under the reserved id, and reads records out of its payload in +// order — the field rides once, so nothing is copied to make a body contiguous +// and the generated body decoder never learns the transport exists. +struct TableNodeScan +{ + TableReader fields; // over the ROOT body, skipping past everything else + const uint8_t * payload; // the node-table field's payload + int64_t payload_size; + int64_t payload_offset; + bool opened; // the root body has been walked for the field + uint64_t declared; + int64_t records; + bool present; // the root body carries a node table at all + bool malformed; + const TableIdTable * ids; +}; + +inline TableNodeScan TableNodeScanBegin( const uint8_t * body, int64_t size, TableReport * report, const TableIdTable * ids ) +{ + TableNodeScan s = { TableReader( body, size, report, ids ), NULL, 0, 0, false, 0, 0, false, false, ids }; + return s; +} + +// find the node-table field, or answer false when the root body has none. A +// body carrying an id more than once is legal input and THE LAST OCCURRENCE +// WINS (docs/SPEC-TABLES.md §3), so the walk runs to the terminator and keeps +// the last rather than stopping at the first. +inline bool TableNodeScanOpen( TableNodeScan & s ) +{ + if ( s.opened ) { return false; } + s.opened = true; + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.fields.getleb( ref ) ) { break; } + if ( ref == 0 ) { break; } // the terminator + if ( s.ids == NULL || ref > (uint64_t) s.ids->count ) { break; } + const uint64_t id = s.ids->at( ref ); + if ( !s.fields.has( 1 ) ) { break; } + const uint8_t kind = s.fields.get8(); + if ( id == kTableNodeTableFieldId ) + { + s.present = true; + if ( kind != 12 ) { s.malformed = true; return false; } + uint64_t length = 0; + if ( !s.fields.getleb( length ) || !s.fields.room( length ) ) { s.malformed = true; return false; } + s.payload = s.fields.buffer + s.fields.offset; + s.payload_size = (int64_t) length; + s.fields.offset += (int64_t) length; + continue; + } + if ( !s.fields.skip( kind ) ) { break; } + } + if ( s.payload == NULL ) { return false; } + TableReader head( s.payload, s.payload_size, s.fields.report, s.ids ); + if ( !head.getleb( s.declared ) ) { s.malformed = true; return false; } + s.payload_offset = head.offset; + return true; +} + +// the next record, or false at the end of the table — s.malformed says whether +// the end was the end or the framing giving out +inline bool TableNodeScanNext( TableNodeScan & s, uint64_t & type_id, const uint8_t * & body, int64_t & length ) +{ + if ( !s.opened && !TableNodeScanOpen( s ) ) { return false; } + if ( s.payload == NULL || s.payload_offset >= s.payload_size ) { return false; } + TableReader rec( s.payload, s.payload_size, s.fields.report, s.ids ); + rec.offset = s.payload_offset; + uint64_t ref = 0; + if ( !rec.getleb( ref ) || ref == 0 || s.ids == NULL || ref > (uint64_t) s.ids->count ) + { + s.malformed = true; // a type id reference of 0, or one past the table + return false; + } + type_id = s.ids->at( ref ); + uint64_t declared_length = 0; + if ( !rec.getleb( declared_length ) ) + { + s.malformed = true; // a record whose length is damaged + return false; + } + if ( declared_length > (uint64_t) ( s.payload_size - rec.offset ) ) + { + s.malformed = true; // a record whose length runs past its field + return false; + } + body = s.payload + rec.offset; + length = (int64_t) declared_length; + s.payload_offset = rec.offset + length; + s.records++; + return true; +} + +// The record scan is AUTHORITATIVE: node_count is data from the wire, and a +// count that disagrees with the scan is malformed. Nothing is sized from it +// before the scan has confirmed it. +inline bool TableNodeScanWhole( TableNodeScan & s ) +{ + if ( s.malformed ) { return false; } + if ( !s.present ) { return true; } // no node table at all is not a broken one + return s.declared == (uint64_t) s.records; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_ARENA + +#ifndef MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES +#define MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES + +namespace mapdemo { + +// ---- the NODE TABLE on the message wire (docs/SPEC-TABLES.md §3.1, §3.3) ---- +// +// THE NODE TABLE, WHEN A BODY HAS ONE, IS THE FIRST FIELD OF THE ROOT BODY: the +// reserved id as a reference, the node count at THIRTY-TWO RAW BITS, then the +// records back to back, each a type id reference and a body: a table's fields +// end at their own zero reference, and a blob's body is a length, an align and +// its bytes. A root +// that reaches no node elides the field, like every other empty thing. +// +// Measure derives the numbering from the graph and save derives the same one, +// and the two thunks stored at numbering time are what let one loop write a +// table of mixed types. +template +inline int64_t TableMessageNodeTableMeasure( const Ctx & ctx, const TableNumbering & n, int64_t index_bits, int64_t at ) +{ + if ( n.count == 0 ) { return 0; } // a root that reaches no nodes writes none of them + int64_t bits = kTableMessageRefBitsHere + 32; + for ( int64_t k = 0; k < n.count; k++ ) + { + bits += kTableMessageRefBitsHere; + const int64_t body = n.entries[k].message_measure( (const void *) &ctx, n, index_bits, at + bits, n.entries[k].node ); + if ( body < 0 ) { return -1; } + bits += body; + } + return bits; +} + +template +inline bool TableMessageNodeTableSave( const Ctx & ctx, const TableNumbering & n, int64_t index_bits, TableBitWriter & w ) +{ + if ( n.count == 0 ) { return true; } + w.put( kTableNodeTableFieldSlot, kTableMessageRefBitsHere ); + w.put( (uint64_t) n.count, 32 ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.put( n.entries[k].type_slot, kTableMessageRefBitsHere ); + if ( !n.entries[k].message_save( (const void *) &ctx, n, index_bits, w, n.entries[k].node ) ) { return false; } + } + return !w.overflow; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES + +#ifndef MAPDEMO_SCHEMA_TABLE_RETAIN +#define MAPDEMO_SCHEMA_TABLE_RETAIN + +namespace mapdemo { + +// ---- RETAIN-UNKNOWN (docs/SPEC-TABLES.md §6.6) ---- +// +// A REGION ROUND TRIP AND ONLY THAT: LoadRetain is Load's path into a region +// and SaveRetain saves from that same region. The builder path carries no +// retention, because a builder has no node directory to anchor a record on and +// re-derives its numbering from the reader's declaration order. +// +// Nothing here allocates. The record bytes and the retained-id list are the +// caller's storage, declared with their capacities, and a record that does not +// fit whole is dropped with one retain_lost. + +// THE PATH NAMES THE BODY, and it is the REGION's own address (§6.6). Step one +// is the node's index in the region's node directory, 1 for the root body and +// k for the node at directory position k - 1. Every further step is the PAIR: +// the field ordinal in the body the step descends from, in the READER's own +// declaration order, and the element index inside that field: zero for a +// scalar body, the element's index for an array of any of the four kinds, the +// ARM's OWN ORDINAL for a union, and the key's slot for a map. +static const int32_t kTableRetainDepthMax = 5; + +struct TableRetainStep +{ + uint32_t ordinal; + uint32_t index; +}; + +// at is the node's own address, which is what the SAVE side matches on: a +// record carries the directory INDEX and the directory answers the address in +// one add, so neither side ever searches a numbering. +struct TableRetainPath +{ + const void * at; + uint32_t node; + int32_t depth; + TableRetainStep steps[ kTableRetainDepthMax ]; +}; + +inline TableRetainPath TableRetainPathRoot( const void * at, uint32_t node ) +{ + TableRetainPath path; + path.at = at; + path.node = node; + path.depth = 0; + return path; +} + +// A STEP IS COMPUTED LOCALLY, at the moment the walk descends (§6.6), and it +// is taken by VALUE so that a descent is an expression: both sides walk the +// same declaration order, so neither numbers a tree and neither pops. +inline TableRetainPath TableRetainStepInto( const TableRetainPath & path, uint32_t ordinal, uint32_t index ) +{ + TableRetainPath out = path; + if ( out.depth < kTableRetainDepthMax ) + { + out.steps[ out.depth ].ordinal = ordinal; + out.steps[ out.depth ].index = index; + } + out.depth++; + return out; +} + +// THE CALLER'S TWO STORES (§6.6): the record bytes and the retained ids, each +// a pointer, a capacity and what has been used of it. A retention buffer +// belongs to ONE loaded region, and the next LoadRetain into it resets both. +struct TableRetain +{ + // AN ENTRY IS THE ID AND ITS SLOT IN THE TRAILER BEING WRITTEN. The two + // stores are numbered into ONE trailer in merged first-use order, so an + // index into this list is not the number a second reference wants and the + // slot rides beside the id. The layout is this port's own. + struct Id + { + uint64_t id; + int32_t slot; + }; + + uint8_t * bytes = NULL; + int64_t capacity = 0; + int64_t used = 0; + Id * ids = NULL; + int32_t id_capacity = 0; + int32_t id_used = 0; + int32_t count = 0; // records held + + // the REGION this buffer belongs to: a record carries a directory index + // and the save resolves it here, so nothing searches and nothing allocates + const uint8_t * base = NULL; + const TableNodeDirEntry * directory = NULL; + int64_t directory_count = 0; +}; + +// A RETAINED RECORD IS READER-PRIVATE (§6.6). It is not a wire form: no form +// byte, no version, no declared byte order, and nothing ever writes one to +// disk or hands one to another process. What it must CARRY is the body it +// belongs to, and the field's identity and bytes with every reference +// resolved. This layout is one sound way to carry them and nothing compares +// two ports' buffers. +// +// u32 record bytes, this header included +// u32 node the path's first step +// u32 depth the step pairs that follow +// u32 payload bytes +// u64 field id +// u8 kind +// u8 placed the save's own mark, cleared before every save +// depth x { u32 ordinal, u32 index } +// payload the field's payload with every reference resolved +static const int64_t kTableRetainRecordHeader = 26; + +inline uint32_t TableRetainRead32( const uint8_t * p ) +{ + return uint32_t( p[0] ) | uint32_t( p[1] ) << 8 | uint32_t( p[2] ) << 16 | uint32_t( p[3] ) << 24; +} + +inline uint64_t TableRetainRead64( const uint8_t * p ) +{ + return uint64_t( TableRetainRead32( p ) ) | ( uint64_t( TableRetainRead32( p + 4 ) ) << 32 ); +} + +inline void TableRetainWrite32( uint8_t * p, uint32_t v ) +{ + p[0] = uint8_t( v ); p[1] = uint8_t( v >> 8 ); p[2] = uint8_t( v >> 16 ); p[3] = uint8_t( v >> 24 ); +} + +inline void TableRetainWrite64( uint8_t * p, uint64_t v ) +{ + TableRetainWrite32( p, uint32_t( v ) ); + TableRetainWrite32( p + 4, uint32_t( v >> 32 ) ); +} + +inline int64_t TableRetainRecordBytes( const uint8_t * record ) { return (int64_t) TableRetainRead32( record ); } +inline uint32_t TableRetainRecordNode( const uint8_t * record ) { return TableRetainRead32( record + 4 ); } +inline int32_t TableRetainRecordDepth( const uint8_t * record ) { return (int32_t) TableRetainRead32( record + 8 ); } +inline int64_t TableRetainRecordPayloadBytes( const uint8_t * record ) { return (int64_t) TableRetainRead32( record + 12 ); } +inline uint64_t TableRetainRecordId( const uint8_t * record ) { return TableRetainRead64( record + 16 ); } +inline uint8_t TableRetainRecordKind( const uint8_t * record ) { return record[24]; } +inline bool TableRetainRecordPlaced( const uint8_t * record ) { return record[25] != 0; } +inline const uint8_t * TableRetainRecordSteps( const uint8_t * record ) { return record + kTableRetainRecordHeader; } +inline const uint8_t * TableRetainRecordPayload( const uint8_t * record ) +{ + return record + kTableRetainRecordHeader + 8 * (int64_t) TableRetainRecordDepth( record ); +} +inline uint8_t * TableRetainRecordPayload( uint8_t * record ) +{ + return record + kTableRetainRecordHeader + 8 * (int64_t) TableRetainRecordDepth( record ); +} + +// THE RECORD'S OWN BODY, resolved through the directory the buffer holds. A +// node index names one node for the life of the region, so this is one add. +inline const void * TableRetainRecordAt( const TableRetain & retain, const uint8_t * record ) +{ + const uint32_t node = TableRetainRecordNode( record ); + if ( retain.directory == NULL || node == 0 || (int64_t) node > retain.directory_count ) { return NULL; } + return (const void *) ( retain.base + retain.directory[ node - 1 ].offset ); +} + +// Does this record belong to the body the walk is standing in? The node first, +// which rejects almost everything in one compare, then the step pairs. +inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * record, const TableRetainPath & path ) +{ + if ( TableRetainRecordDepth( record ) != path.depth ) { return false; } + if ( TableRetainRecordAt( retain, record ) != path.at ) { return false; } + const uint8_t * steps = TableRetainRecordSteps( record ); + for ( int32_t i = 0; i < path.depth; i++ ) + { + if ( TableRetainRead32( steps + 8 * i ) != path.steps[i].ordinal ) { return false; } + if ( TableRetainRead32( steps + 8 * i + 4 ) != path.steps[i].index ) { return false; } + } + return true; +} + +// EVERY ID THIS BUILD CAN NAME, ascending: the set TableIds's capacity is +// derived from. An id inside a retained record takes its trailer entry from +// the GENERATED table when it is here and from the CALLER's list otherwise, so +// no retained id ever enters the generated table and no id is written twice. +static const int32_t kTableRetainKnownIds = 76; +static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, + 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, + 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, + 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, + 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, +}; + +inline bool TableRetainNameable( uint64_t id ) +{ + int32_t low = 0, high = kTableRetainKnownIds - 1; + while ( low <= high ) + { + const int32_t mid = low + ( high - low ) / 2; + if ( kTableRetainKnown[mid] == id ) { return true; } + if ( kTableRetainKnown[mid] < id ) { low = mid + 1; } else { high = mid - 1; } + } + return false; +} + +// THE TWO STORES, NUMBERED INTO ONE TRAILER in merged first-use order (§6.6). +// It answers the surface TableIds answers, ref, count, truncate and +// overflow, so the retain family's codec is the plain one with its names +// changed, and +// the GENERATED TABLE IS UNTOUCHED: its capacity, its overflow rule and its +// -1 stand exactly as they are for every save. +struct TableRetainIds +{ + TableIds known; + int32_t known_slot[ TableIds::kCapacity ]; + TableRetain * retain; + int32_t count; + bool overflow; + bool lost; // a retained id past the caller's capacity: the record is dropped + + TableRetainIds( TableRetain * to_retain ) : retain( to_retain ), count( 0 ), overflow( false ), lost( false ) {} + + // an id this build CAN name, which is every id the generated codec writes + uint64_t ref( uint64_t id ) + { + const int32_t before = known.count; + const uint64_t k = known.ref( id ); + if ( known.overflow ) { overflow = true; return 1; } + if ( known.count != before ) { known_slot[ (int32_t) k - 1 ] = ++count; } + return (uint64_t) known_slot[ (int32_t) k - 1 ]; + } + + // an id from INSIDE a retained record. A retained id takes its entry from + // the caller's list, and one past the capacity sets lost: the record is + // dropped, nothing else about the save changes, and the save is never + // refused (§6.6). + uint64_t record_ref( uint64_t id ) + { + if ( TableRetainNameable( id ) ) { return ref( id ); } + if ( retain == NULL ) { lost = true; return 0; } + for ( int32_t i = 0; i < retain->id_used; i++ ) + { + if ( retain->ids[i].id == id ) { return (uint64_t) retain->ids[i].slot; } + } + if ( retain->id_used >= retain->id_capacity ) { lost = true; return 0; } + retain->ids[ retain->id_used ].id = id; + retain->ids[ retain->id_used ].slot = ++count; + retain->id_used++; + return (uint64_t) count; + } + + // undo every entry taken since mark, in either store. Both are appended in + // slot order, so an entry removed is the last one of its store. + void truncate( int32_t mark ) + { + while ( known.count > 0 && known_slot[ known.count - 1 ] > mark ) { known.truncate( known.count - 1 ); } + while ( retain != NULL && retain->id_used > 0 && retain->ids[ retain->id_used - 1 ].slot > mark ) { retain->id_used--; } + count = mark; + } +}; + +// THE FILE STILL CARRIES ONE ID TABLE (§3): the split is the writer's storage +// rather than the wire's, and the trailer is one merge of two slot-ordered +// stores. +inline int64_t TableRetainIdsBytes( const TableRetainIds & ids ) { return int64_t( ids.count ) * 8 + 8; } + +// A TWO-WAY MERGE over the stores' own slot order, and not a scan for each +// slot. Every entry either store holds took its slot from the same counter, so +// the two runs interleave to exactly the slots 1 to count and the merge has no +// case for a slot neither store took. +inline void TableRetainIdsWrite( TableWriter & w, const TableRetainIds & ids ) +{ + const int32_t retained = ids.retain != NULL ? ids.retain->id_used : 0; + int32_t i = 0, j = 0; + while ( i < ids.known.count || j < retained ) + { + if ( j >= retained || ( i < ids.known.count && ids.known_slot[i] < ids.retain->ids[j].slot ) ) + { + w.put64( ids.known.ids[i] ); + i++; + continue; + } + w.put64( ids.retain->ids[j].id ); + j++; + } + w.put64( uint64_t( ids.count ) ); +} + +// ---- THE RESOLVING WALK (§6.6) ---- +// +// A reference names a SLOT of the file's id table, so a verbatim copy +// re-emitted into a file whose table is ordered differently would point at +// other names in silence. A retained record therefore holds the field with +// every reference replaced by the sixty-four-bit id it names, and every length +// that frames a rewritten reference recomputed. +// +// THE WALK IS AN INTERPRETATION, AND ITS VERDICT IS STATED: it reads kind +// bytes, lengths and references and nothing else. No value is decoded, no +// bound is checked, no branch is taken on a payload byte, and anything it +// cannot frame DROPS THE RECORD, counts one retain_lost, and never raises +// malformed on the plain read. +// +// THE WALK IS ONE PASS EACH WAY, and its cost is linear in the record's own +// bytes. Every length that frames a content in the resolved form is a fixed +// slot rather than a canonical LEB128, so the capture reserves it, writes the +// content, and fills the slot in behind it. A spelling that had to know the +// resolved size before writing it would have to walk each content twice, once +// at every level, and the file chooses the nesting. +// +// A retained record's inner nesting is the WRITER's and not this build's, so +// it is the one depth on this path a file can drive. The cap counts NESTED +// BODIES, and a record past it is dropped on the same rule as any other shape +// the walk cannot take. Time no longer rests on it: it is a small stated +// constant and nothing more. +static const int32_t kTableRetainWalkDepthMax = 64; + +// the three RESERVED ids (§3.1, §3.3). One inside a retained record's payload +// would be re-emitted into a nested body, where it is malformed, so meeting +// one drops the record. +inline bool TableRetainReservedId( uint64_t id ) +{ + return id == kTableNodeTableFieldId || id == kTableBuildVersionFieldId || id == kTableMessageVocabularyFieldId; +} + +struct TableRetainIn +{ + const uint8_t * in; + int64_t size; + int64_t at; + const TableIdTable * ids; + uint8_t * out; // NULL: measuring, and nothing is written + int64_t out_at; +}; + +inline void TableRetainInRaw( TableRetainIn & s, const uint8_t * from, int64_t bytes ) +{ + if ( s.out != NULL ) { memcpy( s.out + s.out_at, from, (size_t) bytes ); } + s.out_at += bytes; +} + +inline void TableRetainInLeb( TableRetainIn & s, uint64_t v ) +{ + uint8_t b[10]; + int64_t n = 0; + while ( v >= 0x80 ) { b[n++] = uint8_t( v ) | 0x80; v >>= 7; } + b[n++] = uint8_t( v ); + TableRetainInRaw( s, b, n ); +} + +inline void TableRetainInId( TableRetainIn & s, uint64_t id ) +{ + uint8_t b[8]; + TableRetainWrite64( b, id ); + TableRetainInRaw( s, b, 8 ); +} + +inline bool TableRetainInLebRead( TableRetainIn & s, uint64_t & value ) +{ + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( s.at >= s.size ) { return false; } + const uint8_t b = s.in[ s.at++ ]; + if ( i == 9 && b > 1 ) { return false; } + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) { return i == 0 || b != 0; } + shift += 7; + } + return false; +} + +// one REFERENCE resolved to the id it names. A zero reference is the wire's +// own "no id", the enum's None and the union's empty arm, and rides as the +// id zero. A reference above the entry count, a reference at an id-table entry +// of zero, and a reference at a reserved id are each damage the plain read +// never looked at, and each drops the record. +inline bool TableRetainInRef( TableRetainIn & s, bool zero_allowed ) +{ + uint64_t ref = 0; + if ( !TableRetainInLebRead( s, ref ) ) { return false; } + if ( ref == 0 ) + { + if ( !zero_allowed ) { return false; } + TableRetainInId( s, 0 ); + return true; + } + if ( s.ids == NULL || ref > (uint64_t) s.ids->count ) { return false; } + const uint64_t id = s.ids->at( ref ); + if ( id == 0 || TableRetainReservedId( id ) ) { return false; } + TableRetainInId( s, id ); + return true; +} + +inline int64_t TableRetainInPayload( TableRetainIn & s, uint8_t kind, int32_t depth ); +inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ); + +// ONE FRAMED LENGTH in the resolved form: a pair of fixed u32. The first is +// the RESOLVED byte count of the content it frames, reserved here and written +// once the content is out. The second is the SAVE's scratch, left zero by the +// capture and filled by the walk that emits. +// +// The record is the reader's own storage and nothing outside this family ever +// reads it, so a length may be written after the bytes it measures. That is +// the whole of what makes the walk one pass. +static const int64_t kTableRetainSlotBytes = 8; + +inline int64_t TableRetainInSlot( TableRetainIn & s ) +{ + const int64_t at = s.out_at; + const uint8_t zero[ kTableRetainSlotBytes ] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + TableRetainInRaw( s, zero, kTableRetainSlotBytes ); + return at; +} + +inline void TableRetainInPatch( TableRetainIn & s, int64_t slot, int64_t resolved ) +{ + if ( s.out != NULL ) { TableRetainWrite32( s.out + slot, (uint32_t) resolved ); } +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it. +// The measuring pass takes the same path and reserves the same fixed width, so +// the size it answers is the size the writing pass lays down. +inline int64_t TableRetainInFramed( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s ); + const int64_t resolved = TableRetainInContent( s, kind, length, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s, slot, resolved ); + return resolved; +} + +inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + if ( length < 0 || s.at + length > s.size ) { return -1; } + const int64_t end = s.at + length; + const int64_t began = s.out_at; + switch ( kind ) + { + case 13: // a table BODY: fields, then the zero reference + { + for ( ;; ) + { + uint64_t ref = 0; + const int64_t mark = s.at; + if ( !TableRetainInLebRead( s, ref ) ) { return -1; } + // THE TERMINATOR IS A REFERENCE, and a reference in the + // resolved form is a fixed eight-byte id: the zero that ends a + // body rides at the width every other one does. + if ( ref == 0 ) { TableRetainInId( s, 0 ); break; } + s.at = mark; + if ( !TableRetainInRef( s, false ) ) { return -1; } + if ( s.at >= end ) { return -1; } + const uint8_t field_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &field_kind, 1 ); + if ( TableRetainInPayload( s, field_kind, depth ) < 0 ) { return -1; } + if ( s.at > end ) { return -1; } + } + break; + } + case 14: // an ARRAY body: the element kind, the count, then the elements + { + if ( s.at >= end ) { return -1; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainInLebRead( s, n ) ) { return -1; } + TableRetainInLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( TableRetainInPayload( s, elem_kind, depth ) < 0 ) { return -1; } + if ( s.at > end ) { return -1; } + } + break; + } + case 16: // an ENUM-KEYED body: N triples of a KEY REFERENCE, an L and the element + { + if ( s.at >= end ) { return -1; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainInLebRead( s, n ) ) { return -1; } + TableRetainInLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + // A KEYED BODY'S KEYS RESOLVE AT EVERY ELEMENT KIND (§6.6, §3.2) + if ( !TableRetainInRef( s, false ) ) { return -1; } + uint64_t slot_bytes = 0; + if ( !TableRetainInLebRead( s, slot_bytes ) ) { return -1; } + if ( slot_bytes > (uint64_t) ( end - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, elem_kind, (int64_t) slot_bytes, depth + 1 ) < 0 ) { return -1; } + } + break; + } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + default: + // every other content is bytes: a string, wide text, an escape, a + // payload-free kind, a scalar under a keyed slot's own length + TableRetainInRaw( s, s.in + s.at, length ); + s.at += length; + break; + } + if ( s.at != end ) { return -1; } + return s.out_at - began; +} + +// the depth a payload carries is its enclosing body's: only a framed CONTENT +// is a level, and TableRetainInContent is the one place the cap is read. +inline int64_t TableRetainInPayload( TableRetainIn & s, uint8_t kind, int32_t depth ) +{ + const int64_t began = s.out_at; + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: // the fixed-width kinds, by width + case 3: case 7: case 21: case 26: + case 4: case 8: case 10: case 22: case 27: + case 5: case 9: case 11: case 23: case 28: + case 18: case 19: case 24: case 29: + { + int64_t width = 1; + switch ( kind ) + { + case 3: case 7: case 21: case 26: width = 2; break; + case 4: case 8: case 10: case 22: case 27: width = 4; break; + case 5: case 9: case 11: case 23: case 28: width = 8; break; + case 18: case 19: case 24: case 29: width = 16; break; + default: width = 1; break; + } + if ( s.at + width > s.size ) { return -1; } + TableRetainInRaw( s, s.in + s.at, width ); + s.at += width; + break; + } + case 12: case 31: case 32: case 33: // L, then L bytes, nothing framed inside + { + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + TableRetainInLeb( s, length ); + TableRetainInRaw( s, s.in + s.at, (int64_t) length ); + s.at += (int64_t) length; + break; + } + case 13: case 14: case 16: // L, then a body the walk resolves + { + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, kind, (int64_t) length, depth + 1 ) < 0 ) { return -1; } + break; + } + case 15: // a UNION: the arm id reference, and when it is not zero its kind, L and payload + { + const int64_t mark = s.at; + uint64_t arm = 0; + if ( !TableRetainInLebRead( s, arm ) ) { return -1; } + s.at = mark; + if ( !TableRetainInRef( s, true ) ) { return -1; } + if ( arm == 0 ) { break; } + if ( s.at >= s.size ) { return -1; } + const uint8_t arm_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &arm_kind, 1 ); + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, arm_kind, (int64_t) length, depth + 1 ) < 0 ) { return -1; } + break; + } + case 30: // an ENUM's variant reference, zero for None + { + if ( !TableRetainInRef( s, true ) ) { return -1; } + break; + } + case 17: return -1; // A NODE INDEX (§3.1): the whole record goes with it + default: return -1; // a kind this walk cannot frame + } + return s.out_at - began; +} + +// ---- CAPTURE: the load side (§6.6) ---- +// +// The field is skipped by its framing exactly as it always was and counted +// unknown exactly as it always was, so a full buffer degrades to the default +// behavior one field at a time. False is what r.skip( kind ) answers false +// for, and nothing else: retention can lose a field, it can never turn a good +// read into a bad one. +inline bool TableRetainCapture( TableRetain * retain, TableReader & r, const TableRetainPath & path, + uint64_t field_id, uint8_t kind ) +{ + const int64_t start = r.offset; + if ( !r.skip( kind ) ) { return false; } + if ( retain == NULL ) { return true; } + const int64_t wire_bytes = r.offset - start; + + TableRetainIn probe; + probe.in = r.buffer + start; + probe.size = wire_bytes; + probe.at = 0; + probe.ids = r.ids; + probe.out = NULL; + probe.out_at = 0; + const int64_t payload = TableRetainInPayload( probe, kind, 0 ); + if ( payload < 0 || probe.at != wire_bytes ) { r.report->retain_lost++; return true; } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL: the buffer never holds a + // truncated field, and the read continues (§6.6) + r.report->retain_lost++; + return true; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, field_id ); + record[24] = kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableRetainIn write; + write.in = r.buffer + start; + write.size = wire_bytes; + write.at = 0; + write.ids = r.ids; + write.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out_at = 0; + if ( TableRetainInPayload( write, kind, 0 ) < 0 ) { r.report->retain_lost++; return true; } + retain->used += need; + retain->count++; + r.report->retained++; + return true; +} + +// LoadRetain RESETS BOTH STORES and writes into neither list (§6.6): a +// retained record carries its field's identity in the record itself, with +// every reference resolved. +inline void TableRetainReset( TableRetain * retain, const TableNodeMap & nodes, const uint8_t * region ) +{ + if ( retain == NULL ) { return; } + retain->used = 0; + retain->id_used = 0; + retain->count = 0; + retain->base = region; + retain->directory = nodes.entries; + retain->directory_count = nodes.count; +} + +// ---- RECORD LIFETIME (docs/SPEC-TABLES.md §6.6) ---- +// +// A RETAINED RECORD BELONGS TO THE BODY OCCURRENCE THAT CARRIED IT, AND DIES +// WITH IT. Legal input can carry a known child body twice, and the later +// occurrence resets the child and wins whole (§3, §4): the records retained +// under the earlier occurrence go with the values it held. The discard moves +// NEITHER counter. The writer superseded the data, so nothing was lost that +// the load could have kept, and retained counted the record when its bytes +// were kept and does not fall when they are let go. +// +// The occurrences are four, and each is a body the wire lets a writer put down +// again: a repeated TABLE field, by value or under ?, a UNION whose arm is +// written again, a MAP's duplicate key, and a KEYED-ARRAY slot written again. +// The FIELD form covers the three where the field itself is read again, arm +// switches and shrinking arrays included; the BODY form covers a duplicate key +// inside one occurrence of a map, where the field is read once and the entry +// twice. +inline bool TableRetainUnder( const uint8_t * record, const void * at, const TableRetain & retain, + const TableRetainPath & path, bool field, uint32_t ordinal ) +{ + if ( TableRetainRecordAt( retain, record ) != at ) { return false; } + const int32_t depth = TableRetainRecordDepth( record ); + if ( field ) + { + if ( depth <= path.depth ) { return false; } + } + else if ( depth < path.depth ) { return false; } + const uint8_t * steps = TableRetainRecordSteps( record ); + for ( int32_t i = 0; i < path.depth; i++ ) + { + if ( TableRetainRead32( steps + 8 * i ) != path.steps[i].ordinal ) { return false; } + if ( TableRetainRead32( steps + 8 * i + 4 ) != path.steps[i].index ) { return false; } + } + if ( field && TableRetainRead32( steps + 8 * path.depth ) != ordinal ) { return false; } + return true; +} + +inline void TableRetainDiscard( TableRetain * retain, const TableRetainPath & path, bool field, uint32_t ordinal ) +{ + if ( retain == NULL || retain->count == 0 ) { return; } + int64_t read = 0, write = 0; + int32_t kept = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + const int64_t bytes = TableRetainRecordBytes( retain->bytes + read ); + if ( !TableRetainUnder( retain->bytes + read, path.at, *retain, path, field, ordinal ) ) + { + if ( write != read ) { memmove( retain->bytes + write, retain->bytes + read, (size_t) bytes ); } + write += bytes; + kept++; + } + read += bytes; + } + retain->used = write; + retain->count = kept; +} + +inline void TableRetainDiscardBody( TableRetain * retain, const TableRetainPath & path ) +{ + TableRetainDiscard( retain, path, false, 0 ); +} + +inline void TableRetainDiscardField( TableRetain * retain, const TableRetainPath & path, uint32_t ordinal ) +{ + TableRetainDiscard( retain, path, true, ordinal ); +} + +// ---- EMIT: the save side (§6.6) ---- +// +// The record read back the other way: every resolved id becomes the reference +// the trailer being written gives it, and every length is recomputed against +// the references' new widths. The walk is the capture's mirror and the same +// damage rules apply, except that damage cannot be met: these bytes are the +// reader's own. +// +// A WIRE LENGTH IS CANONICAL LEB128 AND RIDES BEFORE ITS CONTENT, so this side +// cannot fill a slot in behind the bytes the way the capture does. It takes +// one POST-ORDER pass instead: measuring computes each content's wire size and +// leaves it in that content's own scratch slot, and the emit reads the size +// there rather than walking for it. Measuring runs immediately before the +// emit, on the same record and the same id table, which is what makes the two +// readings one walk. +struct TableRetainOut +{ + uint8_t * in; // the record: only a framed length's scratch half is written + int64_t size; + int64_t at; + TableRetainIds * ids; + TableWriter * w; // NULL: measuring, and the scratch slots are being filled + int64_t bytes; +}; + +inline void TableRetainOutRaw( TableRetainOut & s, const uint8_t * from, int64_t bytes ) +{ + if ( s.w != NULL ) { s.w->raw( from, bytes ); } + s.bytes += bytes; +} + +inline void TableRetainOutLeb( TableRetainOut & s, uint64_t v ) +{ + if ( s.w != NULL ) { s.w->putleb( v ); } + s.bytes += TableLebBytes( v ); +} + +inline bool TableRetainOutLebRead( TableRetainOut & s, uint64_t & value ) +{ + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( s.at >= s.size ) { return false; } + const uint8_t b = s.in[ s.at++ ]; + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) { return true; } + shift += 7; + } + return false; +} + +inline bool TableRetainOutRef( TableRetainOut & s ) +{ + if ( s.at + 8 > s.size ) { return false; } + const uint64_t id = TableRetainRead64( s.in + s.at ); + s.at += 8; + if ( id == 0 ) { TableRetainOutLeb( s, 0 ); return true; } // the wire's own no-id + const uint64_t ref = s.ids->record_ref( id ); + if ( s.ids->lost || s.ids->overflow ) { return false; } + TableRetainOutLeb( s, ref ); + return true; +} + +inline bool TableRetainOutPayload( TableRetainOut & s, uint8_t kind, int32_t depth ); +inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t length, int32_t depth ); + +// ONE FRAMED CONTENT, read out of the record's fixed slot and written with the +// canonical LEB128 length this wire wants. The ids it names are interned on +// the way past, which is what makes measure and save one walk in two readings, +// exactly as every other body on this wire is. +// +// MEASURING walks the content, then leaves the wire size it found in the +// scratch half of the slot. EMITTING reads that size, writes it, and CHECKS +// the content against it: a size no measure of this save left there is a +// record refused rather than a length that does not frame what follows. +inline bool TableRetainOutFramed( TableRetainOut & s, uint8_t kind, int32_t depth ) +{ + if ( s.at + kTableRetainSlotBytes > s.size ) { return false; } + uint8_t * const slot = s.in + s.at; + const int64_t resolved = (int64_t) TableRetainRead32( slot ); + s.at += kTableRetainSlotBytes; + if ( s.w == NULL ) + { + const int64_t began = s.bytes; + if ( !TableRetainOutContent( s, kind, resolved, depth ) ) { return false; } + const int64_t wire = s.bytes - began; + if ( wire > 0xFFFFFFFFll ) { return false; } + TableRetainWrite32( slot + 4, (uint32_t) wire ); + s.bytes += TableLebBytes( (uint64_t) wire ); + return true; + } + const int64_t wire = (int64_t) TableRetainRead32( slot + 4 ); + TableRetainOutLeb( s, (uint64_t) wire ); + const int64_t began = s.bytes; + if ( !TableRetainOutContent( s, kind, resolved, depth ) ) { return false; } + return s.bytes - began == wire; +} + +inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t length, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return false; } + if ( length < 0 || s.at + length > s.size ) { return false; } + const int64_t end = s.at + length; + switch ( kind ) + { + case 13: + { + for ( ;; ) + { + if ( s.at + 8 > end ) { return false; } + const uint64_t id = TableRetainRead64( s.in + s.at ); + if ( id == 0 ) { s.at += 8; TableRetainOutLeb( s, 0 ); break; } + if ( !TableRetainOutRef( s ) ) { return false; } + if ( s.at >= end ) { return false; } + const uint8_t field_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &field_kind, 1 ); + if ( !TableRetainOutPayload( s, field_kind, depth ) ) { return false; } + } + break; + } + case 14: + { + if ( s.at >= end ) { return false; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainOutLebRead( s, n ) ) { return false; } + TableRetainOutLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( !TableRetainOutPayload( s, elem_kind, depth ) ) { return false; } + } + break; + } + case 16: + { + if ( s.at >= end ) { return false; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainOutLebRead( s, n ) ) { return false; } + TableRetainOutLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( !TableRetainOutRef( s ) ) { return false; } + if ( !TableRetainOutFramed( s, elem_kind, depth + 1 ) ) { return false; } + } + break; + } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; + default: + TableRetainOutRaw( s, s.in + s.at, length ); + s.at += length; + break; + } + return s.at == end; +} + +// the depth a payload carries is its enclosing body's, exactly as on the +// capture side: only a framed CONTENT is a level. +inline bool TableRetainOutPayload( TableRetainOut & s, uint8_t kind, int32_t depth ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: + case 3: case 7: case 21: case 26: + case 4: case 8: case 10: case 22: case 27: + case 5: case 9: case 11: case 23: case 28: + case 18: case 19: case 24: case 29: + { + int64_t width = 1; + switch ( kind ) + { + case 3: case 7: case 21: case 26: width = 2; break; + case 4: case 8: case 10: case 22: case 27: width = 4; break; + case 5: case 9: case 11: case 23: case 28: width = 8; break; + case 18: case 19: case 24: case 29: width = 16; break; + default: width = 1; break; + } + if ( s.at + width > s.size ) { return false; } + TableRetainOutRaw( s, s.in + s.at, width ); + s.at += width; + break; + } + case 12: case 31: case 32: case 33: + { + uint64_t length = 0; + if ( !TableRetainOutLebRead( s, length ) ) { return false; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return false; } + TableRetainOutLeb( s, length ); + TableRetainOutRaw( s, s.in + s.at, (int64_t) length ); + s.at += (int64_t) length; + break; + } + case 13: case 14: case 16: + { + if ( !TableRetainOutFramed( s, kind, depth + 1 ) ) { return false; } + break; + } + case 15: + { + if ( s.at + 8 > s.size ) { return false; } + const uint64_t arm = TableRetainRead64( s.in + s.at ); + if ( !TableRetainOutRef( s ) ) { return false; } + if ( arm == 0 ) { break; } + if ( s.at >= s.size ) { return false; } + const uint8_t arm_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &arm_kind, 1 ); + if ( !TableRetainOutFramed( s, arm_kind, depth + 1 ) ) { return false; } + break; + } + case 30: + { + if ( !TableRetainOutRef( s ) ) { return false; } + break; + } + default: return false; + } + return true; +} + +// ---- THE RETAINED TAIL: where the records go back (§6.6) ---- +// +// AT THE END OF THEIR OWN BODY, IN THE ORDER RETAINED. Position carries +// nothing on this wire, so appending is chosen for three properties: it is a +// write with no splice, the retained order is preserved, and the result is +// IDEMPOTENT after the first save. +// +// A RETAINED ID PAST THE CAPACITY COUNTS ONE retain_lost AND ITS RECORD IS +// DROPPED, and the save is never refused. MeasureRetain and SaveRetain drop +// the same records under the same walk, so the measure sees the same overflow +// and its answer is the size the save writes. + +// one record's WIRE bytes under the trailer being written, and -1 for a record +// this save cannot place: an id the caller's list had no room for, or a +// resolved form the walk cannot read back. The ids it names are interned on +// the way past, which is what makes measure and save one rule read twice. +// +// THIS IS THE MEASURING PASS, and it leaves every framed content's wire size +// in that content's own scratch slot. The record is the caller's buffer and +// the pass writes nothing else into it. +inline int64_t TableRetainRecordWire( uint8_t * record, TableRetainIds & ids, uint64_t & ref ) +{ + const int32_t mark = ids.count; + ids.lost = false; + ref = ids.record_ref( TableRetainRecordId( record ) ); + if ( !ids.lost && !ids.overflow ) + { + TableRetainOut s; + s.in = TableRetainRecordPayload( record ); + s.size = TableRetainRecordPayloadBytes( record ); + s.at = 0; + s.ids = &ids; + s.w = NULL; + s.bytes = 0; + if ( TableRetainOutPayload( s, TableRetainRecordKind( record ), 0 ) && s.at == s.size ) + { + return TableLebBytes( ref ) + 1 + s.bytes; + } + } + // the record is not written at all, and nothing else about the save + // changes: a full id list degrades to the default behavior one record at a + // time, and the entries this attempt took are given back + ids.truncate( mark ); + ids.lost = false; + return -1; +} + +inline int64_t TableRetainTailMeasure( TableRetain * retain, TableRetainIds & ids, const TableRetainPath & path ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return 0; } + int64_t bytes = 0; + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordHere( *retain, record, path ) ) { continue; } + uint64_t ref = 0; + const int64_t wire = TableRetainRecordWire( record, ids, ref ); + if ( wire < 0 ) { continue; } + bytes += wire; + } + return bytes; +} + +inline bool TableRetainTailSave( TableRetain * retain, TableRetainIds & ids, TableWriter & w, const TableRetainPath & path ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return true; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordHere( *retain, record, path ) ) { continue; } + uint64_t ref = 0; + if ( TableRetainRecordWire( record, ids, ref ) < 0 ) { continue; } + w.putleb( ref ); + w.put8( TableRetainRecordKind( record ) ); + TableRetainOut s; + s.in = TableRetainRecordPayload( record ); + s.size = TableRetainRecordPayloadBytes( record ); + s.at = 0; + s.ids = &ids; + s.w = &w; + s.bytes = 0; + if ( !TableRetainOutPayload( s, TableRetainRecordKind( record ), 0 ) ) { return false; } + record[25] = 1; // PLACED: the one mark the save leaves on the buffer + } + return !w.overflow; +} + +// THE SAVE'S OWN SHARE OF retain_lost, counted ONCE and read after the save +// (§6.6): every record the walk did not place. A record whose path no longer +// names a body, one the caller's id list had no room for, and one the walk +// could not read back are one number here, because the check a caller reads is +// one number. A record is marked as it is written, so this cannot double-count +// a body measured twice. +inline void TableRetainClearPlaced( TableRetain * retain ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + retain->bytes[ at + 25 ] = 0; + at += TableRetainRecordBytes( retain->bytes + at ); + } +} + +inline void TableRetainCountLost( const TableRetain * retain, TableReport * report ) +{ + if ( retain == NULL || retain->bytes == NULL || report == NULL ) { return; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + const uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordPlaced( record ) ) { report->retain_lost++; } + } +} + +// THE NODE TABLE under retention (§3.1, §6.6): the same fill rule the plain +// pair derives, with the retain family's ids and each record's own body +// reached through a dispatch the CALL supplies rather than a second pair of +// thunks on the numbering. A store per node on the PLAIN save path would be a +// cost this feature is not allowed to have. +template +inline int64_t TableNodeTablePayloadRetain( const Ctx & ctx, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure ) +{ + int64_t payload = TableLebBytes( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + payload += TableLebBytes( ids.ref( n.entries[k].type_id ) ); + const int64_t body = measure( ctx, n, ids, n.entries[k].type_id, n.entries[k].node, retain ); + if ( body < 0 ) { return -1; } + payload += TableLebBytes( (uint64_t) body ) + body; + } + return payload; +} + +template +inline int64_t TableNodeTableMeasureRetain( const Ctx & ctx, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure ) +{ + if ( n.count == 0 ) { return 0; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayloadRetain( ctx, ids, n, retain, measure ); + if ( payload < 0 ) { return -1; } + return TableLebBytes( ref ) + 1 + TableLebBytes( (uint64_t) payload ) + payload; +} + +template +inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure, Save save ) +{ + if ( n.count == 0 ) { return true; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayloadRetain( ctx, ids, n, retain, measure ); + if ( payload < 0 ) { return false; } + w.putleb( ref ); + w.put8( 12 ); // kind 12 is the opaque byte payload, exactly as the plain save writes it + w.putleb( (uint64_t) payload ); + w.putleb( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.putleb( ids.ref( n.entries[k].type_id ) ); + const int64_t body = measure( ctx, n, ids, n.entries[k].type_id, n.entries[k].node, retain ); + if ( body < 0 ) { return false; } + w.putleb( (uint64_t) body ); + if ( !save( ctx, n, w, ids, n.entries[k].type_id, n.entries[k].node, retain ) ) { return false; } + } + return true; +} +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_RETAIN + +#ifndef MAPDEMO_SCHEMA_TABLE_EXTENT +#define MAPDEMO_SCHEMA_TABLE_EXTENT + +namespace mapdemo { + +// ---- the NODE EXTENT: where a map's entries and a list's elements live (§2.8, §2.9) ---- + +// TableExtentCarve is a node's extent cursor, PRE-ORDER: a container's whole +// array first, then, element by element in the container's own order, the +// arrays of any list or map an element holds by value. The cursor is the node +// map's, because the generated decoder is threaded with that and not with a +// region. +struct TableExtentCarve +{ + uint8_t * at = NULL; // the region path: the node's extent, unspent + int64_t left = 0; + TableWorker * worker = NULL; // the TOOL's path: the arrays come from the arena +}; + +// AN UNREACHED SLOT MUST HOLD NO LIST OR MAP WITH ELEMENTS IN IT (§2.8, §2.9, +// §7.6). An empty one takes no bytes, so a record whose extent measures ZERO is +// a record whose every by-value list and map is empty. A measure that REFUSED +// answers non-zero here too, and refusing on it is the same answer one level up. +inline bool TableExtentUnreachedEmpty( int64_t extent ) { return extent == 0; } + +// ---- LoadMeasure's framing walk (§6.5) ---- +// +// The measure reads no field value: it walks each record's field headers, +// skipping every payload by its framing, to reach each N at every depth. A +// false is a REFUSAL, and it carries its reason (§6.5). +typedef bool ( * TableWireExtentFn )( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ); + +// the framing walk over an ARRAY OF TABLES held by value: its elements' own +// lists and maps are part of this node's extent too +inline bool TableWireExtentElements( const uint8_t * body, int64_t length, int64_t & at, TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } + if ( r.get8() != 13 ) { return true; } + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +// and over an ENUM-KEYED array, whose triples carry a key REFERENCE before each +// length-prefixed element (docs/SPEC-TABLES.md §3.2) +inline bool TableWireExtentKeyed( const uint8_t * body, int64_t length, int64_t & at, TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } + if ( r.get8() != 13 ) { return true; } + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t key = 0; + if ( !r.getleb( key ) ) { return true; } + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_EXTENT + +#ifndef MAPDEMO_SCHEMA_TABLE_MAP +#define MAPDEMO_SCHEMA_TABLE_MAP + +namespace mapdemo { + +// ---- a MAP: a sorted entry array, and the lookup over it (§2.8) ---- +// +// On the wire, in a region and in a cook a map is an array of one generated +// ENTRY table held in ascending key order. What this adds is Find — a binary +// search over that array where it lies — and a builder that inserts, replaces +// and erases by key. Nothing here is stored: a region and a cook carry the +// array and the count, and not one byte about a hash or a probe. + +// entries carved from ONE call to the allocator pair; a new segment is +// appended when the current one fills, and nothing ever moves (§6.4) +static const int32_t kTableMapSegmentEntries = 32; + +// TableDeclRef names a type in an unevaluated context and is never defined — +// what 's declval is for, without the include the generated corpus +// refuses to pay for (the iterator_traits note, §13.9). +template T & TableDeclRef(); + +// THE ORDER IS TOTAL, AND IT IS THE SAME IN NINE LANGUAGES (§2.8). Integers +// compare by VALUE, signed for the signed kinds and unsigned for the unsigned. +// Strings compare by BYTES, unsigned, a shorter string that is a prefix of a +// longer one first: memcmp over the common length, then the lengths. Never a +// locale, never a code point, never a case fold. +inline int TableKeyOrder( uint64_t a, uint64_t b ) { return a < b ? -1 : ( a > b ? 1 : 0 ); } +inline int TableKeyOrder( int64_t a, int64_t b ) { return a < b ? -1 : ( a > b ? 1 : 0 ); } +inline int TableKeyOrder( const char * a, int32_t a_length, const char * b, int32_t b_length ) +{ + const int32_t common = a_length < b_length ? a_length : b_length; + if ( common > 0 ) + { + const int order = memcmp( (const void *) a, (const void *) b, (size_t) common ); + if ( order != 0 ) { return order < 0 ? -1 : 1; } + } + return a_length < b_length ? -1 : ( a_length > b_length ? 1 : 0 ); +} + +// the length of a NUL-terminated key at a call site, bounded by the storage it +// has to fit: a key one byte longer than the bound is refused, never truncated +inline int32_t TableKeyLength( const char * key, int32_t bound ) +{ + if ( key == NULL ) { return 0; } + for ( int32_t i = 0; i <= bound; i++ ) { if ( key[i] == 0 ) { return i; } } + return bound + 1; // longer than the bound: the caller refuses it +} + +// A KEY IS DATA AND A LENGTH, and the length is CARRIED, never recomputed +// (§2.8, §3). A string(N) key holds any byte a wire or a text can spell, +// U+0000 included, so a lookup that measures to the first NUL answers that "a" +// and "a", 0, "b" are the same key: the first entry is found, RESET, and +// relabeled with the second key, which deletes an entry the report never +// mentions. Every internal lookup and every insertion takes this pair, and the +// public const char * surface builds one and is a wrapper over it. +struct TableMapKeyRef +{ + const char * data; + int32_t length; +}; + +// ---- the storage: SIXTEEN BYTES in the holder's record (§2.8, §7.2) ---- +// +// An int64 self-relative reference to the entry array and an int32 count, then +// padding to eight. The reference is a TableRef like a pointer's: in the arena +// it names the builder's HEAD, in a region it is the delta from the slot to +// the first entry, and 0 is the empty map in both. +template struct TableMap +{ + TableRef entries; + int32_t count = 0; // the LIVE count, in both forms + int32_t padding = 0; // named, so the record has no unwritten byte in it + + // ---- the CONST form: a locked region, a loaded one, an opened cook ---- + // + // One surface over one encoding (§6.3). A region reference resolves from + // the slot's own address, so every one of these is a member and needs no + // base and no context. + const Entry * Entries() const + { + return entries.value != 0 ? (const Entry *) ( (const uint8_t *) &entries + entries.value ) : NULL; + } + int32_t size() const { return count; } + + // FIND: floor( log2 n ) + 1 key compares, in place, no allocation. NULL + // when absent, and on a map[K]*T the RESOLVED pointer, which is what a + // pointer field's accessor answers. + template const Entry * FindEntry( Key key ) const + { + const Entry * base = Entries(); + int32_t low = 0, high = count; + while ( low < high ) + { + const int32_t mid = low + ( high - low ) / 2; + const int order = TableEntryOrder( base[mid], key ); + if ( order == 0 ) { return base + mid; } + if ( order < 0 ) { low = mid + 1; } else { high = mid; } + } + return NULL; + } + // the return type is DEDUCED, so it is worked out when a call site + // instantiates Find and not when the holder's record declares the slot — + // which is what lets the entry's own overloads be declared after it + template auto Find( Key key ) const + { + return TableEntryFound( FindEntry( key ) ); + } + + // ---- iteration: ASCENDING key order, the key beside the value ---- + // + // A proxy BY VALUE, the keyed array's shape (§2.4): for ( auto [ key, + // value ] : map ). It carries no iterator_traits, for the reason + // TableKeyed's does not (§13.9). + struct ConstEntry + { + decltype( TableEntryKey( TableDeclRef() ) ) key; + decltype( TableEntryFound( (const Entry *) NULL ) ) value; + }; + + struct ConstIterator + { + const Entry * at; + ConstEntry operator*() const { return ConstEntry{ TableEntryKey( *at ), TableEntryFound( at ) }; } + ConstIterator & operator++() { at++; return *this; } + bool operator==( const ConstIterator & other ) const { return at == other.at; } + bool operator!=( const ConstIterator & other ) const { return at != other.at; } + }; + + ConstIterator begin() const { return ConstIterator{ Entries() }; } + ConstIterator end() const { return ConstIterator{ Entries() + count }; } +}; + +// ---- the BUILDER's side: a head, and segments that never move (§2.8, §6.4) ---- +// +// The head is a small node in the arena holding the segment chain, the live +// count and the dead count, allocated when the first entry is inserted. Each +// segment is a fixed number of entries carved from one call to the allocator +// pair. An entry's address is stable for the arena's life, so a value handed +// back by an insert stays valid while other entries arrive. +struct TableMapHead +{ + TableRef first; // the arena offset of the first segment + TableRef last; // and of the one an insert appends into + int32_t live; + int32_t dead; +}; + +template struct TableMapSegment +{ + TableRef next; + int32_t used; // entries carved from this segment + int32_t padding; + uint32_t dead[ ( kTableMapSegmentEntries + 31 ) / 32 ]; // Erase marks one bit, never the entry + Entry entries[ kTableMapSegmentEntries ]; +}; + +inline bool TableMapSegmentDead( const uint32_t * dead, int32_t index ) +{ + return ( dead[ index / 32 ] & ( 1u << ( index % 32 ) ) ) != 0; +} + +// ---- the ORDERED CURSOR the four writing walks read (§2.8) ---- +// +// Measure, Save, Lock and Cook each write a map's entries in ascending key +// order with no key twice, deriving the order from the builder's entries as +// each walk derives the numbering (§3.1). Nothing passes between them, so +// measure == save over a map is a real check on two sorts agreeing. +// +// A REGION is already sorted, so its cursor is the array in place and +// allocates nothing. The BUILDER's is the sort: an array of entry pointers +// allocated through the pair and released before the walk returns, because +// sorting the segments themselves would move entries whose addresses a caller +// holds. +template struct TableMapCursor +{ + const Entry * const * order = NULL; // the builder's form: sorted pointers + const Entry * entries = NULL; // the region's form: the array in place + int32_t count = 0; + TableAllocator allocator; + bool ok = false; + const Entry * operator[]( int32_t index ) const + { + return order != NULL ? order[index] : entries + index; + } +}; + +// heapsort: O( n log n ) once per map, no recursion, no allocation past the +// pointer array the caller already paid for +template inline void TableMapSort( const Entry ** order, int32_t count ) +{ + for ( int32_t start = count / 2 - 1; start >= 0; start-- ) + { + int32_t root = start; + for ( ;; ) + { + int32_t child = 2 * root + 1; + if ( child >= count ) { break; } + if ( child + 1 < count && TableEntryOrder( *order[child], *order[child + 1] ) < 0 ) { child++; } + if ( TableEntryOrder( *order[root], *order[child] ) >= 0 ) { break; } + const Entry * swap = order[root]; order[root] = order[child]; order[child] = swap; + root = child; + } + } + for ( int32_t end = count - 1; end > 0; end-- ) + { + const Entry * swap = order[0]; order[0] = order[end]; order[end] = swap; + int32_t root = 0; + for ( ;; ) + { + int32_t child = 2 * root + 1; + if ( child >= end ) { break; } + if ( child + 1 < end && TableEntryOrder( *order[child], *order[child + 1] ) < 0 ) { child++; } + if ( TableEntryOrder( *order[root], *order[child] ) >= 0 ) { break; } + const Entry * hold = order[root]; order[root] = order[child]; order[child] = hold; + root = child; + } + } +} + +// the REGION form: the array is already sorted, so the cursor is the array +template +inline TableMapCursor TableMapOrder( const TableRegionCtx &, const TableMap & map ) +{ + TableMapCursor cursor; + cursor.entries = map.Entries(); + cursor.count = map.count; + cursor.ok = true; + return cursor; +} + +// the BUILDER's form: gather the LIVE entries out of the segment chain in +// insertion order, then sort. A dead entry costs nothing on any wire (§2.8). +template +inline TableMapCursor TableMapOrder( const TableArena & arena, const TableMap & map ) +{ + TableMapCursor cursor; + cursor.allocator = arena.allocator; + cursor.count = map.count; + if ( map.entries.value == 0 || map.count <= 0 ) { cursor.ok = map.count == 0; cursor.count = 0; return cursor; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + if ( head->live != map.count ) { return cursor; } // the slot and the head disagree: refused, never guessed + const Entry ** order = (const Entry **) arena.allocator.alloc( arena.allocator.context, (int64_t) map.count * (int64_t) sizeof( const Entry * ) ); + if ( order == NULL ) { return cursor; } + int32_t at = 0; + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 && at < map.count ) + { + const TableMapSegment * segment = (const TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used && at < map.count; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + order[at++] = segment->entries + i; + } + segment_ref = segment->next; + } + if ( at != map.count ) + { + arena.allocator.free( arena.allocator.context, order ); + return cursor; + } + TableMapSort( order, map.count ); + cursor.order = order; + cursor.ok = true; + return cursor; +} + +template +inline TableMapCursor TableMapOrder( const TableArenaCtx & ctx, const TableMap & map ) +{ + return TableMapOrder( *ctx.arena, map ); +} + +template inline void TableMapRelease( TableMapCursor & cursor ) +{ + if ( cursor.order != NULL ) { cursor.allocator.free( cursor.allocator.context, (void *) cursor.order ); } + cursor.order = NULL; +} + +// ---- the builder's five (§2.8) ---- +// +// Insert APPENDS after one LINEAR SCAN of the live entries for the key it may +// replace, Find is that same scan, and Erase is the scan and one bit. The +// builder builds NO INDEX, and that is a rule: the sort happens once, at Lock, +// Save or Cook, and every lookup that matters runs over the sorted region. + +// the head, allocated when the first entry is inserted +template +inline TableMapHead * TableMapReach( TableWorker & worker, TableMap & map ) +{ + if ( worker.arena == NULL || worker.arena->locked ) { return NULL; } + if ( map.entries.value != 0 ) { return (TableMapHead *) TableArenaAt( *worker.arena, (uint32_t) map.entries.value ); } + uint32_t at = 0; + TableMapHead * head = (TableMapHead *) worker.AllocRaw( (int64_t) sizeof( TableMapHead ), (int64_t) alignof( TableMapHead ), at ); + if ( head == NULL ) { return NULL; } + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + map.entries.value = (int64_t) at; + return head; +} + +// one entry's storage, appended: the current segment when it has room, a new +// one carved from one call to the pair when it does not +template +inline Entry * TableMapAppend( TableWorker & worker, TableMapHead * head, TableMap & map ) +{ + TableMapSegment * segment = NULL; + if ( head->last.value != 0 ) + { + segment = (TableMapSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + if ( segment->used >= kTableMapSegmentEntries ) { segment = NULL; } + } + if ( segment == NULL ) + { + uint32_t at = 0; + segment = (TableMapSegment *) worker.AllocRaw( (int64_t) sizeof( TableMapSegment ), (int64_t) alignof( TableMapSegment ), at ); + if ( segment == NULL ) { return NULL; } // the arena could not carve another segment + segment->next.value = 0; + segment->used = 0; + segment->padding = 0; + for ( int32_t i = 0; i < (int32_t) ( sizeof( segment->dead ) / sizeof( segment->dead[0] ) ); i++ ) { segment->dead[i] = 0; } + if ( head->last.value != 0 ) + { + TableMapSegment * previous = (TableMapSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + previous->next.value = (int64_t) at; + } + else + { + head->first.value = (int64_t) at; + } + head->last.value = (int64_t) at; + } + Entry * entry = segment->entries + segment->used; + segment->used++; + head->live++; + map.count++; + return entry; +} + +// the LINEAR SCAN: the live entries in insertion order, O( n ) key compares +template +inline Entry * TableMapScan( const TableArena & arena, const TableMap & map, Key key ) +{ + if ( map.entries.value == 0 ) { return NULL; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( TableEntryOrder( segment->entries[i], key ) == 0 ) { return segment->entries + i; } + } + segment_ref = segment->next; + } + return NULL; +} + +// ERASE marks the entry DEAD, one bit in the segment's slot and not in the +// entry table, and decrements the live count. Its storage is reclaimed at +// RESET and never reused mid-build, because reusing a slot would make "an +// entry's address is stable" false for exactly one case. +template +inline bool TableMapErase( TableArena & arena, TableMap & map, Key key ) +{ + if ( map.entries.value == 0 ) { return false; } + TableMapHead * head = (TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( TableEntryOrder( segment->entries[i], key ) != 0 ) { continue; } + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + map.count--; + return true; + } + segment_ref = segment->next; + } + return false; +} + +// ---- iterate on the BUILDER: INSERTION order, live entries only (§2.8) ---- +template struct TableMapEach +{ + const TableArena * arena; + TableRef first; + + struct Iterator + { + const TableArena * arena; + TableMapSegment * segment; + int32_t index; + + void Skip() + { + for ( ;; ) + { + if ( segment == NULL ) { return; } + if ( index >= segment->used ) + { + segment = segment->next.value != 0 ? (TableMapSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + index = 0; + continue; + } + if ( TableMapSegmentDead( segment->dead, index ) ) { index++; continue; } + return; + } + } + auto operator*() const { return TableEntryEach( segment->entries + index ); } + Iterator & operator++() { index++; Skip(); return *this; } + bool operator==( const Iterator & other ) const { return segment == other.segment && index == other.index; } + bool operator!=( const Iterator & other ) const { return !( *this == other ); } + }; + + Iterator begin() const + { + Iterator it = { arena, first.value != 0 ? (TableMapSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL, 0 }; + it.Skip(); + return it; + } + Iterator end() const { Iterator it = { arena, NULL, 0 }; return it; } +}; + +template +inline TableMapEach TableMapEachOf( const TableArena & arena, const TableMap & map ) +{ + TableMapEach each = { &arena, TableRef() }; + if ( map.entries.value != 0 ) + { + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + each.first = head->first; + } + return each; +} + +// ---- the LOAD side: where a decoded entry lands (§2.8) ---- +// +// THE READER TRUSTS NOTHING and spends one compare per entry. Every load path +// applies the same rules and produces one report (§4), so the region load of +// §6.5 and LoadBuilder never disagree about a wire. These two shapes are what +// makes that true with one generated decoder: a REGION carves the entry array +// out of the holder node's own extent, and the TOOL's path appends into the +// builder's arena, and the decoder above them cannot tell which it has. + +// The node's extent cursor is TableExtentCarve, the extent runtime's (§2.8, +// §2.9): a map's whole entry array is carved first, then, entry by entry in +// key order, the arrays of any list or map an entry's value holds by value. + +// TableMapFill is one map field being decoded: where the next entry lands, and +// the entry that last LANDED, which is what the ascending check compares +// against. +template struct TableMapFill +{ + TableMap * map = NULL; + Entry * array = NULL; // the region path: the carved array + int32_t capacity = 0; + TableWorker * worker = NULL; // the TOOL's path + bool ok = false; +}; + +template +inline TableMapFill TableMapFillBegin( const TableNodeMap & nodes, TableMap & map, uint32_t n ) +{ + TableMapFill fill; + fill.map = ↦ + map.entries.value = 0; + map.count = 0; + if ( nodes.carve == NULL ) { return fill; } + if ( nodes.carve->worker != NULL ) + { + fill.worker = nodes.carve->worker; // the tool's path: the arena carves + fill.ok = true; + return fill; + } + const int64_t align = (int64_t) alignof( Entry ); + uint8_t * base = (uint8_t *) ( ( (uintptr_t) nodes.carve->at + (uintptr_t) ( align - 1 ) ) & ~( (uintptr_t) ( align - 1 ) ) ); + const int64_t bytes = (int64_t) n * (int64_t) sizeof( Entry ); + const int64_t pad = (int64_t) ( base - nodes.carve->at ); + if ( pad + bytes > nodes.carve->left ) { return fill; } // the measure and the load disagree: refused + nodes.carve->at = base + bytes; + nodes.carve->left -= pad + bytes; + fill.array = (Entry *) base; + fill.capacity = (int32_t) n; + map.entries.value = (int64_t) ( base - (const uint8_t *) &map.entries ); + fill.ok = true; + return fill; +} + +// the entry that last LANDED — NULL before the first +template inline Entry * TableMapFillLast( TableMapFill & fill ) +{ + if ( fill.map->count <= 0 ) { return NULL; } + if ( fill.array != NULL ) { return fill.array + ( fill.map->count - 1 ); } + return TableMapLive( *fill.worker->arena, *fill.map, fill.map->count - 1 ); +} + +// the next slot, at the entry type's declared defaults +template inline Entry * TableMapFillNext( TableMapFill & fill ) +{ + if ( fill.array != NULL ) + { + if ( fill.map->count >= fill.capacity ) { return NULL; } + Entry * entry = fill.array + fill.map->count; + TableReset( *entry ); + fill.map->count++; + return entry; + } + TableMapHead * head = TableMapReach( *fill.worker, *fill.map ); + if ( head == NULL ) { return NULL; } + Entry * entry = TableMapAppend( *fill.worker, head, *fill.map ); + if ( entry != NULL ) { TableReset( *entry ); } + return entry; +} + +// A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): at the first entry whose key +// kind disagrees with the reader's declaration the map resets to EMPTY, one +// kind_mismatch is counted for the map, and its remaining bytes are skipped. +template inline void TableMapFillReset( TableMapFill & fill ) +{ + if ( fill.array != NULL ) + { + fill.map->entries.value = 0; + fill.map->count = 0; + return; + } + if ( fill.map->entries.value != 0 ) + { + TableMapHead * head = (TableMapHead *) TableArenaAt( *fill.worker->arena, (uint32_t) fill.map->entries.value ); + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + } + fill.map->count = 0; +} + +// an EMPTY map's reference is null in both encodings, so a load that placed +// nothing leaves the slot exactly as a Reset does +template inline void TableMapFillEnd( TableMapFill & fill ) +{ + if ( fill.array != NULL && fill.map->count == 0 ) { fill.map->entries.value = 0; } +} + +// the k-th LIVE entry of a builder map, in insertion order — what the tool +// path's ascending check compares against +template +inline Entry * TableMapLive( const TableArena & arena, const TableMap & map, int32_t index ) +{ + if ( map.entries.value == 0 ) { return NULL; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + int32_t at = 0; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( at == index ) { return segment->entries + i; } + at++; + } + segment_ref = segment->next; + } + return NULL; +} + +// ---- LoadMeasure's term, from the FRAMING alone (§2.8, §6.5) ---- +// +// LoadMeasure's term for a map is N x sizeof( Entry ) rounded to +// alignof( Entry ), AT EVERY DEPTH. N is framing and not a value, so this +// reads no field: it walks the map's own header and, where an entry's value +// holds a map or a list of its own, the entries' headers under it. The caller +// owns the allocation precisely so it can refuse a number it did not expect. +// Every -1 carries its REASON (§6.5): the int32 cap first, because a count +// past it cannot fit any body, and then the body's own L, the one rule a +// list's term answers by. +// A MAP ENTRY'S SMALLEST WIRE FOOTPRINT that commands one storage unit is its +// own L and the body's terminator, and under this form's variable lengths that +// footprint is TWO BYTES (docs/SPEC-TABLES.md §4.2). It is what bounds the N a +// map's L can carry, and therefore what a LoadMeasure may be asked for. +static const int64_t kTableMapEntryFloor = 2; + +inline bool TableMapWireExtent( const uint8_t * body, int64_t length, int64_t & at, + int64_t entry_size, int64_t entry_align, TableWireExtentFn inner, + const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } // no array header: nothing rides + if ( r.get8() != 13 ) { return true; } // not an array of tables: §4's ordinary kind mismatch + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + if ( n > (uint64_t) INT32_MAX ) { reason = count_over_extent_cap; return false; } + const int64_t rest = length - r.offset; + if ( n > (uint64_t) ( rest / kTableMapEntryFloor ) ) { reason = count_over_length; return false; } // an N the map's L cannot carry + at = ( at + entry_align - 1 ) & ~( entry_align - 1 ); + at += (int64_t) n * entry_size; + if ( inner == NULL ) { return true; } // no map below an entry: one depth is the whole term + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } // framing damage: the load reports it + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +// ---- the TEXT form's placement (docs/SPEC-TABLES.md §2.8, §16) ---- +// +// The text is a plain JSON object keyed by the KEY, and the generic walk fills +// it through the ENTRY'S OWN descriptor — so all it needs from here is one +// entry at one key, handed back at its defaults. It is the builder's Insert +// with the ENTRY returned rather than its value, because the walk writes the +// value through a field row and not through a typed pointer. +// +// THE ONE INSERTION PRIMITIVE. Lookup, reset, allocation and the KEY COPY are +// all here, so no caller mutates an entry this did not create and no caller +// relabels one it found. A key is copied only when an entry is created, which +// is what makes a duplicate key leave the identity it matched untouched. NULL +// is one thing and one thing only: the arena refused. +template +inline Entry * TableMapPlace( TableWorker & worker, TableMap & map, Key key ) +{ + if ( worker.arena == NULL ) { return NULL; } + Entry * found = TableMapScan( *worker.arena, map, key ); + if ( found != NULL ) + { + TableResetMapValue( *found ); // a repeated key is LAST-WINS, whole + return found; + } + TableMapHead * head = TableMapReach( worker, map ); + if ( head == NULL ) { return NULL; } + Entry * entry = TableMapAppend( worker, head, map ); + if ( entry == NULL ) { return NULL; } + TableReset( *entry ); + TableEntrySetKey( *entry, key ); + return entry; +} + +// ---- the OPTIONAL RUNTIME INDEX (§2.8) ---- +// +// Open addressing with LINEAR PROBING over the sorted array, built AT LOAD for +// a map large enough that log n compares over a cold array cost more than one +// hash and a probe. IT IS NEVER STORED: the caller measures it, owns its +// storage, builds it in one pass and releases it whenever. +// +// ITS HASH AND ITS LOAD FACTOR ARE NOT A CROSS-PORT CONTRACT, and that is a +// rule. What a port is held to is the CONTRACT of the lookup: the same value +// the sorted array's Find returns for the same key, and no allocation past the +// storage the caller handed in. +struct TableMapIndex +{ + int32_t * slots = NULL; // entry indices, +1; 0 is an empty slot + int32_t capacity = 0; + bool good = false; +}; + +// this runtime's own, and no port reproduces it: fnv1a64 over the key's bytes +inline uint64_t TableMapHash( const void * bytes, int32_t length ) +{ + uint64_t hash = 0xCBF29CE484222325ull; + const uint8_t * at = (const uint8_t *) bytes; + for ( int32_t i = 0; i < length; i++ ) { hash ^= (uint64_t) at[i]; hash *= 0x100000001B3ull; } + return hash; +} +inline uint64_t TableMapHash( uint64_t key ) { return TableMapHash( (const void *) &key, (int32_t) sizeof( key ) ); } + +// this runtime's own load factor, and no port reproduces it either: the next +// power of two at or above twice the count, so a probe run stays short +inline int32_t TableMapIndexSlots( int32_t count ) +{ + int32_t slots = 8; + while ( slots < count * 2 ) { slots *= 2; } + return slots; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_MAP + +#ifndef MAPDEMO_SCHEMA_TABLE_LIST +#define MAPDEMO_SCHEMA_TABLE_LIST + +namespace mapdemo { + +// ---- an UNBOUNDED ARRAY: a counted array whose count the data decides (§2.9) ---- +// +// On the wire, in a region and in a cook a list is the kind 14 body a [..N]T +// writes, its elements by-value records inside the holder's node extent. What +// this adds is the slot, a builder that appends into segments that never +// move, and a const surface that indexes and iterates in place. There is no +// sort, no key and no lookup: the order is INSERTION order, and it is +// identity the way position is identity in a fixed array. + +// elements carved from ONE call to the allocator pair. A new segment is +// appended when the current one fills, and nothing ever moves (§6.4) +static const int32_t kTableListSegmentElements = 32; + +// THE ELEMENT STORAGE: T itself, and a TableRef slot for a []*T, whose +// elements are references exactly as a pointer field's slot is (§2.1) +template struct TableListStorage { typedef T Element; }; +template struct TableListStorage { typedef TableRef Element; }; + +// WHAT THE CONST FORM ANSWERS: the element by reference, and on a []*T the +// RESOLVED pointer, one add on the self-relative delta, NULL for a null slot, +// exactly as At answers it (§6.2, §6.3) +template struct TableListConst +{ + typedef const T & Result; + static Result At( const T * element ) { return *element; } +}; +template struct TableListConst +{ + typedef const T * Result; + static Result At( const TableRef * element ) + { + return element->value != 0 ? (const T *) ( (const uint8_t *) element + element->value ) : NULL; + } +}; + +// ---- the storage: SIXTEEN BYTES in the holder's record (§2.9, §7.2) ---- +// +// An int64 self-relative reference to the element array and an int32 count, +// then padding to eight. The reference is a TableRef like a pointer's: in the +// arena it names the builder's HEAD, in a region it is the delta from the slot +// to the first element, and 0 is the empty list in both. It is the map's slot +// exactly, because it is the same two facts. +template struct TableList +{ + typedef typename TableListStorage::Element Element; + + TableRef elements; + int32_t count = 0; // the LIVE count, in both forms + int32_t padding = 0; // named, so the record has no unwritten byte in it + + // ---- the CONST form: a locked region, a loaded one, an opened cook ---- + // + // One surface over one encoding (§6.3). A region reference resolves from + // the slot's own address, so every one of these is a member and needs no + // base and no context. + const Element * Elements() const + { + return elements.value != 0 ? (const Element *) ( (const uint8_t *) &elements + elements.value ) : NULL; + } + int32_t size() const { return count; } + + // INDEXING IS BOUNDS-CHECKED IN EVERY BUILD (§2.4, §2.9): the extent is a + // number that CAME FROM A FILE, so an index past it is not a mistake a + // release build gets to make cheaply. There is no undefined-behavior path + // here in any configuration. The assert carries the message where a + // debugger can read it and NDEBUG removes that. The fatal is what stands + // after it. Both go through the hooks: define schema_assert and + // schema_fatal and this refusal lands in your own handler. + void RefuseIndex( int32_t index ) const + { + if ( (uint32_t) index >= (uint32_t) count ) + { + schema_assert( false && "an unbounded array is indexed inside its count, which came from a file" ); + schema_fatal(); + } + } + typename TableListConst::Result operator[]( int32_t index ) const + { + RefuseIndex( index ); + return TableListConst::At( Elements() + index ); + } + + // ---- iteration: INDEX order, the element and no key ---- + // + // It carries no iterator_traits, for the reason TableKeyed's does not + // (§13.9). + struct ConstIterator + { + const Element * at; + typename TableListConst::Result operator*() const { return TableListConst::At( at ); } + ConstIterator & operator++() { at++; return *this; } + bool operator==( const ConstIterator & other ) const { return at == other.at; } + bool operator!=( const ConstIterator & other ) const { return at != other.at; } + }; + + ConstIterator begin() const { return ConstIterator{ Elements() }; } + ConstIterator end() const { return ConstIterator{ Elements() + count }; } +}; + +// ---- the BUILDER's side: a head, and segments that never move (§2.9, §6.4) ---- +// +// The head is a small node in the arena holding the segment chain, the live +// count and the dead count, allocated when the first element is added. Each +// segment is a fixed number of elements carved from one call to the allocator +// pair. An element's address is stable for the arena's life, so a T * handed +// back by Add stays valid while other elements arrive. +struct TableListHead +{ + TableRef first; // the arena offset of the first segment + TableRef last; // and of the one an Add appends into + int32_t live; + int32_t dead; +}; + +template struct TableListSegment +{ + TableRef next; + int32_t used; // elements carved from this segment + int32_t padding; + uint32_t dead[ ( kTableListSegmentElements + 31 ) / 32 ]; // Erase marks one bit, never the element + Element elements[ kTableListSegmentElements ]; +}; + +inline bool TableListSegmentDead( const uint32_t * dead, int32_t index ) +{ + return ( dead[ index / 32 ] & ( 1u << ( index % 32 ) ) ) != 0; +} + +// the head, allocated when the first element is added +template +inline TableListHead * TableListReach( TableWorker & worker, TableList & list ) +{ + if ( worker.arena == NULL || worker.arena->locked ) { return NULL; } + if ( list.elements.value != 0 ) { return (TableListHead *) TableArenaAt( *worker.arena, (uint32_t) list.elements.value ); } + uint32_t at = 0; + TableListHead * head = (TableListHead *) worker.AllocRaw( (int64_t) sizeof( TableListHead ), (int64_t) alignof( TableListHead ), at ); + if ( head == NULL ) { return NULL; } + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + list.elements.value = (int64_t) at; + return head; +} + +// one element's storage, appended: the current segment when it has room, a +// new one carved from one call to the pair when it does not. NULL means NOT +// ADDED: an arena that cannot carve another segment, or a count at the int32 +// cap (§2.2, §2.9). +template +inline typename TableList::Element * TableListAppend( TableWorker & worker, TableListHead * head, TableList & list ) +{ + typedef typename TableList::Element Element; + if ( list.count >= INT32_MAX ) { return NULL; } // the int32 storage cap + TableListSegment * segment = NULL; + if ( head->last.value != 0 ) + { + segment = (TableListSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + if ( segment->used >= kTableListSegmentElements ) { segment = NULL; } + } + if ( segment == NULL ) + { + uint32_t at = 0; + segment = (TableListSegment *) worker.AllocRaw( (int64_t) sizeof( TableListSegment ), (int64_t) alignof( TableListSegment ), at ); + if ( segment == NULL ) { return NULL; } // the arena could not carve another segment + segment->next.value = 0; + segment->used = 0; + segment->padding = 0; + for ( int32_t i = 0; i < (int32_t) ( sizeof( segment->dead ) / sizeof( segment->dead[0] ) ); i++ ) { segment->dead[i] = 0; } + if ( head->last.value != 0 ) + { + TableListSegment * previous = (TableListSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + previous->next.value = (int64_t) at; + } + else + { + head->first.value = (int64_t) at; + } + head->last.value = (int64_t) at; + } + Element * element = segment->elements + segment->used; + segment->used++; + head->live++; + list.count++; + return element; +} + +// ADD, whole: the head, the append, and the element at its declared defaults +// (§2.9). The text form's placement is this same call, because a list has no +// key to place under (§16). +template +inline typename TableList::Element * TableListPlace( TableWorker & worker, TableList & list ) +{ + typedef typename TableList::Element Element; + TableListHead * head = TableListReach( worker, list ); + if ( head == NULL ) { return NULL; } + Element * element = TableListAppend( worker, head, list ); + if ( element == NULL ) { return NULL; } + new ( element ) Element(); // value-init: the declared defaults, and null for a slot + return element; +} + +// ERASE, ADDRESSED BY THE POINTER (§2.9): the element Add handed back is the +// handle, because a list has no key and the address is the one thing the +// builder promises never moves (§6.4). It marks the element DEAD, one bit in +// the segment's slot and not in the element storage, and decrements the live +// count. False when the pointer is not this list's. Its storage is reclaimed +// at RESET and never reused mid-build, the map's rule for the map's reason. +template +inline bool TableListErase( TableArena & arena, TableList & list, const typename TableList::Element * element ) +{ + typedef typename TableList::Element Element; + if ( list.elements.value == 0 || element == NULL ) { return false; } + TableListHead * head = (TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableListSegment * segment = (TableListSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + if ( element >= segment->elements && element < segment->elements + segment->used ) + { + const int32_t i = (int32_t) ( element - segment->elements ); + if ( TableListSegmentDead( segment->dead, i ) ) { return false; } // already erased + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + list.count--; + return true; + } + segment_ref = segment->next; + } + return false; +} + +// ---- iterate on the BUILDER: INDEX order, live elements only (§2.9) ---- +template struct TableListEach +{ + typedef typename TableList::Element Element; + const TableArena * arena; + TableRef first; + + struct Iterator + { + const TableArena * arena; + TableListSegment * segment; + int32_t index; + + void Skip() + { + for ( ;; ) + { + if ( segment == NULL ) { return; } + if ( index >= segment->used ) + { + segment = segment->next.value != 0 ? (TableListSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + index = 0; + continue; + } + if ( TableListSegmentDead( segment->dead, index ) ) { index++; continue; } + return; + } + } + Element * operator*() const { return segment->elements + index; } + Iterator & operator++() { index++; Skip(); return *this; } + bool operator==( const Iterator & other ) const { return segment == other.segment && index == other.index; } + bool operator!=( const Iterator & other ) const { return !( *this == other ); } + }; + + Iterator begin() const + { + Iterator it = { arena, first.value != 0 ? (TableListSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL, 0 }; + it.Skip(); + return it; + } + Iterator end() const { Iterator it = { arena, NULL, 0 }; return it; } +}; + +template +inline TableListEach TableListEachOf( const TableArena & arena, const TableList & list ) +{ + TableListEach each = { &arena, TableRef() }; + if ( list.elements.value != 0 ) + { + const TableListHead * head = (const TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + each.first = head->first; + } + return each; +} + +// ---- the INDEX-ORDER CURSOR the four writing walks read (§2.9) ---- +// +// Measure, Save, Lock and Cook each visit a list's live elements in the order +// they were added, and they allocate nothing to do it: a region's cursor is +// the array in place, and the builder's walks the segment chain. Indexing the +// builder's form is SEQUENTIAL by construction, every walk steps i, i + 1, +// i + 2, so the cursor remembers where the last access landed and moves one +// live slot per step. An access behind the memo restarts from the first +// segment, which no walk here does. +template struct TableListCursor +{ + const Element * elements = NULL; // the region's form: the array in place + const TableArena * arena = NULL; // the builder's form: the segments + TableRef first; + int32_t count = 0; + bool ok = false; + // the memo: the segment and slot the last access landed on, and the live + // index that slot holds + mutable const TableListSegment * segment = NULL; + mutable int32_t within = -1; + mutable int32_t logical = -1; + + const Element * At( int32_t index ) const + { + if ( elements != NULL ) { return elements + index; } + if ( segment == NULL || index < logical ) + { + segment = first.value != 0 ? (const TableListSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL; + within = -1; + logical = -1; + } + while ( logical < index ) + { + for ( ;; ) + { + within++; + while ( segment != NULL && within >= segment->used ) + { + segment = segment->next.value != 0 ? (const TableListSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + within = 0; + } + if ( segment == NULL ) { return NULL; } // the slot and the head disagree + if ( !TableListSegmentDead( segment->dead, within ) ) { break; } + } + logical++; + } + return segment->elements + within; + } + const Element & operator[]( int32_t index ) const { return *At( index ); } +}; + +// the REGION form: the array is the cursor +template +inline TableListCursor::Element> TableListElements( const TableRegionCtx &, const TableList & list ) +{ + TableListCursor::Element> cursor; + cursor.elements = list.Elements(); + cursor.count = list.count; + cursor.ok = true; + return cursor; +} + +// the BUILDER's form: the live elements out of the segment chain, in the +// order they were added. A dead element costs nothing on any wire (§2.9). +template +inline TableListCursor::Element> TableListElements( const TableArena & arena, const TableList & list ) +{ + TableListCursor::Element> cursor; + cursor.arena = &arena; + cursor.count = list.count; + if ( list.elements.value == 0 || list.count <= 0 ) { cursor.ok = list.count == 0; cursor.count = 0; return cursor; } + const TableListHead * head = (const TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + if ( head->live != list.count ) { return cursor; } // the slot and the head disagree: refused, never guessed + cursor.first = head->first; + cursor.ok = true; + return cursor; +} + +template +inline TableListCursor::Element> TableListElements( const TableArenaCtx & ctx, const TableList & list ) +{ + return TableListElements( *ctx.arena, list ); +} + +// ---- the LOAD side: where a decoded element lands (§2.9) ---- +// +// The same two shapes the map's fill takes, because the decoder above them +// cannot tell which it has: a REGION carves the element array out of the +// holder node's own extent, PRE-ORDER, and the TOOL's path appends into the +// builder's arena. +template struct TableListFill +{ + typedef typename TableList::Element Element; + TableList * list = NULL; + Element * array = NULL; // the region path: the carved array + int32_t capacity = 0; + TableWorker * worker = NULL; // the TOOL's path + bool ok = false; + bool refused = false; // a count above the int32 cap on the tool's path: LoadBuilder answers NULL +}; + +template +inline TableListFill TableListFillBegin( const TableNodeMap & nodes, TableList & list, uint64_t n ) +{ + typedef typename TableList::Element Element; + TableListFill fill; + fill.list = &list; + list.elements.value = 0; + list.count = 0; + if ( nodes.carve == NULL ) { return fill; } + if ( n > (uint64_t) INT32_MAX ) + { + // A COUNT ABOVE THE int32 STORAGE CAP (§2.2, §2.9): into a region it was + // refused by LoadMeasure before this ran, and into a builder it is the + // refusal LoadBuilder answers NULL for, moving no counter + fill.refused = nodes.carve->worker != NULL; + return fill; + } + if ( nodes.carve->worker != NULL ) + { + fill.worker = nodes.carve->worker; // the tool's path: the arena carves + fill.ok = true; + return fill; + } + const int64_t align = (int64_t) alignof( Element ); + uint8_t * base = (uint8_t *) ( ( (uintptr_t) nodes.carve->at + (uintptr_t) ( align - 1 ) ) & ~( (uintptr_t) ( align - 1 ) ) ); + const int64_t bytes = (int64_t) n * (int64_t) sizeof( Element ); + const int64_t pad = (int64_t) ( base - nodes.carve->at ); + if ( pad + bytes > nodes.carve->left ) { return fill; } // the measure and the load disagree: refused + nodes.carve->at = base + bytes; + nodes.carve->left -= pad + bytes; + fill.array = (Element *) base; + fill.capacity = (int32_t) n; + list.elements.value = (int64_t) ( base - (const uint8_t *) &list.elements ); + fill.ok = true; + return fill; +} + +// the next slot, at the element's declared defaults. NULL when the arena +// could not carve, which the decoder reports as framing damage +template inline typename TableList::Element * TableListFillNext( TableListFill & fill ) +{ + typedef typename TableList::Element Element; + if ( fill.array != NULL ) + { + if ( fill.list->count >= fill.capacity ) { return NULL; } + Element * element = fill.array + fill.list->count; + new ( element ) Element(); + fill.list->count++; + return element; + } + return TableListPlace( *fill.worker, *fill.list ); +} + +// A SLOT WHOSE ELEMENT NEVER LANDED is given back (§2.9, §4): the array keeps +// what it decoded, and an element whose own framing gave out before one byte +// of it decoded was not decoded. The region's form uncounts it, and the builder's +// marks it dead, which is what the storage rule allows mid-build. +template inline void TableListFillDrop( TableListFill & fill ) +{ + typedef typename TableList::Element Element; + if ( fill.array != NULL ) + { + if ( fill.list->count > 0 ) { fill.list->count--; } + return; + } + if ( fill.list->elements.value == 0 ) { return; } + TableListHead * head = (TableListHead *) TableArenaAt( *fill.worker->arena, (uint32_t) fill.list->elements.value ); + if ( head->last.value == 0 ) { return; } + TableListSegment * segment = (TableListSegment *) TableArenaAt( *fill.worker->arena, (uint32_t) head->last.value ); + if ( segment->used <= 0 ) { return; } + const int32_t i = segment->used - 1; + if ( TableListSegmentDead( segment->dead, i ) ) { return; } + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + fill.list->count--; +} + +// an EMPTY list's reference is null in both encodings, so a load that placed +// nothing leaves the slot exactly as a Reset does +template inline void TableListFillEnd( TableListFill & fill ) +{ + if ( fill.array != NULL && fill.list->count == 0 ) { fill.list->elements.value = 0; } +} + +// ---- LoadMeasure's term, from the FRAMING alone (§2.9, §6.5) ---- +// +// N x sizeof( T ) rounded to alignof( T ), AT EVERY DEPTH. N is framing and +// not a value, so this reads no field: it walks the list's own header and, +// where a table element holds a list or a map of its own, the elements' +// headers under it. Every -1 carries its REASON (§6.5): the int32 cap first, +// because a count past it cannot fit any body, and then the body's own L. +inline bool TableListWireExtent( const uint8_t * body, int64_t length, int64_t & at, + int64_t elem_size, int64_t elem_align, uint8_t elem_kind, int64_t elem_floor, + TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } // no array header: nothing rides + if ( r.get8() != elem_kind ) { return true; } // another element kind: §4's ordinary kind mismatch, the field reads empty + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + if ( n > (uint64_t) INT32_MAX ) { reason = count_over_extent_cap; return false; } + const int64_t rest = length - r.offset; + if ( n > (uint64_t) ( rest / elem_floor ) ) { reason = count_over_length; return false; } // an N the list's L cannot carry + at = ( at + elem_align - 1 ) & ~( elem_align - 1 ); + at += (int64_t) n * elem_size; + if ( inner == NULL ) { return true; } // nothing below an element: one depth is the whole term + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } // framing damage: the load reports it + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_LIST + +#ifndef MAPDEMO_SCHEMA_BUILD_VERSION +#define MAPDEMO_SCHEMA_BUILD_VERSION + +namespace mapdemo { + +// THE BUILD VERSION (docs/SPEC-TABLES.md §20): one digest over every fact the bytes +// this build produces depend on — the type wire's protocol id, every record's +// layout as the compiler's own C ABI model computes it, and the facts that +// decide what a load PUTS in those slots. It is the number a cook's header +// carries and the number Open compares, and the number a block's prologue +// carries and BlockOpen compares: a build version answers "which build?" and +// not "which form?", and what separates the two forms is their MAGIC. +// +// There are TWO ids in the design and they are not interchangeable: the +// PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is +// what everything cooked or blocked is keyed by. A table edit moves this and +// never the protocol id; a type edit moves both. +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_BUILD_VERSION + +#ifndef MAPDEMO_SCHEMA_TABLE_COOK +#define MAPDEMO_SCHEMA_TABLE_COOK + +namespace mapdemo { + +// ---- the cooked form (docs/SPEC-TABLES.md §7) ---- +// +// A cooked file is a HEADER, a DATA part and an ATTRIBUTION part, in that +// order. Every word of the header is a u64 written in the byte order the cook +// was produced in, and the header is 64 bytes: +// +// 0 magic 0x4b4f4f434d484353, read BYTEWISE before anything else +// 8 build_version the unit's id (docs/SPEC-TABLES.md §20) +// 16 byte_order 1 little, 2 big — the order that WROTE the file +// 24 data_length the region's bytes, rounded up to alignment +// 32 attribution_length the directory's bytes, or 0 +// 40 alignment the region's alignment, never below eight +// 48 reserved zero +// 56 reserved zero +// +// The DATA part is Lock's region written verbatim (§7.2) — the root at its +// base — and it is what a runtime points at. The ATTRIBUTION part is the node +// directory (§6.3), and NOTHING THAT READS THE STRUCTURE TOUCHES IT: it is +// written beside the data for schema cook-check, so a build that ships no +// tooling need not carry it at all. +static const int64_t kTableCookHeaderBytes = 64; + +// THE MAGIC'S VALUE, and a consumer written from the page needs the constant +// rather than a description of one. It is "SCHMCOOK" read as ASCII in the byte +// order a little-endian store produces — the same shape the block form's +// SCHMABLK takes, so a hex dump of a little-endian cook is legible and the two +// accelerators sit in one vocabulary. +// +// IT IS STORED IN THE PRODUCER'S ORDER, which is what makes it the byte-order +// check as well as the form check: a consumer reads back this build's +// constant, or that constant byte-reversed — which identifies a cook of the +// OTHER order — or something that is not a cook. All three answers but the +// first refuse, and a cook and a BLOCK are separated here too, because a +// form's identity belongs in its magic rather than in a second digest. +static const uint64_t TableCookMagic = 0x4b4f4f434d484353ull; + +// THIS BUILD's byte order, as the header's own word carries it. The magic is +// what REFUSES a foreign order; this word is what RECORDS which order wrote +// the file, so a refusal names the order rather than inferring it and a tool +// dumping a cook reads the fact. A file whose magic matched and whose order +// word did not is corrupt, and there is no reading that recovers it. +// +// The BUILD VERSION cannot do either job: §20.1 digests byteorder as a +// GENERATION input, little for every target schema generates for today, so +// two builds of one schema for two orders emit the same id. +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +static const uint64_t TableCookByteOrder = 2; // big +#else +static const uint64_t TableCookByteOrder = 1; // little +#endif + +// The greatest region alignment a cooked file may name. The DATA part begins +// at align_up( 64, alignment ), which is 64 for every unit this language can +// declare — the largest alignment it has is sixteen — so a word past this cap +// describes a file no build of this schema wrote (docs/SPEC-TABLES.md §7.1). +static const uint64_t TableCookMaxAlign = 64; + +// The header read, BYTEWISE. memcpy is the portable spelling of "these eight +// bytes, in this machine's order"; every compiler this repo builds under folds +// it to one load, and it is the only read in the whole of Open that is not a +// comparison. +inline uint64_t table_cook_read64( const uint8_t * p ) +{ + uint64_t v; + memcpy( &v, p, sizeof( v ) ); + return v; +} + +// TableCookOpen: THE WHOLE CHECK, in one place, because §7 states the +// enumeration once and every generated Open is that one enumeration plus +// its own root's two layout facts. +// +// THE CHECK, in order: the magic read bytewise, the byte order it establishes, +// the build version against this build's own, both RESERVED words zero, the +// region alignment the header names, the two part lengths against the length +// the caller passed — a truncated file and a file with trailing bytes are the +// same refusal — the root's own storage inside the data part, and the +// alignment of the base. +// +// AND THAT IS ALL OF IT. On a match the bytes ARE what this build wrote, in +// this build's layout and this build's byte order, so there is nothing to +// validate and nothing to fix up: the caller gets the root. Nothing per node +// happens here, which is what makes open O(1) in the file's size; a walk of +// any shape would forfeit that, and validating an untrusted file is schema +// cook-check's job and a person's decision (§7.4). +// +// EVERY NUMBER BELOW COMES OUT OF THE FILE, so the arithmetic is unsigned and +// each term is BOUNDED BEFORE IT IS ADDED: a forged length near 2^64 must +// refuse, and an addition that wrapped would be the defect the comparison +// after it was supposed to catch. Nothing past length is read on any path, +// including every refusing one. +// A REFUSAL NAMES ITSELF, beside the null (docs/SPEC-TABLES.md §7): the reason +// is written on the refusal path only, so a match costs nothing and a caller +// that passed no out-parameter pays nothing. +inline const uint8_t * TableCookRefuse( TableRefuseReason * reason, TableRefuseReason why ) +{ + if ( reason != NULL ) { *reason = why; } + return NULL; +} + +inline uint64_t table_cook_byteswap64( uint64_t v ) +{ + return ( v >> 56 ) | ( ( v >> 40 ) & 0xff00ull ) | ( ( v >> 24 ) & 0xff0000ull ) | ( ( v >> 8 ) & 0xff000000ull ) + | ( ( v << 8 ) & 0xff00000000ull ) | ( ( v << 24 ) & 0xff0000000000ull ) | ( ( v << 40 ) & 0xff000000000000ull ) + | ( v << 56 ); +} + +inline const uint8_t * TableCookOpen( const void * bytes, uint64_t length, uint64_t root_size, uint64_t root_align, TableRefuseReason * reason ) +{ + // a null buffer is the CALLER's defect, as an unaligned base is; a buffer + // shorter than the header has no header to read and is truncated + if ( bytes == NULL ) { return TableCookRefuse( reason, unaligned_base ); } + if ( length < (uint64_t) kTableCookHeaderBytes ) { return TableCookRefuse( reason, truncated ); } + const uint8_t * raw = (const uint8_t *) bytes; + // the MAGIC, bytewise and first: it is what establishes the byte order + // every other header word is read in, so nothing else may be read before + // it. A byte-reversed constant is a cook of the other order and refuses + // here, which is why the order never reaches a fix-up pass; anything else + // is not a cook at all, a BLOCK's magic included. + const uint64_t magic = table_cook_read64( raw ); + if ( magic != TableCookMagic ) + { + return TableCookRefuse( reason, magic == table_cook_byteswap64( TableCookMagic ) ? foreign_order : not_a_cook ); + } + // a byte-order word that contradicts its own magic describes no cook in + // EITHER order, so it shares the magic's own value (§7.1) + if ( table_cook_read64( raw + 16 ) != TableCookByteOrder ) { return TableCookRefuse( reason, not_a_cook ); } + if ( table_cook_read64( raw + 8 ) != BuildVersion ) { return TableCookRefuse( reason, wrong_build_version ); } + // the RESERVED words: a non-zero one means a writer used a form this build + // does not understand, and Open refuses rather than ignoring it. + if ( table_cook_read64( raw + 48 ) != 0 ) { return TableCookRefuse( reason, reserved_not_zero ); } + if ( table_cook_read64( raw + 56 ) != 0 ) { return TableCookRefuse( reason, reserved_not_zero ); } + const uint64_t data_length = table_cook_read64( raw + 24 ); + const uint64_t attribution_length = table_cook_read64( raw + 32 ); + const uint64_t alignment = table_cook_read64( raw + 40 ); + // THE ALIGNMENT WORD IS DATA, and it is the one header field the rest of + // the check does arithmetic WITH rather than only comparison against. A + // region's alignment is a power of two, never below eight (the floor that + // puts the attribution part on an eight-byte boundary without a second + // padding rule) and never past the cap above; a word that is none of those + // rounds nothing and aligns nothing, so it is refused before it is used, + // which is why bad_alignment precedes both truncated clauses (§7). + if ( alignment < 8 || alignment > TableCookMaxAlign ) { return TableCookRefuse( reason, bad_alignment ); } + if ( ( alignment & ( alignment - 1 ) ) != 0 ) { return TableCookRefuse( reason, bad_alignment ); } + // and it must be an alignment THE ROOT CAN SIT AT, since the root is at + // the region's base: both are powers of two, so "at least the root's" + // is one division. + if ( ( alignment % root_align ) != 0 ) { return TableCookRefuse( reason, bad_alignment ); } + // The DATA part begins at align_up( 64, alignment ). It is DERIVED and not + // a header field, because a fact a reader computes is a fact two writers + // cannot disagree about. + const uint64_t data_offset = ( (uint64_t) kTableCookHeaderBytes + alignment - 1 ) & ~( alignment - 1 ); + if ( length < data_offset ) { return TableCookRefuse( reason, truncated ); } + // the two part lengths against the length the caller passed. The whole + // file is data_offset + data_length + attribution_length, and a length + // that is not EXACTLY that refuses — truncation and trailing bytes are one + // refusal, and both terms are subtracted rather than added so no sum can + // carry. + if ( data_length > length - data_offset ) { return TableCookRefuse( reason, truncated ); } + if ( attribution_length != length - data_offset - data_length ) { return TableCookRefuse( reason, truncated ); } + // the ROOT sits at the region's base, so the region has to hold it: a + // shorter data part describes a root partly outside the file, which is the + // one way a match-and-point reader could hand back storage it never + // received. It is the second clause on truncated (§7). + if ( data_length < root_size ) { return TableCookRefuse( reason, truncated ); } + const uint8_t * base = raw + data_offset; + // the alignment of the BASE, LAST, because it is the only clause that reads + // nothing out of the file. The header pads the data part to the region's + // alignment, so a base an allocator or mmap gave you is already aligned — + // mmap gives page alignment for free — and a base that is not is a caller's + // buffer this form cannot be read out of: the caller's defect, not the file's. + if ( ( (uintptr_t) base % (uintptr_t) alignment ) != 0 ) { return TableCookRefuse( reason, unaligned_base ); } + return base; +} + +// ---- the cooked form, the WRITE side (docs/SPEC-TABLES.md §7.6) ---- +// +// THE BYTE ORDER IS THE TARGET'S, NOT THE HOST'S. A cook is produced in the +// byte order of the build that will read it (§7), so the fixing happens here — +// offline, once, on the writing side — and never at Open. Passing +// TableByteOrder::Big on a little-endian machine produces a big-endian build's +// file, and nothing about the writing host reaches the bytes. +enum class TableByteOrder +{ + Little = 1, // the header's byte_order word, and the order every scalar is written in + Big = 2, +}; + +// One store, width as an argument. Every call site passes a literal width, so +// the loop folds to a store (and a byte swap on the foreign order); a name per +// width would claim four §11 names to save nothing. +inline void table_cook_put( uint8_t * at, uint64_t value, int32_t width, TableByteOrder order ) +{ + if ( order == TableByteOrder::Little ) + { + for ( int32_t i = 0; i < width; i++ ) { at[i] = (uint8_t) ( value >> ( 8 * i ) ); } + } + else + { + for ( int32_t i = 0; i < width; i++ ) { at[i] = (uint8_t) ( value >> ( 8 * ( width - 1 - i ) ) ); } + } +} + +// A 128-bit store as two lanes: sixteen bytes, the low lane first in the +// little order and the high lane first — each lane big-endian — in the big +// order, exactly as a u64 is one lane of eight (docs/SPEC-TABLES.md §7.2). +inline void table_cook_put128( uint8_t * at, uint64_t lo, uint64_t hi, TableByteOrder order ) +{ + if ( order == TableByteOrder::Little ) { table_cook_put( at, lo, 8, order ); table_cook_put( at + 8, hi, 8, order ); } + else { table_cook_put( at, hi, 8, order ); table_cook_put( at + 8, lo, 8, order ); } +} + +// A buffer piece: the USED bytes and nothing else. The tail is already zero — +// the whole extent was zeroed before any field was written — so this copies the +// used prefix and leaves the rest, which is what makes a string's unused tail a +// consequence of one memset rather than a rule per buffer. A used length past +// the buffer, or below zero, is a value no reader could have produced and it is +// clamped rather than trusted: this writes inside the caller's buffer on every +// input. +inline void table_cook_bytes( uint8_t * at, const void * source, int64_t used, int64_t capacity ) +{ + if ( used <= 0 ) { return; } + const int64_t n = used < capacity ? used : capacity; + memcpy( at, source, (size_t) n ); +} + +// A WIDE TEXT buffer piece (docs/SPEC-TABLES.md §7.2): the USED code units, +// each a TWO-BYTE SCALAR in the cook's byte order. A record is written piece +// by piece and never memcpy'd, and a swap has to know where every scalar +// begins — a char16_t is one, so the units go one store each rather than as +// bytes. The tail is already zero, as the narrow twin's is, so the terminating +// zero unit at index used costs nothing here. +inline void table_cook_units( uint8_t * at, const char16_t * source, int64_t used, int64_t capacity, TableByteOrder order ) +{ + if ( used <= 0 ) { return; } + const int64_t n = used < capacity ? used : capacity; + for ( int64_t i = 0; i < n; i++ ) { table_cook_put( at + i * 2, (uint64_t) (uint16_t) source[i], 2, order ); } +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_COOK + +#ifndef MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE +#define MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE + +namespace mapdemo { + +// ---- the cooked form's WRITE side for a POINTERED root (docs/SPEC-TABLES.md §7.6) ---- +// +// A pointered root's cook is the region of §7.2: every node the numbering +// reached (§3.1), once, at its own type's alignment, in index order, the root +// at offset zero. This is that region while it is being laid out and written — +// the tool's own Layout and Write, in one struct. +// +// The OFFSETS are one per node, the root's zero at position 0 and node index k +// at position k - 1, which is the directory's own order (§6.3); they are the +// one allocation the write makes beyond the numbering, and they go through the +// same pair. A measure needs no offsets and leaves the pointer NULL. +struct TableCookRegion +{ + const TableNumbering * numbering = NULL; // node -> index, from the walk that placed it + int64_t * offsets = NULL; // index - 1 -> the node's region offset; NULL while measuring + int64_t count = 0; // nodes, the root included + int64_t bytes = 0; // the data part's length, rounded to align + int64_t align = 0; // the region's alignment: the nodes' greatest, never below eight + uint8_t * base = NULL; // where the data part is being written; NULL while measuring +}; + +// A reference slot: the SELF-RELATIVE delta from the slot's own address to the +// node's start (§6.3), and zero for null. The node is found by the address the +// numbering keyed it under, which is the same address the walk resolved through +// the same context — so a reference the numbering does not carry is a slot the +// walk never reached (a counted array's slot past its count, an absent +// optional's value) holding a node the region will not hold, and it is refused +// rather than written as a delta to nowhere. +inline bool table_cook_ref( const TableCookRegion & region, uint8_t * at, const void * pointee, TableByteOrder order ) +{ + if ( pointee == NULL ) { table_cook_put( at, 0, 8, order ); return true; } + uint64_t index = 0; + if ( !TableNumberingIndex( *region.numbering, pointee, index ) ) { return false; } + if ( index == 0 || index > (uint64_t) region.count ) { return false; } + const int64_t delta = region.offsets[index - 1] - (int64_t) ( at - region.base ); + table_cook_put( at, (uint64_t) delta, 8, order ); + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE + +namespace mapdemo { + +// table PairsSlotsEntry — TABLE-wire storage: relocatable, bounded, defaults in the +// member initializers (docs/SPEC-TABLES.md) +struct PairsSlotsEntry { + uint32_t key = 0; + TableRef value[2]; // [2]*Item — every slot null until assigned +}; + +// table Pairs — TABLE-wire storage: relocatable, bounded, defaults in the +// member initializers (docs/SPEC-TABLES.md) +struct Pairs { + TableMap slots; // map[uint32]*Item — the sorted entry array, empty until an insert + int32_t after = 0; +}; + +// ---- prefill: the declared defaults, in place (docs/SPEC-TABLES.md) ---- + +inline void PairsSlotsEntryReset( PairsSlotsEntry & value ); +inline void PairsReset( Pairs & value ); + +inline void PairsSlotsEntryReset( PairsSlotsEntry & value ) +{ + value.key = 0; + for ( int32_t i = 0; i < 2; i++ ) { value.value[i].value = 0; } // [2]*Item — every slot null +} + +inline void PairsReset( Pairs & value ) +{ + value.slots.entries.value = 0; // map[uint32]*Item: empty + value.slots.count = 0; + value.slots.padding = 0; + value.after = 0; +} + +template inline int64_t PairsSlotsEntryMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const PairsSlotsEntry & value ); +template inline bool PairsSlotsEntrySaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const PairsSlotsEntry & value ); +inline bool PairsSlotsEntryLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, PairsSlotsEntry & value ); +template inline int64_t PairsMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Pairs & value ); +template inline bool PairsSaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Pairs & value ); +inline bool PairsLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Pairs & value ); +inline bool PairsMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ); + +// PairsSlotsEntryMessageKeyRead: the key of one entry on the message wire, before the +// slot is chosen (docs/SPEC-TABLES.md §2.8, §3.3), and the bit the entry's +// body ends at. Field order inside a body is not contractual, so this scans +// the whole body by its announced shapes rather than assuming a position. +struct PairsSlotsEntryMessageKeyRead +{ + uint32_t key; + int64_t end; // the bit after the entry's own zero reference + bool found; // the body carried the key's id + bool kind_bad; // it carried it under another kind: the MAP's event + bool widened; // under a kind the declaration WIDENS (§4): decoded exactly, the MAP counts one + bool over; // longer than this reader's bound: the ENTRY is dropped + bool malformed; // the entry's framing gave out +}; + +inline PairsSlotsEntryMessageKeyRead PairsSlotsEntryMessageReadKey( TableBitReader r, const TableVocabulary & vocabulary, int64_t index_bits ) +{ + PairsSlotsEntryMessageKeyRead out = { 0, 0, false, false, false, false, false }; + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { out.malformed = true; return out; } + if ( ref == 0 ) { out.end = r.offset; return out; } // the terminator: no key field is the key's DEFAULT + if ( ref > (uint64_t) vocabulary.count ) { out.malformed = true; return out; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { out.malformed = true; return out; } + if ( entry.id == 0x3dc94a19365b10ecull ) // `key`, the ordinary hash of an ordinary name + { + const bool kind_bad = entry.kind != 8 && !TableKindWidens( entry.kind, 8 ); // THE KEY KIND IS THE READER'S DECLARATION + out.kind_bad = kind_bad; + out.found = !kind_bad; + out.widened = entry.kind != 8; + if ( kind_bad ) + { + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { out.malformed = true; return out; } + continue; + } + { + uint64_t raw = 0; + const int64_t width = entry.value_bits; + if ( width < 0 || !r.get( raw, width ) ) { out.malformed = true; return out; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + out.key = (uint32_t) decoded_wide; + } + continue; // the LAST occurrence is the one §3 keeps + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { out.malformed = true; return out; } + } +} + +// ---- the arena's reset hook (docs/SPEC-TABLES.md §6) ---- +// +// TableWorker::Alloc is a template and cannot name a member's Reset, so +// the arena reaches it through this overload set by argument-dependent +// lookup. It is how a node born in raw arena storage comes to hold the +// declared defaults without value-initialising the whole aggregate. + +inline void TableReset( PairsSlotsEntry & value ) { PairsSlotsEntryReset( value ); } +inline void TableReset( Pairs & value ) { PairsReset( value ); } + +// ---- pointer targets: allocation and resolution (docs/SPEC-TABLES.md §2) ---- +// +// A reference resolves differently in the two forms, and the CONTEXT says +// which: in the arena it is an offset; in a region it is a self-relative +// delta, so the const deref below is one add and needs no base pointer. + +// ---- codecs: measure/save/load per closure member ---- + +template inline int64_t PairsSlotsEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const PairsSlotsEntry & value ); +template inline bool PairsSlotsEntrySaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const PairsSlotsEntry & value ); +template inline bool PairsSlotsEntrySaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const PairsSlotsEntry & value ); +inline bool PairsSlotsEntryLoadBody( TableReader & r, const TableNodeMap & nodes, PairsSlotsEntry & value ); +template inline int64_t PairsMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Pairs & value ); +template inline bool PairsSaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Pairs & value ); +template inline bool PairsSaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Pairs & value ); +inline bool PairsLoadBody( TableReader & r, const TableNodeMap & nodes, Pairs & value ); + +// ---- pointer-graph walkers: number (measure/save), pack (Lock) ---- + +template inline bool PairsSlotsEntryNumber( const Ctx & ctx, TableNumbering & numbering, const PairsSlotsEntry & value ); +template inline int64_t PairsSlotsEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const PairsSlotsEntry & value ); +template inline bool PairsSlotsEntryPack( const Ctx & ctx, TablePackMap & seen, const PairsSlotsEntry & src, PairsSlotsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); +template inline bool PairsNumber( const Ctx & ctx, TableNumbering & numbering, const Pairs & value ); +template inline int64_t PairsPackMeasure( const Ctx & ctx, TablePackMap & seen, const Pairs & value ); +template inline bool PairsPack( const Ctx & ctx, TablePackMap & seen, const Pairs & src, Pairs & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +// ---- the numbering's bridge to each member's codec (docs/SPEC-TABLES.md §3.1) ---- + +template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const PairsSlotsEntry & value ) { return PairsSlotsEntryMeasureBody( ctx, numbering, ids, value ); } +template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const PairsSlotsEntry & value ) { return PairsSlotsEntrySaveBody( ctx, numbering, w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const PairsSlotsEntry & value ) { return PairsSlotsEntryMeasureMessageBody( ctx, numbering, index_bits, at, value ); } +template inline bool TableNodeMessageSave( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const PairsSlotsEntry & value ) { return PairsSlotsEntrySaveMessageBody( ctx, numbering, index_bits, w, value ); } +template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Pairs & value ) { return PairsMeasureBody( ctx, numbering, ids, value ); } +template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Pairs & value ) { return PairsSaveBody( ctx, numbering, w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Pairs & value ) { return PairsMeasureMessageBody( ctx, numbering, index_bits, at, value ); } +template inline bool TableNodeMessageSave( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Pairs & value ) { return PairsSaveMessageBody( ctx, numbering, index_bits, w, value ); } + +// ---- PairsSlotsEntry: the order, the key and the value (docs/SPEC-TABLES.md §2.8) ---- +// +// The four overloads the map runtime's templates reach by argument-dependent +// lookup. Nothing outside this file names them. +static_assert( alignof( PairsSlotsEntry ) <= kTableAlign, "a map entry's alignment must fit the arena's" ); + +inline int TableEntryOrder( const PairsSlotsEntry & a, const PairsSlotsEntry & b ) +{ + return TableKeyOrder( (uint64_t) a.key, (uint64_t) b.key ); // integers compare by VALUE, unsigned here +} +inline int TableEntryOrder( const PairsSlotsEntry & entry, uint32_t key ) +{ + return TableKeyOrder( (uint64_t) entry.key, (uint64_t) key ); +} +inline uint32_t TableEntryKey( const PairsSlotsEntry & entry ) { return entry.key; } +inline void TableEntrySetKey( PairsSlotsEntry & entry, uint32_t key ) { entry.key = key; } +// A FIXED ARRAY VALUE IS ONE MEMBER (§2.8, §4.2): the handle points at the +// ARRAY and not at its first element, so the extent survives the handoff. +typedef TableRef PairsSlotsEntryValue[2]; +inline const PairsSlotsEntryValue * TableEntryFound( const PairsSlotsEntry * entry ) { return entry != NULL ? &entry->value : NULL; } +inline PairsSlotsEntryValue * TableEntryValue( PairsSlotsEntry * entry ) { return &entry->value; } +struct PairsSlotsEntryEach { uint32_t key; decltype( TableEntryValue( (PairsSlotsEntry *) NULL ) ) value; }; +inline PairsSlotsEntryEach TableEntryEach( PairsSlotsEntry * entry ) { return PairsSlotsEntryEach{ TableEntryKey( *entry ), TableEntryValue( entry ) }; } +inline void TableResetMapValue( PairsSlotsEntry & value ) +{ + for ( int32_t i = 0; i < 2; i++ ) { value.value[i].value = 0; } // [2]*Item — every slot null +} + +// PairsSlotsEntryReadKey: the key, before the slot is chosen (docs/SPEC-TABLES.md §2.8). +// Field order inside a body is not contractual (§3), so this scans rather +// than assuming a position — and this implementation writes the key first, +// so on any wire it wrote the scan ends at the first header. +struct PairsSlotsEntryKeyRead +{ + uint32_t key; + bool found; // the body carried the key's id + bool kind_bad; // it carried it under another kind: the MAP's event + bool widened; // under a kind the declaration WIDENS (§4): decoded exactly, the MAP counts one + bool over; // longer than this reader's bound: the ENTRY is dropped + bool malformed; // the entry's framing gave out +}; + +inline PairsSlotsEntryKeyRead PairsSlotsEntryReadKey( const uint8_t * body, int64_t length, const TableIdTable * ids ) +{ + PairsSlotsEntryKeyRead out = { 0, false, false, false, false, false }; + TableReport scratch; // the scan's own framing damage is the MAP's, raised by the caller + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { out.malformed = true; return out; } + if ( field_ref == 0 ) { return out; } // the terminator: no key field is the key's DEFAULT + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { out.malformed = true; return out; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { out.malformed = true; return out; } + uint8_t field_kind = r.get8(); + if ( field_id == 0x3dc94a19365b10ecull ) // `key`, the ordinary hash of an ordinary name + { + if ( field_kind != 8 && TableKindWidens( field_kind, 8 ) ) + { + out.widened = true; + out.found = true; + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, field_kind, widened_v ) ) { out.malformed = true; return out; } + out.key = (uint32_t) widened_v; + continue; // the LAST occurrence is the one §3 keeps + } + out.kind_bad = field_kind != 8; // THE KEY KIND IS THE READER'S DECLARATION + out.found = !out.kind_bad; + if ( !out.kind_bad ) + { + if ( !r.has( 4 ) ) { out.malformed = true; return out; } + out.key = (uint32_t) r.get32(); + continue; // the LAST occurrence is the one §3 keeps + } + } + if ( !r.skip( field_kind ) ) { out.malformed = true; return out; } + } +} + +// ---- retain-unknown: the second family (docs/SPEC-TABLES.md §6.6) ---- +// +// The same walks, with the PATH threaded and the unknown arm capturing. The +// three above are untouched and cost nothing for these being here: a caller +// that does not ask instantiates none of them. + +template inline int64_t PairsSlotsEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool PairsSlotsEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool PairsSlotsEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool PairsSlotsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline int64_t PairsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool PairsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool PairsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ); +inline bool PairsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Pairs & value, TableRetain * retain, const TableRetainPath & path ); + +template +inline int64_t PairsSlotsEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const PairsSlotsEntry & value ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + { + bool any_value = false; + for ( int32_t i = 0; i < 2; i++ ) { if ( ItemAt( ctx, value.value[i] ) != NULL ) { any_value = true; break; } } + if ( any_value ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( 2 ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < 2; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return -1; } + body_value += TableLebBytes( slot_index ); + } + } + bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); // value: [2]*Item + } + } + return bytes; +} + +template +inline bool PairsSlotsEntrySaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const PairsSlotsEntry & value ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 8 ); // key + w.put32( uint32_t( value.key ) ); + } + { + bool any_value = false; + for ( int32_t i = 0; i < 2; i++ ) { if ( ItemAt( ctx, value.value[i] ) != NULL ) { any_value = true; break; } } + if ( any_value ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( 2 ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < 2; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + body_value += TableLebBytes( slot_index ); + } + } + w.putleb( ref_value ); w.put8( 14 ); w.putleb( (uint64_t) body_value ); // value + w.put8( 17 ); w.putleb( (uint64_t) ( 2 ) ); + for ( int32_t elem_i = 0; elem_i < 2; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + w.putleb( slot_index ); + } + } + } + } + return !w.overflow; +} + +template +inline bool PairsSlotsEntrySaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const PairsSlotsEntry & value ) +{ + if ( !PairsSlotsEntrySaveBodyFields( ctx, numbering, w, ids, value ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool PairsSlotsEntryLoadBody( TableReader & r, const TableNodeMap & nodes, PairsSlotsEntry & value ) +{ + PairsSlotsEntryReset( value ); // prefill declared defaults in place, then overlay + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x3dc94a19365b10ecull: // key + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.key = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.key = decoded_v; + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER — the element kind byte and the + // count, so fewer than two bytes — is INERT (§4): the field keeps the + // value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + // A DAMAGED COUNT stops the elements and nothing else: the field + // RODE, so an optional is still PRESENT (§2.3) — only a foreign + // ELEMENT KIND says the payload is not this array's at all. + if ( !counted_ok ) { r.report->malformed = true; } + else if ( elem_kind != 17 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + uint64_t keep = count; + if ( keep > 2 ) { keep = 2; r.report->clamped++; } + // elements are BOUNDED by the field body: a count the length + // cannot cover keeps the decoded prefix, flags malformed, and + // the parent continues at the next field — following fields' + // bytes are never fabricated into elements + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + for ( uint64_t i = 0; i < keep; i++ ) + { + { + uint64_t node_index = 0; + if ( !sub.getleb( node_index ) ) { r.report->malformed = true; break; } + TableNodeResolve( nodes, value.value[(int32_t) i], node_index, 0x52cfa1d198476806ull, r.report ); // *Item + } + } + } + } + r.offset = body_end; // excess elements and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's cost, in BITS (docs/SPEC-TABLES.md §3.3). `at` is the +// body's own bit position in the batch, because a `string(N)` ALIGNS before +// its bytes and an align costs what the position says it costs. +template +inline int64_t PairsSlotsEntryMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const PairsSlotsEntry & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + int64_t bits = 0; + if ( value.key != 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + } + { + bool rides_value = false; + for ( int32_t i = 0; i < 2; i++ ) { if ( ItemAt( ctx, value.value[i] ) != NULL ) { rides_value = true; break; } } + if ( rides_value ) + { + bits += kTableMessageRefBitsHere; + bits += (int64_t) ( 2 ) * index_bits; + } + } + bits += kTableMessageRefBitsHere; // the ZERO REFERENCE that ends the body + (void) at; + return bits; +} + +// The BITPACKED body: the fields, then the ZERO REFERENCE that ends it. No +// kind byte rides at all, and no length frames a nested body, because a +// body is self-delimiting: it is written where the file form put an L. +template +inline bool PairsSlotsEntrySaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const PairsSlotsEntry & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + if ( value.key != 0 ) + { + w.put( 9, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.key ), 32 ); + } + { + bool rides_value = false; + for ( int32_t i = 0; i < 2; i++ ) { if ( ItemAt( ctx, value.value[i] ) != NULL ) { rides_value = true; break; } } + if ( rides_value ) + { + w.put( 10, kTableMessageRefBitsHere ); + for ( int32_t i = 0; i < 2; i++ ) + { + const Item * pointee_value = ItemAt( ctx, value.value[i] ); // *Item + uint64_t index_value = 0; + if ( pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) pointee_value, index_value ) ) { return false; } + w.put( index_value, index_bits ); + } + } + } + w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +inline bool PairsSlotsEntryLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, PairsSlotsEntry & value ) +{ + (void) nodes; (void) index_bits; + PairsSlotsEntryReset( value ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 2 ) { kept = 2; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + TableRef scratch; + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( in_bounds ? value.value[i] : scratch ), node_index_2, 0x52cfa1d198476806ull, report ); // *Item + } + } + } + break; + } + default: + report->unknown++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + } +} + +template +inline int64_t PairsMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Pairs & value ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // slots: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); + if ( !order_slots.ok ) { return -1; } // the sort could not run + if ( order_slots.count > 0 ) + { + const uint64_t ref_slots = ids.ref( 0xe68c2e6bb1ee5646ull ); + int64_t body_slots = 1 + TableLebBytes( (uint64_t) order_slots.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_slots = PairsSlotsEntryMeasureBody( ctx, numbering, ids, *order_slots[i] ); + if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return -1; } + body_slots += TableLebBytes( (uint64_t) ( elem_slots ) ) + ( elem_slots ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_slots ) + 1 + TableLebBytes( (uint64_t) ( body_slots ) ) + ( body_slots ); + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + return bytes; +} + +template +inline bool PairsSaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Pairs & value ) +{ + { + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); // slots + if ( !order_slots.ok ) { return false; } + if ( order_slots.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_slots = ids.ref( 0xe68c2e6bb1ee5646ull ); + int64_t body_slots = 1 + TableLebBytes( (uint64_t) order_slots.count ); + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_slots = PairsSlotsEntryMeasureBody( ctx, numbering, ids, *order_slots[i] ); + if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return false; } + body_slots += TableLebBytes( (uint64_t) ( elem_slots ) ) + ( elem_slots ); + } + w.putleb( ref_slots ); w.put8( 14 ); w.putleb( (uint64_t) body_slots ); + w.put8( 13 ); w.putleb( (uint64_t) order_slots.count ); + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_len_slots = PairsSlotsEntryMeasureBody( ctx, numbering, ids, *order_slots[i] ); + if ( elem_len_slots < 0 ) { TableMapRelease( order_slots ); return false; } + w.putleb( (uint64_t) elem_len_slots ); + if ( !PairsSlotsEntrySaveBody( ctx, numbering, w, ids, *order_slots[i] ) ) { TableMapRelease( order_slots ); return false; } + } + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + return !w.overflow; +} + +template +inline bool PairsSaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Pairs & value ) +{ + if ( !PairsSaveBodyFields( ctx, numbering, w, ids, value ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool PairsLoadBody( TableReader & r, const TableNodeMap & nodes, Pairs & value ) +{ + PairsReset( value ); // prefill declared defaults in place, then overlay + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0xe68c2e6bb1ee5646ull: // slots + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + TableMapFill fill = TableMapFillBegin( nodes, value.slots, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + PairsSlotsEntryKeyRead read = PairsSlotsEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + PairsSlotsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + PairsSlotsEntryLoadBody( elem, nodes, *slot ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's cost, in BITS (docs/SPEC-TABLES.md §3.3). `at` is the +// body's own bit position in the batch, because a `string(N)` ALIGNS before +// its bytes and an align costs what the position says it costs. +template +inline int64_t PairsMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Pairs & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + int64_t bits = 0; + { + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); // slots + if ( !order_slots.ok ) { return -1; } // the sort could not run + if ( order_slots.count > 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; // the count the data decides + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_slots = PairsSlotsEntryMeasureMessageBody( ctx, numbering, index_bits, at + bits, *order_slots[i] ); + if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return -1; } + bits += elem_slots; // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + } + bits += kTableMessageRefBitsHere; // the ZERO REFERENCE that ends the body + (void) at; + return bits; +} + +// The BITPACKED body: the fields, then the ZERO REFERENCE that ends it. No +// kind byte rides at all, and no length frames a nested body, because a +// body is self-delimiting: it is written where the file form put an L. +template +inline bool PairsSaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Pairs & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + { + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); // slots + if ( !order_slots.ok ) { return false; } // the sort could not run + if ( order_slots.count > 0 ) + { + w.put( 37, kTableMessageRefBitsHere ); + w.put( (uint64_t) order_slots.count, 32 ); // the count the data decides + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + if ( !PairsSlotsEntrySaveMessageBody( ctx, numbering, index_bits, w, *order_slots[i] ) ) { TableMapRelease( order_slots ); return false; } + } + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) + { + w.put( 18, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.after ), 32 ); + } + w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +// PairsMessageExtent: the extent Pairs's maps command on the message wire, from +// the FRAMING alone (docs/SPEC-TABLES.md §2.8, §3.3, §6.5). +inline bool PairsMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { return false; } + if ( entry.id == 0xe68c2e6bb1ee5646ull && entry.kind == 14 && entry.elem_kind == 13 ) // slots + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { return false; } + n += (uint64_t) entry.min; + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( PairsSlotsEntry ) + at += (int64_t) n * (int64_t) sizeof( PairsSlotsEntry ); // the whole array FIRST + for ( uint64_t i = 0; i < n; i++ ) // then, entry by entry in key order + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { return false; } + } + continue; + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { return false; } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +inline bool PairsLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Pairs & value ) +{ + (void) nodes; (void) index_bits; + PairsReset( value ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xe68c2e6bb1ee5646ull: // slots + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + TableMapFill fill = TableMapFillBegin( nodes, value.slots, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const PairsSlotsEntryMessageKeyRead read = PairsSlotsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + PairsSlotsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !PairsSlotsEntryLoadMessageBody( r, vocabulary, report, nodes, index_bits, *slot ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + } +} + +// PairsSlotsEntryWireExtent: the extent PairsSlotsEntry's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool PairsSlotsEntryWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + (void) body; (void) length; (void) at; (void) ids; (void) reason; // no list or map below this record + return true; +} + +// PairsSlotsEntryExtentAt: the node extent PairsSlotsEntry's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as PairsSlotsEntryExtentPack advances it (§2.8, §2.9). +template +inline bool PairsSlotsEntryExtentAt( const Ctx & ctx, const PairsSlotsEntry & value, int64_t & at ) +{ + (void) ctx; (void) value; (void) at; // no list or map below this record + return true; +} + +template +inline int64_t PairsSlotsEntryExtent( const Ctx & ctx, const PairsSlotsEntry & value ) +{ + (void) ctx; (void) value; // no list or map below this record + return 0; +} + +// PairsSlotsEntryExtentPack: carve PairsSlotsEntry's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset PairsSlotsEntryExtentAt advances (§2.8, §2.9). +template +inline bool PairsSlotsEntryExtentPack( const Ctx & ctx, const PairsSlotsEntry & src, PairsSlotsEntry & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + (void) ctx; (void) src; (void) dst; (void) extent; (void) at; (void) capacity; // no list or map below this record + return true; +} + +// PairsWireExtent: the extent Pairs's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool PairsWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; // the scan's framing damage is the LOAD's to report + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { return true; } + if ( field_ref == 0 ) { return true; } + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { return true; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { return true; } + uint8_t field_kind = r.get8(); + if ( field_id == 0xe68c2e6bb1ee5646ull && field_kind == 14 ) // slots + { + uint64_t map_len = 0; + if ( !r.getleb( map_len ) || !r.room( map_len ) ) { return true; } + const uint8_t * map_body = r.buffer + r.offset; + r.offset += (int64_t) map_len; + if ( !TableMapWireExtent( map_body, (int64_t) map_len, at, (int64_t) sizeof( PairsSlotsEntry ), (int64_t) alignof( PairsSlotsEntry ), NULL, ids, reason ) ) { return false; } + continue; + } + if ( !r.skip( field_kind ) ) { return true; } + } +} + +// PairsExtentAt: the node extent Pairs's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as PairsExtentPack advances it (§2.8, §2.9). +template +inline bool PairsExtentAt( const Ctx & ctx, const Pairs & value, int64_t & at ) +{ + { + TableMapCursor cursor = TableMapOrder( ctx, value.slots ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( PairsSlotsEntry ) + at += (int64_t) cursor.count * (int64_t) sizeof( PairsSlotsEntry ); // the whole array FIRST + for ( int32_t i = 0; i < cursor.count; i++ ) // then, entry by entry in key order + { + if ( !PairsSlotsEntryExtentAt( ctx, *cursor[i], at ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// the whole extent of one node, from a fresh offset: what a pack reserves +// for it beside the record's own storage. +template +inline int64_t PairsExtent( const Ctx & ctx, const Pairs & value ) +{ + int64_t at = 0; + if ( !PairsExtentAt( ctx, value, at ) ) { return -1; } + return at; +} + +// PairsExtentPack: carve Pairs's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset PairsExtentAt advances (§2.8, §2.9). +template +inline bool PairsExtentPack( const Ctx & ctx, const Pairs & src, Pairs & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + { + TableMapCursor cursor = TableMapOrder( ctx, src.slots ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; + const int64_t bytes = (int64_t) cursor.count * (int64_t) sizeof( PairsSlotsEntry ); + if ( at + bytes > capacity ) { TableMapRelease( cursor ); return false; } + PairsSlotsEntry * placed = (PairsSlotsEntry *) ( extent + at ); + at += bytes; + dst.slots.count = cursor.count; + dst.slots.padding = 0; + dst.slots.entries.value = cursor.count > 0 ? (int64_t) ( (uint8_t *) placed - (const uint8_t *) &dst.slots.entries ) : 0; + for ( int32_t i = 0; i < cursor.count; i++ ) + { + memcpy( (void *) ( placed + i ), (const void *) cursor[i], sizeof( PairsSlotsEntry ) ); // trivially copyable, by construction + } + for ( int32_t i = 0; i < cursor.count; i++ ) + { + if ( !PairsSlotsEntryExtentPack( ctx, *cursor[i], placed[i], extent, at, capacity ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// ---- Pairs.slots: the builder's five and the side index (§2.8) ---- + +// INSERT: the key is copied, the value is handed back at its defaults to +// fill. A DUPLICATE key REPLACES — the value is reset and the same entry +// handed back, key and address unchanged — so a caller that wants to know +// writes Find first. NULL is NOT INSERTED: a key longer than the bound, +// because a truncated key would be a merged entry, and an arena that +// cannot carve another segment, alike. +// +// It is a WRAPPER: TableMapPlace owns the lookup, the reset, the +// allocation and the key copy, and this half is the bound and the +// const char * key's length. Nothing here mutates an entry (§2.8). +inline PairsSlotsEntryValue * PairsSlotsInsert( TableWorker & worker, TableMap & map, uint32_t key ) +{ + PairsSlotsEntry * entry = TableMapPlace( worker, map, key ); + return entry != NULL ? TableEntryValue( entry ) : NULL; +} + +// FIND on the builder: the same linear scan, O( n ) key compares over the +// segments in insertion order. NULL when absent. The builder builds NO +// INDEX, and that is a rule — the sort happens once, at Lock, Save or +// Cook, and every lookup that matters runs over the sorted region. +inline PairsSlotsEntryValue * PairsSlotsFind( TableArena & arena, TableMap & map, uint32_t key ) +{ + PairsSlotsEntry * found = TableMapScan( arena, map, key ); + return found != NULL ? TableEntryValue( found ) : NULL; +} + +// ERASE: marks the entry DEAD, one bit in the segment's slot and not in the +// entry table. False when absent. Its storage is held until the builder +// resets and never reused mid-build, because reusing a slot would make "an +// entry's address is stable" false for exactly one case. +inline bool PairsSlotsErase( TableArena & arena, TableMap & map, uint32_t key ) +{ + return TableMapErase( arena, map, key ); +} + +// EACH on the builder: INSERTION order, live entries only. +inline TableMapEach PairsSlotsEach( const TableArena & arena, const TableMap & map ) +{ + return TableMapEachOf( arena, map ); +} + +// ---- the OPTIONAL INDEX: caller-owned, built at load, never stored ---- +// +// Open addressing with linear probing over the sorted array, for a map large +// enough that log n compares over a cold array cost more than one hash and a +// probe. ITS HASH AND ITS LOAD FACTOR ARE NOT A CROSS-PORT CONTRACT: the +// index is never stored, so no golden, no cook-check rule and no +// build-version line ever names either. What a port is held to is the +// CONTRACT of the lookup — the same value the sorted array's Find returns +// for the same key, and no allocation past the storage the caller handed in. +inline int64_t PairsSlotsIndexMeasure( const TableMap & map ) +{ + return (int64_t) TableMapIndexSlots( map.count ) * (int64_t) sizeof( int32_t ); +} + +inline TableMapIndex PairsSlotsIndex( const TableMap & map, void * storage, int64_t bytes ) +{ + TableMapIndex index; + const int32_t slots = TableMapIndexSlots( map.count ); + if ( storage == NULL || bytes < (int64_t) slots * (int64_t) sizeof( int32_t ) ) { return index; } + index.slots = (int32_t *) storage; + index.capacity = slots; + for ( int32_t i = 0; i < slots; i++ ) { index.slots[i] = 0; } + const PairsSlotsEntry * entries = map.Entries(); + for ( int32_t i = 0; i < map.count; i++ ) // ONE PASS over the sorted array + { + int32_t at = (int32_t) ( TableMapHash( (uint64_t) entries[i].key ) & (uint64_t) ( slots - 1 ) ); + while ( index.slots[at] != 0 ) { at = ( at + 1 ) & ( slots - 1 ); } + index.slots[at] = i + 1; // slots are ENTRY INDICES; 0 is an empty slot + } + index.good = true; + return index; +} + +inline const PairsSlotsEntryValue * PairsSlotsIndexFind( const TableMapIndex & index, const TableMap & map, uint32_t key ) +{ + if ( !index.good ) { return map.Find( key ); } // an index that did not build is not a wrong answer + const PairsSlotsEntry * entries = map.Entries(); + int32_t at = (int32_t) ( TableMapHash( (uint64_t) key ) & (uint64_t) ( index.capacity - 1 ) ); + for ( int32_t probe = 0; probe < index.capacity; probe++ ) + { + const int32_t slot = index.slots[at]; + if ( slot == 0 ) { return NULL; } + if ( TableEntryOrder( entries[slot - 1], key ) == 0 ) { return TableEntryFound( entries + slot - 1 ); } + at = ( at + 1 ) & ( index.capacity - 1 ); + } + return NULL; +} + +// PairsSlotsEntryNumber: number everything PairsSlotsEntry POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool PairsSlotsEntryNumber( const Ctx & ctx, TableNumbering & numbering, const PairsSlotsEntry & value ) +{ + for ( int32_t k = 0; k < 2; k++ ) // value: [2]*Item + { + { + const Item * pointee = ItemAt( ctx, value.value[k] ); // value + if ( pointee != NULL ) + { + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( numbering.seen, (const void *) pointee, + (int64_t) ( numbering.count + 2 ), taken, slot ); // its index, if this is its first visit + if ( entry == NULL ) { return false; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return false; } // a data cycle + } + else + { + TableNodeEntry node; + node.node = (const void *) pointee; + node.type_id = 0x52cfa1d198476806ull; // fnv1a64( "Item" ) + node.type_slot = 65; // its slot in the unit's vocabulary (§3.3) + node.measure = &TableNodeMeasureThunk; + node.save = &TableNodeSaveThunk; + node.message_measure = &TableNodeMessageMeasureThunk; + node.message_save = &TableNodeMessageSaveThunk; + if ( !TableNumberingAppend( numbering, node ) ) { return false; } + if ( !ItemNumber( ctx, numbering, *pointee ) ) { return false; } + TablePackMapClose( numbering.seen, (const void *) pointee, slot ); + } + } + } + } + return true; +} + +// PairsSlotsEntryPackMeasure: the packed region bytes of everything PairsSlotsEntry POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t PairsSlotsEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const PairsSlotsEntry & value ) +{ + int64_t bytes = 0; + for ( int32_t k = 0; k < 2; k++ ) // value: [2]*Item + { + { + const Item * pointee = ItemAt( ctx, value.value[k] ); // value + if ( pointee != NULL ) + { + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( seen, (const void *) pointee, 0, taken, slot ); + if ( entry == NULL ) { return -1; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return -1; } // a data cycle + } + else + { + int64_t inner = ItemPackMeasure( ctx, seen, *pointee ); + if ( inner < 0 ) { return -1; } + TablePackMapClose( seen, (const void *) pointee, slot ); + bytes += TableAlignUp64( (int64_t) sizeof( Item ) ) + inner; + } + } + } + } + return bytes; +} + +// PairsSlotsEntryPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool PairsSlotsEntryPackEdges( const Ctx & ctx, TablePackMap & seen, const PairsSlotsEntry & src, PairsSlotsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool PairsSlotsEntryPack( const Ctx & ctx, TablePackMap & seen, const PairsSlotsEntry & src, PairsSlotsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( PairsSlotsEntry ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( PairsSlotsEntry ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !PairsSlotsEntryExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return PairsSlotsEntryPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool PairsSlotsEntryPackEdges( const Ctx & ctx, TablePackMap & seen, const PairsSlotsEntry & src, PairsSlotsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + for ( int32_t k = 0; k < 2; k++ ) // value: [2]*Item + { + { + dst.value[k].value = 0; // value + const Item * pointee = ItemAt( ctx, src.value[k] ); + if ( pointee != NULL ) + { + int64_t at = TableAlignUp64( used ); // where it WOULD land, if this is its first visit + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( seen, (const void *) pointee, at, taken, slot ); + if ( entry == NULL ) { return false; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return false; } // a data cycle + dst.value[k].value = (int64_t) ( ( base + entry->offset ) - (const uint8_t *) &dst.value[k] ); // the one body it already has + } + else + { + if ( at + (int64_t) sizeof( Item ) > capacity ) { return false; } + used = at + TableAlignUp64( (int64_t) sizeof( Item ) ); + Item * child = new ( base + at ) Item; // lifetime only: the Pack below memcpy's the whole node over it + dst.value[k].value = (int64_t) ( ( base + at ) - (const uint8_t *) &dst.value[k] ); + if ( !ItemPack( ctx, seen, *pointee, *child, base, capacity, used ) ) { return false; } + TablePackMapClose( seen, (const void *) pointee, slot ); + } + } + } + } + return true; +} + +// PairsNumber: number everything Pairs POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool PairsNumber( const Ctx & ctx, TableNumbering & numbering, const Pairs & value ) +{ + { // slots: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_slots = TableMapOrder( ctx, value.slots ); + if ( !cursor_slots.ok ) { return false; } + for ( int32_t i = 0; i < cursor_slots.count; i++ ) + { + if ( !PairsSlotsEntryNumber( ctx, numbering, *cursor_slots[i] ) ) { TableMapRelease( cursor_slots ); return false; } + } + TableMapRelease( cursor_slots ); + } + return true; +} + +// PairsPackMeasure: the packed region bytes of everything Pairs POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t PairsPackMeasure( const Ctx & ctx, TablePackMap & seen, const Pairs & value ) +{ + int64_t bytes = 0; + { // slots: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_slots = TableMapOrder( ctx, value.slots ); + if ( !cursor_slots.ok ) { return -1; } + for ( int32_t i = 0; i < cursor_slots.count; i++ ) + { + int64_t inner = PairsSlotsEntryPackMeasure( ctx, seen, *cursor_slots[i] ); + if ( inner < 0 ) { TableMapRelease( cursor_slots ); return -1; } + bytes += inner; + } + TableMapRelease( cursor_slots ); + } + return bytes; +} + +// PairsPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool PairsPackEdges( const Ctx & ctx, TablePackMap & seen, const Pairs & src, Pairs & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool PairsPack( const Ctx & ctx, TablePackMap & seen, const Pairs & src, Pairs & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( Pairs ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( Pairs ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !PairsExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return PairsPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool PairsPackEdges( const Ctx & ctx, TablePackMap & seen, const Pairs & src, Pairs & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + { // slots: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_slots = TableMapOrder( ctx, src.slots ); + if ( !cursor_slots.ok ) { return false; } + PairsSlotsEntry * placed_slots = (PairsSlotsEntry *) ( dst.slots.entries.value != 0 ? ( (uint8_t *) &dst.slots.entries + dst.slots.entries.value ) : NULL ); + for ( int32_t i = 0; i < cursor_slots.count; i++ ) + { + if ( !PairsSlotsEntryPackEdges( ctx, seen, *cursor_slots[i], placed_slots[i], base, capacity, used ) ) { TableMapRelease( cursor_slots ); return false; } + } + TableMapRelease( cursor_slots ); + } + return true; +} + +// ---- Pairs: the variable-length life (docs/SPEC-TABLES.md §2, §6, §9) ---- +// +// MUTABLE: PairsBuilder — allocate nodes, wire them together, then Lock. +// CONST: one packed region, root at its base. Lock produces it and Load +// produces it, so a locked structure and a loaded one are the +// SAME representation with one view API. There is no unlock: +// re-editing means loading the const form into a fresh builder. +// Pairs is never held by value — a file-format-scale structure is a region +// and a root pointer, not a struct you copy. + +struct PairsBuilder +{ + TableArena arena; + TableWorker main; // the calling thread's allocation front + TableRef root_ref; + uint8_t * region = NULL; // the packed const form, produced by Lock() + int64_t region_bytes = 0; + + // THE ALLOCATOR IS THE BUILDER'S, and everything this structure ever + // allocates goes through it: the arena's segments, Lock's identity map, + // the packed region, the wire walks' numbering, and the tool path's node + // directory. Name your own and a profiler sees every byte under it. + PairsBuilder( TableAllocator allocator = TableDefaultAllocator() ) + { + TableArenaInit( arena, allocator ); + main.arena = &arena; + TableSlot slot = main.Alloc(); + root_ref = slot.ref; + } + ~PairsBuilder() { TableArenaShutdown( arena ); arena.allocator.free( arena.allocator.context, region ); } + PairsBuilder( const PairsBuilder & ) = delete; + PairsBuilder & operator=( const PairsBuilder & ) = delete; + + // Alloc a node in THIS thread's slab: no lock, no atomic per node. + // The result is usable both as the node pointer and as the reference + // to store in a pointer field. + template TableSlot Alloc() { return main.Alloc(); } + // a BYTE BUFFER's node of exactly `length` bytes (docs/SPEC-TABLES.md §2.5): + // the bytes to write through, and the reference to store in a *bytes + // or *string slot; a blob past a slab takes a span of its own + TableBytesSlot AllocBytes( int64_t length ) { return main.AllocBytes( length ); } + TableStringSlot AllocString( int64_t length ) { return main.AllocString( length ); } + // one worker per thread; allocate on your own, and synchronize your own + // writes to nodes another worker allocated + TableWorker Worker() { TableWorker worker; worker.arena = &arena; return worker; } + + // GetRoot/AsConst, not Root/Const: a member function hides the type + // name it shares, and `table Root` is this spec's own canonical + // example. The checker refuses a table named after any member here, + // so the remaining spellings cannot collide either. + Pairs * GetRoot() { return arena.locked ? NULL : (Pairs *) TableArenaAt( arena, (uint32_t) root_ref.value ); } + bool Locked() const { return arena.locked; } + const Pairs * AsConst() const { return (const Pairs *) region; } + const uint8_t * Region() const { return region; } + int64_t RegionBytes() const { return region_bytes; } + + // Lock is ONE WAY and it is the compaction: the segmented arena becomes + // one exact-packed region with zero slack, references rewritten + // self-relative, and the mutable life released. Single-threaded: call + // it after the workers have joined. + bool Lock(); +}; + +inline bool PairsBuilder::Lock() +{ + if ( arena.locked ) { return region != NULL; } + if ( root_ref.null() ) { return false; } + TableArenaCtx ctx = { &arena }; + const Pairs & root = *(const Pairs *) TableArenaAt( arena, (uint32_t) root_ref.value ); + // The ROOT takes the map's first entry: it is packed at offset 0, and its + // descent is open for the whole walk (docs/SPEC-TABLES.md §3.1). + TablePackMap seen; + TablePackMapInit( seen, arena.allocator ); + bool root_taken = false; + int64_t root_slot = 0; + int64_t below = -1; + if ( TablePackMapReach( seen, (const void *) &root, 0, root_taken, root_slot ) != NULL ) + { + below = PairsPackMeasure( ctx, seen, root ); + } + if ( below < 0 ) { TablePackMapShutdown( seen ); return false; } // a data cycle, named at the reference that closes it + int64_t root_extent = PairsExtent( ctx, root ); + if ( root_extent < 0 ) { TablePackMapShutdown( seen ); return false; } // the sort could not run + int64_t total = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ) + below; + // the AUTHORING path may allocate (§6.5), and it does so through the + // builder's own pair. The region comes back ZEROED, which is the + // allocator's contract: a packed region carries node padding. + uint8_t * packed = (uint8_t *) arena.allocator.alloc( arena.allocator.context, total ); + if ( packed == NULL ) { TablePackMapShutdown( seen ); return false; } + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + Pairs * destination = new ( packed ) Pairs; // lifetime only: the Pack below memcpy's the whole node over it + // The pack walk RE-DERIVES the same numbering rather than carrying the + // measure's — nothing passes between them, which is what makes + // `used == total` below a real check and not a tautology (§3.1). The + // map keeps the capacity the measure paid for, so the second walk + // rehashes nothing. + TablePackMapReset( seen ); + if ( TablePackMapReach( seen, (const void *) &root, 0, root_taken, root_slot ) == NULL || + !PairsPack( ctx, seen, root, *destination, packed, total, used ) || used != total ) + { + TablePackMapShutdown( seen ); + arena.allocator.free( arena.allocator.context, packed ); + return false; + } + TablePackMapShutdown( seen ); + region = packed; + region_bytes = total; + arena.locked = true; // MONOTONIC: there is no unlock + TableArenaShutdown( arena ); + return true; +} + +// ---- Pairs on the wire: the FLAT NODE TABLE (docs/SPEC-TABLES.md §3.1) ---- +// +// A pointered save writes every reachable node ONCE, into a node table under +// the reserved id 0xFFFF, and a pointer field rides as a u32 INDEX into it +// under kind 17. No pointer edge is a nesting level, so a chain's length is +// not a depth and two references to one node are one node. + +// PairsNodeStorage: the region bytes one record commands, or -1 for a type id +// this build cannot name — which keeps its index and reads null. A BYTE +// BUFFER's record commands its header and its bytes (docs/SPEC-TABLES.md §2.5), +// which is the one answer the record's LENGTH decides, and a blob past the +// size cap answers kTableNodeRefused with its reason (§3.1, §6.5). +// A MAP'S ENTRIES RIDE IN THEIR HOLDER'S EXTENT (docs/SPEC-TABLES.md §2.8), +// so a record's storage is its type's PLUS N x sizeof( Entry ) at every +// depth, summed from the FRAMING: N is framing and not a value, and this +// reads no field. kTableNodeRefused is a wire whose N its L cannot carry. +inline int64_t PairsNodeStorage( uint64_t type_id, int64_t length, TableRefuseReason & reason ) +{ + (void) length; // no byte buffer below this root: every node's storage is its type's + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( (int64_t) sizeof( Item ) ); // Item + default: break; + } + (void) reason; // no blob and no extent below this root: nothing here refuses + return -1; +} + +// PairsNodePlace: start one record's node's lifetime in the storage pass one +// reserved for it, holding exactly the declared defaults — a byte buffer's +// header holds its length, and its bytes come in pass two. +inline void PairsNodePlace( uint64_t type_id, uint8_t * at, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: { Item * node = new ( at ) Item; ItemReset( *node ); break; } // Item + default: break; + } +} + +// PairsNodeRecordBytes: one record's OWN storage, before the extent its maps +// take (docs/SPEC-TABLES.md §2.8) — where a node's extent begins. +inline int64_t PairsNodeRecordBytes( uint64_t type_id ) +{ + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( (int64_t) sizeof( Item ) ); // Item + default: break; + } + return 0; +} + +// PairsNodeAlloc: the TOOL's path — one record's node in the builder's arena. +// Zero is the arena's null, and it is also what a type id this build cannot +// name answers. +inline uint32_t PairsNodeAlloc( uint64_t type_id, TableWorker & worker, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return (uint32_t) worker.Alloc().ref.value; // Item + default: break; + } + return 0; +} + +// PairsNodeBody: PASS TWO's half — decode one record's body into the storage it +// already owns. +inline void PairsNodeBody( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = PairsNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? PairsNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ItemLoadBody( r, *(Item *) at ); break; // Item + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// PairsNodeMessageStorage: the region bytes one record commands on the message +// wire, or -1 for a type id this build cannot name. A table's is its own +// storage plus the extent its maps take; a byte buffer's is its header and +// its bytes, which is the one answer the record's LENGTH decides. +inline int64_t PairsNodeMessageStorage( uint64_t type_id, int64_t extent, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Item ) ) + extent ); // Item + default: break; + } + return -1; +} + +// PairsNodeMessageExtent: step over one TABLE record's body, tallying the extent +// its maps take where its type has any (§2.8). A type this build cannot +// name is stepped over by its announced shapes and takes no extent. +inline bool PairsNodeMessageExtent( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & extent ) +{ + extent = 0; + (void) type_id; // no map below any node this root can name + return TableMessageSkipBody( r, vocabulary, index_bits ); +} + +// PairsNodeMessageBody: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +inline bool PairsNodeMessageBody( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at ) +{ + TableExtentCarve carve; + carve.at = at + PairsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !PairsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemLoadMessageBody( r, vocabulary, report, index_bits, *(Item *) at ); break; // Item + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// The numbering both wire walks derive, and NEITHER CARRIES THE OTHER'S: the +// root takes index 1 and its entry stays open for the whole walk, so a +// reference back at it is the cycle it is (§3.1). +template +inline bool PairsNumberFrom( const Ctx & ctx, TableNumbering & numbering, const Pairs & root ) +{ + bool taken = false; + int64_t slot = 0; + if ( TablePackMapReach( numbering.seen, (const void *) &root, (int64_t) kTableNodeIndexRoot, taken, slot ) == NULL ) { return false; } + return PairsNumber( ctx, numbering, root ); +} + +template +inline int64_t PairsMeasureWire( const Ctx & ctx, const Pairs & root, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bytes = -1; + if ( PairsNumberFrom( ctx, numbering, root ) ) + { + TableIds ids; + bytes = PairsMeasureBody( ctx, numbering, ids, root ); + if ( bytes >= 0 ) + { + const int64_t table = TableNodeTableMeasure( ctx, ids, numbering ); + // the FORM BYTE, the ROOT BODY — its own fields, the node table + // and the terminator — and the ID TABLE (docs/SPEC-TABLES.md §3) + bytes = table < 0 || ids.overflow ? -1 : 1 + bytes + table + TableIdsBytes( ids ); + } + } + TableNumberingShutdown( numbering ); + return bytes; +} + +template +inline int64_t PairsSaveWire( const Ctx & ctx, const Pairs & root, uint8_t * buffer, int64_t capacity, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + if ( !PairsNumberFrom( ctx, numbering, root ) ) { TableNumberingShutdown( numbering ); return -1; } + TableWriter w( buffer, capacity ); + TableIds ids; + w.put8( kTableWireForm ); // the FORM BYTE is the whole header (§3) + // the root's own fields, then the node table's field, then the + // terminator: a reader that gives up inside the table has already + // decoded the ROOT'S OWN FIELDS (§3.1) + bool ok = PairsSaveBodyFields( ctx, numbering, w, ids, root ) && TableNodeTableSave( ctx, w, ids, numbering ); + TableNumberingShutdown( numbering ); + if ( !ok || ids.overflow ) { return -1; } + w.put8( 0 ); // the ZERO REFERENCE that ends the root body + TableIdsWrite( w, ids ); + if ( w.overflow ) { return -1; } // the caller's buffer was too small + return w.offset; // == PairsMeasure( root ) +} + +inline int64_t PairsMeasure( const Pairs * root, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return PairsMeasureWire( ctx, *root, allocator ); +} + +inline int64_t PairsSave( const Pairs * root, uint8_t * buffer, int64_t capacity, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return PairsSaveWire( ctx, *root, buffer, capacity, allocator ); +} + +inline int64_t PairsMeasure( const PairsBuilder & builder ) +{ + if ( builder.region != NULL ) { return PairsMeasure( builder.AsConst(), builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return PairsMeasureWire( ctx, *(const Pairs *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), builder.arena.allocator ); +} + +inline int64_t PairsSave( const PairsBuilder & builder, uint8_t * buffer, int64_t capacity ) +{ + if ( builder.region != NULL ) { return PairsSave( builder.AsConst(), buffer, capacity, builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return PairsSaveWire( ctx, *(const Pairs *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), buffer, capacity, builder.arena.allocator ); +} + +// ---- Pairs on the MESSAGE wire: the batch over a region (docs/SPEC-TABLES.md §3.3) ---- + +// PairsMessageBodyBits: one root body's bits at bit position `at` of the batch, +// with the numbering derived from the graph, the node table FIRST, then the +// fields, then the zero reference. Measure derives the numbering and save +// derives the same one, and nothing passes between them (§3.1). +template +inline int64_t PairsMessageBodyBits( const Ctx & ctx, const Pairs & root, int64_t at, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bits = -1; + if ( PairsNumberFrom( ctx, numbering, root ) ) + { + const int64_t index_bits = TableBitsRequired( 0, numbering.count + 1 ); + const int64_t table = TableMessageNodeTableMeasure( ctx, numbering, index_bits, at ); + if ( table >= 0 ) + { + const int64_t body = PairsMeasureMessageBody( ctx, numbering, index_bits, at + table, root ); + bits = body < 0 ? -1 : table + body; + } + } + TableNumberingShutdown( numbering ); + return bits; +} + +template +inline bool PairsMessageBodySave( const Ctx & ctx, const Pairs & root, TableBitWriter & w, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + bool ok = false; + if ( PairsNumberFrom( ctx, numbering, root ) ) + { + const int64_t index_bits = TableBitsRequired( 0, numbering.count + 1 ); + ok = TableMessageNodeTableSave( ctx, numbering, index_bits, w ) && PairsSaveMessageBody( ctx, numbering, index_bits, w, root ); + } + TableNumberingShutdown( numbering ); + return ok && !w.overflow; +} + +// THE PRIMITIVE IS A BATCH (§3.3): a number of ROOTS in one buffer, one count +// and one continuous bit stream, each body carrying its own numbering. A +// root is a locked region's, `builder.AsConst()`, or a loaded one's. M above +// 256 is a refusal by name, batch_too_large, with nothing written. +inline int64_t PairsMeasureMessages( const Pairs * const * roots, int64_t count, TableReport * report, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( roots == NULL || count < 1 ) { return -1; } + if ( count > kTableMessageBatchMax ) { TableMessageRefuseBatch( report ); return -1; } + TableRegionCtx ctx; + int64_t bits = 8; // the body count + for ( int64_t i = 0; i < count; i++ ) + { + if ( roots[i] == NULL ) { return -1; } + const int64_t body = PairsMessageBodyBits( ctx, *roots[i], bits, allocator ); + if ( body < 0 ) { return -1; } + bits += body; + } + return 1 + ( bits + 7 ) / 8; +} + +inline int64_t PairsSaveMessages( const Pairs * const * roots, int64_t count, uint8_t * buffer, int64_t capacity, TableReport * report, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( roots == NULL || count < 1 ) { return -1; } + if ( count > kTableMessageBatchMax ) { TableMessageRefuseBatch( report ); return -1; } + TableMessageBatch batch; + if ( !TableMessageBatchBegin( batch, buffer, capacity, count ) ) { return -1; } + TableRegionCtx ctx; + for ( int64_t i = 0; i < count; i++ ) + { + if ( roots[i] == NULL || !PairsMessageBodySave( ctx, *roots[i], batch.w, allocator ) ) { return -1; } + batch.written++; + } + return TableMessageBatchEnd( batch ); // == PairsMeasureMessages( roots, count, report, allocator ) +} + +// PairsMessageRecordScan: one node record's type id and the extent its maps +// take, or a blob's length, the reader left after the record. A type id +// reference of 0, one past E, or one naming anything but a kind-0 entry is +// damage, as §3.1 and §3.3 say. +inline bool PairsMessageRecordScan( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, uint64_t & type_id, int64_t & extent, int64_t & length ) +{ + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { return false; } + TableMessageEntry type_entry; + if ( !TableMessageNameEntry( vocabulary, type_ref, type_entry ) ) { return false; } + type_id = type_entry.id; + extent = 0; + length = 0; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + // A BLOB RECORD CARRIES A LENGTH AT THIRTY-TWO RAW BITS, then ALIGNS, + // then the bytes verbatim (§3.3) + uint64_t n = 0; + if ( !r.get( n, 32 ) || !r.align() || !r.skip( (int64_t) n * 8 ) ) { return false; } + length = (int64_t) n; + return true; + } + return PairsNodeMessageExtent( type_id, r, vocabulary, index_bits, extent ); +} + +// PairsMessageBodyStorage: one body's node count and data bytes from the FRAMING +// alone, the reader left at the next body. The node table is walked record +// by record, a table record's body stepped over by its announced shapes and +// a blob's by its length, then the root's own fields. False is a numbering +// that could not be sized; `complete` false is a ROOT body whose own framing +// gave out, which the load meets as damage inside this body after the +// bodies before it were delivered, so the batch is sized through this body +// and no further (§3.3). +inline bool PairsMessageBodyStorage( TableBitReader & r, const TableVocabulary & vocabulary, int64_t & records, int64_t & data, bool & complete ) +{ + complete = true; + records = 0; + data = 0; + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { return false; } + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !PairsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { return false; } + const int64_t storage = PairsNodeMessageStorage( type_id, extent, length ); + if ( storage > 0 ) { data += storage; } // a type id this build cannot name commands none + records++; + } + int64_t root_extent = 0; + if ( !PairsMessageExtent( r, vocabulary, index_bits, root_extent ) ) { complete = false; } + data += TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + return true; +} + +// PairsLoadMeasure's MESSAGE overload: the exact region bytes ONE BATCH needs, +// which is one measurement, one allocation and one bounds check for however +// many bodies ride (§3.3, §6.5). It is a scan by the announced shapes and +// reads no field value. The answer is the data bytes plus the attribution, +// one node directory a body, and -1 for a wire it cannot size: no vocabulary, +// another form, or framing that gives out. +inline int64_t PairsLoadMeasure( const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, int64_t * attribution_bytes = NULL ) +{ + TableReport ignored; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, &ignored ); + if ( bodies < 0 ) { return -1; } + int64_t data = 0, attribution = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + int64_t records = 0, body_data = 0; + bool complete = true; + if ( !PairsMessageBodyStorage( br.r, vocabulary, records, body_data, complete ) ) { return -1; } + data += body_data; + attribution += ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( !complete ) { break; } // damage inside this body: the load delivers the ones before it + } + if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; } + return data + attribution; +} + +// PairsLoadMessageBodyInto: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +inline bool PairsLoadMessageBodyInto( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Pairs * & root_out ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !PairsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = PairsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + PairsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !PairsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x1404200dab337086ull; + Pairs * root = new ( region + used ) Pairs; // lifetime only: LoadMessageBody's first act is PairsReset + PairsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Pairs ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + out->unknown += unknown_records; + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !PairsNodeMessageBody( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return PairsLoadMessageBody( r, vocabulary, out, nodes, index_bits, *root ); +} + +// PairsLoadMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +inline bool PairsLoadMessages( const Pairs ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !PairsLoadMessageBodyInto( br.r, vocabulary, out, region, region_bytes, used, roots[b] ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + +// PairsLoadMeasure: the exact region bytes a wire buffer will need, and it is +// ONE SCAN — a record's type id gives its storage size, its length gives the +// next record — reading no field value at all, so the caller owns the +// allocation and can refuse a number it did not expect (§6.5). +// +// It reports the DATA bytes and the ATTRIBUTION bytes separately, because the +// attribution is the wire's numbering made resident (§6.3) and a caller may +// release it once Load returns. The answer is their sum. +inline int64_t PairsLoadMeasure( const uint8_t * wire_file, int64_t wire_file_bytes, int64_t * attribution_bytes = NULL, TableRefuseReason * reason_out = NULL ) +{ + TableReport ignored; + TableIdTable ids_table; + int64_t body_bytes = 0; + // a FORM BYTE this build does not carry is refused by name (§3, §6.5); + // a trailer that cannot be read whole is damage and names no reason + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict == TableOpenRefused ) { if ( reason_out != NULL ) { *reason_out = unknown_form; } return -1; } + if ( verdict != TableOpenOk ) { return -1; } + // ANY BYTE BETWEEN THE ROOT'S TERMINATOR AND THE TABLE'S FIRST ENTRY + // IS MALFORMED (docs/SPEC-TABLES.md §3): the two ends of the file have + // met, nothing is decoded, and no region is sized from it. + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) { return -1; } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &ignored, &ids_table ); + TableRefuseReason reason = count_over_length; + int64_t root_extent = 0; + if ( !PairsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { if ( reason_out != NULL ) { *reason_out = reason; } return -1; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + int64_t records = 0; + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = PairsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { if ( reason_out != NULL ) { *reason_out = reason; } return -1; } // an N the record's framing cannot carry, or a blob past the cap (§2.8, §2.9, §3.1) + if ( storage > 0 ) { data += storage; } // a type id this build cannot name commands none + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; } + return data + attribution; +} + +// PairsLoad: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +inline const Pairs * PairsLoad( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Pairs ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !PairsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = PairsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0x1404200dab337086ull; + Pairs * root = new ( region ) Pairs; // lifetime only: LoadBody's first act is PairsReset + PairsReset( *root ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = PairsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + PairsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + if ( nodes.good ) { out->unknown += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + PairsNodeBody( type_id, sub, nodes, region + directory[k + 1].offset ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Pairs ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + PairsLoadBody( r, nodes, *root ); + return root; +} + +// PairsLoadBuilder: the TOOL's path — the same tolerant decode into a fresh +// builder, so loaded data can be edited and locked again. The numbering is +// the same one; what differs is where a node lives and therefore what a +// resolved slot holds — an arena offset here, a self-relative delta there. +inline bool PairsLoadBuilder( PairsBuilder & builder, const uint8_t * wire_file, int64_t wire_file_bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; } + return false; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return false; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + Pairs * root = builder.GetRoot(); + if ( root == NULL ) { out->malformed = true; return false; } + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) { records++; } + } + // the AUTHORING side may allocate (§6.5), and this is the tool's path. + // It goes through the builder's own pair, like everything else the + // builder reaches, and the entries come back zeroed. + const TableAllocator allocator = builder.arena.allocator; + TableNodeDirEntry * directory = (TableNodeDirEntry *) allocator.alloc( allocator.context, ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ) ); + if ( directory == NULL ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) builder.root_ref.value; + directory[0].type_id = 0x1404200dab337086ull; + TableNodeMap nodes; + nodes.base = NULL; + nodes.entries = directory; + nodes.count = records + 1; + nodes.arena = true; // a resolved slot holds the node's ARENA OFFSET here + nodes.worker = &builder.main; // and a map's entries and a list's elements are the arena's, not a node extent's (§2.8, §2.9) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + uint32_t at = PairsNodeAlloc( type_id, builder.main, length ); + if ( at == 0 ) + { + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + } + else + { + directory[k + 1].offset = (uint64_t) at; + } + directory[k + 1].type_id = type_id; + k++; + } + nodes.good = TableNodeScanWhole( scan ); + if ( nodes.good ) { out->unknown += unknown_records; } else { out->malformed = true; } + } + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + PairsNodeBody( type_id, sub, nodes, TableArenaAt( builder.arena, (uint32_t) directory[k + 1].offset ) ); + } + k++; + } + } + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.worker = &builder.main; + nodes.carve = &root_carve; + bool ok = PairsLoadBody( r, nodes, *root ); + // A COUNT ABOVE THE int32 CAP is this path's refusal (docs/SPEC-TABLES.md + // §2.9): the partial builder is the caller's to discard, and the report + // holds what it held when the count was met + ok = ok && !nodes.refused; + allocator.free( allocator.context, directory ); + return ok; +} + +template +inline int64_t PairsSlotsEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + { + bool any_value = false; + for ( int32_t i = 0; i < 2; i++ ) { if ( ItemAt( ctx, value.value[i] ) != NULL ) { any_value = true; break; } } + if ( any_value ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( 2 ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < 2; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return -1; } + body_value += TableLebBytes( slot_index ); + } + } + bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); // value: [2]*Item + } + } + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool PairsSlotsEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 8 ); // key + w.put32( uint32_t( value.key ) ); + } + { + bool any_value = false; + for ( int32_t i = 0; i < 2; i++ ) { if ( ItemAt( ctx, value.value[i] ) != NULL ) { any_value = true; break; } } + if ( any_value ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( 2 ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < 2; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + body_value += TableLebBytes( slot_index ); + } + } + w.putleb( ref_value ); w.put8( 14 ); w.putleb( (uint64_t) body_value ); // value + w.put8( 17 ); w.putleb( (uint64_t) ( 2 ) ); + for ( int32_t elem_i = 0; elem_i < 2; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + w.putleb( slot_index ); + } + } + } + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool PairsSlotsEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !PairsSlotsEntrySaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool PairsSlotsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + PairsSlotsEntryReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x3dc94a19365b10ecull: // key + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.key = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.key = decoded_v; + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER — the element kind byte and the + // count, so fewer than two bytes — is INERT (§4): the field keeps the + // value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + // A DAMAGED COUNT stops the elements and nothing else: the field + // RODE, so an optional is still PRESENT (§2.3) — only a foreign + // ELEMENT KIND says the payload is not this array's at all. + if ( !counted_ok ) { r.report->malformed = true; } + else if ( elem_kind != 17 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + uint64_t keep = count; + if ( keep > 2 ) { keep = 2; r.report->clamped++; } + // elements are BOUNDED by the field body: a count the length + // cannot cover keeps the decoded prefix, flags malformed, and + // the parent continues at the next field — following fields' + // bytes are never fabricated into elements + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + for ( uint64_t i = 0; i < keep; i++ ) + { + { + uint64_t node_index = 0; + if ( !sub.getleb( node_index ) ) { r.report->malformed = true; break; } + TableNodeResolve( nodes, value.value[(int32_t) i], node_index, 0x52cfa1d198476806ull, r.report ); // *Item + } + } + } + } + r.offset = body_end; // excess elements and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +template +inline int64_t PairsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // slots: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); + if ( !order_slots.ok ) { return -1; } // the sort could not run + if ( order_slots.count > 0 ) + { + const uint64_t ref_slots = ids.ref( 0xe68c2e6bb1ee5646ull ); + int64_t body_slots = 1 + TableLebBytes( (uint64_t) order_slots.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_slots = PairsSlotsEntryMeasureBodyRetain( ctx, numbering, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return -1; } + body_slots += TableLebBytes( (uint64_t) ( elem_slots ) ) + ( elem_slots ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_slots ) + 1 + TableLebBytes( (uint64_t) ( body_slots ) ) + ( body_slots ); + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool PairsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + { + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); // slots + if ( !order_slots.ok ) { return false; } + if ( order_slots.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_slots = ids.ref( 0xe68c2e6bb1ee5646ull ); + int64_t body_slots = 1 + TableLebBytes( (uint64_t) order_slots.count ); + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_slots = PairsSlotsEntryMeasureBodyRetain( ctx, numbering, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return false; } + body_slots += TableLebBytes( (uint64_t) ( elem_slots ) ) + ( elem_slots ); + } + w.putleb( ref_slots ); w.put8( 14 ); w.putleb( (uint64_t) body_slots ); + w.put8( 13 ); w.putleb( (uint64_t) order_slots.count ); + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_len_slots = PairsSlotsEntryMeasureBodyRetain( ctx, numbering, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_len_slots < 0 ) { TableMapRelease( order_slots ); return false; } + w.putleb( (uint64_t) elem_len_slots ); + if ( !PairsSlotsEntrySaveBodyRetain( ctx, numbering, w, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_slots ); return false; } + } + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool PairsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !PairsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool PairsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + PairsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0xe68c2e6bb1ee5646ull: // slots + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.slots, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + PairsSlotsEntryKeyRead read = PairsSlotsEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + PairsSlotsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + PairsSlotsEntryLoadBodyRetain( elem, nodes, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// PairsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void PairsNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = PairsNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? PairsNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ItemLoadBodyRetain( r, *(Item *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Item + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// PairsLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +// UNDER RETENTION it also fills the caller's two stores with the fields +// this build cannot name, and the report carries what it could not keep +// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the +// reader's data is exactly what it would have been with retention off. +inline const Pairs * PairsLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Pairs ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !PairsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = PairsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0x1404200dab337086ull; + Pairs * root = new ( region ) Pairs; // lifetime only: LoadBody's first act is PairsReset + PairsReset( *root ); + + // LoadRetain RESETS BOTH STORES and writes into neither id list: a + // retained record carries its field's identity in the record itself, + // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer + // belongs to this region from here on. + TableRetainReset( retain, nodes, region ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = PairsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + PairsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + PairsNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Pairs ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + PairsLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// PairsMeasureRetain and PairsSaveRetain: the pair, with the retained tail in +// every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same +// records under the same walk, so Measure's answer is the size the save +// writes even where a record could not be placed. +template +inline int64_t PairsMeasureWireRetain( const Ctx & ctx, const Pairs & root, TableRetain * retain, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bytes = -1; + auto retain_measure = []( const Ctx & c, const TableNumbering & nn, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> int64_t + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemMeasureBodyRetain( ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return -1; + }; + if ( PairsNumberFrom( ctx, numbering, root ) ) + { + TableRetainIds ids( retain ); + if ( retain != NULL ) { retain->id_used = 0; } // one walk fills the list, and the save's own walk refills it + bytes = PairsMeasureBodyRetain( ctx, numbering, ids, root, retain, TableRetainPathRoot( (const void *) &root, 1 ) ); + if ( bytes >= 0 ) + { + const int64_t table = TableNodeTableMeasureRetain( ctx, ids, numbering, retain, retain_measure ); + bytes = table < 0 || ids.overflow ? -1 : 1 + bytes + table + TableRetainIdsBytes( ids ); + } + } + TableNumberingShutdown( numbering ); + return bytes; +} + +template +inline int64_t PairsSaveWireRetain( const Ctx & ctx, const Pairs & root, TableRetain * retain, uint8_t * buffer, int64_t capacity, TableReport * report ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, TableDefaultAllocator() ); + auto retain_measure = []( const Ctx & c, const TableNumbering & nn, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> int64_t + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemMeasureBodyRetain( ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return -1; + }; + auto retain_save = []( const Ctx & c, const TableNumbering & nn, TableWriter & ww, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> bool + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ww; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemSaveBodyRetain( ww, ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return false; + }; + if ( !PairsNumberFrom( ctx, numbering, root ) ) { TableNumberingShutdown( numbering ); return -1; } + TableWriter w( buffer, capacity ); + TableRetainIds ids( retain ); + if ( retain != NULL ) { retain->id_used = 0; } + TableRetainClearPlaced( retain ); + w.put8( kTableWireForm ); // the FORM BYTE is the whole header (§3) + // the root's own fields, then the RETAINED TAIL, then the node table's + // field: a retained field is one of the root's own values, and the tail + // is pinned before the large and damage-prone part (§6.6, §3.1) + bool ok = PairsSaveBodyFieldsRetain( ctx, numbering, w, ids, root, retain, TableRetainPathRoot( (const void *) &root, 1 ) ) && + TableNodeTableSaveRetain( ctx, w, ids, numbering, retain, retain_measure, retain_save ); + TableNumberingShutdown( numbering ); + if ( !ok || ids.overflow ) { return -1; } + w.put8( 0 ); // the ZERO REFERENCE that ends the root body + TableRetainIdsWrite( w, ids ); + if ( w.overflow ) { return -1; } // the caller's buffer was too small + // THE SAVE'S OWN SHARE OF retain_lost, read after the save (§6.6): every + // record the walk did not place, counted once. + TableRetainCountLost( retain, report ); + return w.offset; +} + +inline int64_t PairsMeasureRetain( const Pairs * root, TableRetain * retain, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return PairsMeasureWireRetain( ctx, *root, retain, allocator ); +} + +// SaveRetain REFUSES A NULL REPORT and returns -1 (docs/SPEC-TABLES.md +// §6.6): the save is the only place a caller learns that a record was +// dropped, so the report is required here where it is optional everywhere +// else. A surface that let a caller retain, save and never find out would +// be a promise it could not check. +inline int64_t PairsSaveRetain( const Pairs * root, TableRetain * retain, uint8_t * buffer, int64_t capacity, TableReport * report ) +{ + if ( root == NULL || report == NULL ) { return -1; } + TableRegionCtx ctx; + return PairsSaveWireRetain( ctx, *root, retain, buffer, capacity, report ); +} + +// PairsSaveRetainMessages: RETENTION WRITING FORM 2 IS REFUSED BY NAME +// (docs/SPEC-TABLES.md §3.3). It is a MISUSE refusal on §6.6's own +// precedent and never a silent drop, and the two answers are named: a +// caller that must carry unknowns across a rewrite writes the FILE form, +// which carries its own table and takes §6.6 unchanged, and a RELAY +// forwards the sending peer's announcement and its batch bytes verbatim. +template +inline int64_t PairsSaveRetainMessages( Args &&... ) +{ + static_assert( sizeof...( Args ) == (size_t) -1, + "Pairs: a form 2 writer names entries through slots of a vocabulary the compiler settled, and a retained id is one this build's closure does not contain, so it has neither a slot nor an announced shape. Retention writing the MESSAGE form is refused by name (docs/SPEC-TABLES.md §3.3). Write the FILE form, which carries its own table and takes §6.6 unchanged, or relay the sender's announcement and batch bytes verbatim." ); + return -1; +} + +// ---- the cooked form: point at a cook (docs/SPEC-TABLES.md §7) ---- + +// PairsOpen: match the header and POINT. On a match the bytes ARE what this +// build wrote, in this build's layout and this build's byte order, so there +// is nothing to validate and nothing to fix up and the root comes back as it +// lies. On ANY refusal it returns NULL and NAMES the refusal in the caller's +// TableRefuseReason, the first failing clause in §7's order (a wrong build +// version is a re-cook, a foreign order a cross-endian cook, a truncated +// file a bad download, an unaligned base the caller's own buffer), and the +// caller falls back to a wire load, which is the path that carries every +// version. The reason is written on the refusal path only; a caller that +// passes nothing gets the null alone. +// +// It is O(1) IN THE FILE'S SIZE — the header and nothing per node — so a one +// megabyte cook and a one gigabyte cook open in the same time, and a mapped +// file's pages are touched only as they are used. That is a property of +// touching nothing at open rather than a separate mechanism. +// +// A REFERENCE INSIDE THE REGION IS DEREFERENCED THROUGH PairsAt: the slot holds +// the signed self-relative byte delta of §6.3, so a deref is one add and +// needs no base pointer, a whole region relocates by plain memcpy, and a +// delta of zero is null. +// +// There is ONE entry point and no tolerant twin: a build either wrote this +// file or it did not, and the build version is what says which. Validating a +// file whose provenance a person doubts is schema cook-check, offline, +// over the ATTRIBUTION part beside the data — a person's decision, never a +// parameter on a load. +inline const Pairs * PairsOpen( const void * bytes, uint64_t length, TableRefuseReason * reason = NULL ) +{ + return (const Pairs *) TableCookOpen( bytes, length, (uint64_t) sizeof( Pairs ), (uint64_t) alignof( Pairs ), reason ); +} + +// ---- the cooked form: WRITE a cook (docs/SPEC-TABLES.md §7.6) ---- +// +// The bytes are `schema cook`'s, and the tool stays the reference: the two +// writers are held to one file, byte for byte, in both byte orders. A cook is +// content-addressed by (asset hash, build version), so two writers of one +// instance produce ONE artifact or the pair means nothing. + +template inline bool PairsSlotsEntryCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const PairsSlotsEntry & value, TableByteOrder order ); +template inline bool PairsCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Pairs & value, TableByteOrder order ); + +template inline bool PairsSlotsEntryCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const PairsSlotsEntry & value, TableByteOrder order ) +{ + table_cook_put( at + 0, (uint64_t) value.key, 4, order ); + for ( int32_t i = 0; i < 2; i++ ) // value: an array of pointers, every slot + { + if ( !table_cook_ref( region, at + 8 + i * 8, (const void *) ItemAt( ctx, value.value[ i ] ), order ) ) { return false; } + } + return true; +} + +template inline bool PairsCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Pairs & value, TableByteOrder order ) +{ + (void) ctx; (void) region; // no reference resolves in this body: a list's and a map's slots are the extent writer's, and the class was decided elsewhere in the closure + table_cook_put( at + 0, 0, 8, order ); // slots: the array's delta, filled by the extent writer + table_cook_put( at + 8, 0, 4, order ); // and its count + table_cook_put( at + 16, (uint64_t) value.after, 4, order ); + return true; +} + +template inline bool PairsSlotsEntryCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const PairsSlotsEntry & value, TableByteOrder order ); +template inline bool PairsCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const Pairs & value, TableByteOrder order ); + +// PairsSlotsEntryCookExtent: PairsSlotsEntry's arrays into the node's extent, PRE-ORDER, a map's entries +// in ASCENDING key order and a list's elements in INDEX order, each through its +// own cook writer (§2.8, §2.9, §7.6). +template inline bool PairsSlotsEntryCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const PairsSlotsEntry & value, TableByteOrder order ) +{ + (void) ctx; (void) region; (void) extent; (void) at; (void) record; (void) value; (void) order; + return true; // no list or map below this record +} + +// PairsCookExtent: Pairs's arrays into the node's extent, PRE-ORDER, a map's entries +// in ASCENDING key order and a list's elements in INDEX order, each through its +// own cook writer (§2.8, §2.9, §7.6). +template inline bool PairsCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const Pairs & value, TableByteOrder order ) +{ + (void) region; // a table element's and an entry's references resolve through their own bodies + { // slots + TableMapCursor cursor = TableMapOrder( ctx, value.slots ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( PairsSlotsEntry ) + uint8_t * array = extent + at; + at += (int64_t) cursor.count * 24; // the whole array FIRST + // the SIXTEEN BYTES of the slot: the self-relative delta, then the count + table_cook_put( record + 0, cursor.count > 0 ? (uint64_t) (int64_t) ( array - ( record + 0 ) ) : 0, 8, order ); + table_cook_put( record + 8, (uint64_t) (uint32_t) cursor.count, 4, order ); + for ( int32_t i = 0; i < cursor.count; i++ ) + { + if ( !PairsSlotsEntryCookBody( ctx, region, array + i * 24, *cursor[i], order ) ) { return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// PairsSlotsEntryCookNode: one node, the record, then the extent its lists and maps take (§2.8, §2.9). +template inline bool PairsSlotsEntryCookNode( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const PairsSlotsEntry & value, TableByteOrder order ) +{ + if ( !PairsSlotsEntryCookBody( ctx, region, at, value, order ) ) { return false; } + int64_t extent_at = 0; + return PairsSlotsEntryCookExtent( ctx, region, at + 24, extent_at, at, value, order ); +} + +// PairsCookNode: one node, the record, then the extent its lists and maps take (§2.8, §2.9). +template inline bool PairsCookNode( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Pairs & value, TableByteOrder order ) +{ + if ( !PairsCookBody( ctx, region, at, value, order ) ) { return false; } + int64_t extent_at = 0; + if ( !PairsCookExtent( ctx, region, at + 24, extent_at, at, value, order ) ) { return false; } + return extent_at == PairsExtent( ctx, value ); // the extent written is the extent measured, or no header is written +} + +// PairsCookLayout: the tool's own Layout (docs/SPEC-TABLES.md §7.2) over one +// numbering — the root at zero, then every node in index order at +// align_up( offset, alignof ) for its OWN type, no slack between them, the +// data length rounded to the greatest alignment among them and never below +// eight. The offsets go into the region's table when it has one, and are only +// summed when it does not (a measure). A type id the numbering carries that +// this root cannot name is the two walks disagreeing, and it is refused. +// A NODE'S SIZE DEPENDS ON ITS VALUE where a list or a map rides in its extent +// (docs/SPEC-TABLES.md §2.8), so the layout takes the resolution context +// the numbering walked and reads the same arrays that walk read. +template +inline bool PairsCookLayout( const Ctx & ctx, const Pairs & root, const TableNumbering & numbering, TableCookRegion & region ) +{ + region.numbering = &numbering; + region.count = numbering.count + 1; + const int64_t root_extent = PairsExtent( ctx, root ); + if ( root_extent < 0 ) { return false; } + int64_t offset = 24 + root_extent; // the root at zero, its extent behind it + int64_t align = 8; + if ( region.offsets != NULL ) { region.offsets[0] = 0; } + for ( int64_t k = 0; k < numbering.count; k++ ) + { + int64_t size = 0; + int64_t node_align = 0; + switch ( numbering.entries[k].type_id ) + { + case 0x52cfa1d198476806ull: size = 4; node_align = 4; break; // Item + default: return false; + } + offset = ( offset + node_align - 1 ) & ~( node_align - 1 ); + if ( region.offsets != NULL ) { region.offsets[k + 1] = offset; } + offset += size; + if ( node_align > align ) { align = node_align; } + } + region.bytes = ( offset + align - 1 ) & ~( align - 1 ); + region.align = align; + return true; +} + +// PairsCookMeasureFrom: the whole cooked file's bytes for one graph — the header, +// the data part and the attribution part (§7.1). IT DEPENDS ON THE VALUE, +// because the answer is the numbering: the depth-first walk of §3.1 is run +// here and run again by the write, and neither carries the other's (§7.6). A +// data cycle is refused by the walk and answers -1. +template +inline int64_t PairsCookMeasureFrom( const Ctx & ctx, const Pairs & root, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + TableCookRegion region; + int64_t bytes = -1; + if ( PairsNumberFrom( ctx, numbering, root ) && PairsCookLayout( ctx, root, numbering, region ) ) + { + const int64_t data_offset = ( kTableCookHeaderBytes + region.align - 1 ) & ~( region.align - 1 ); + bytes = data_offset + region.bytes + region.count * (int64_t) sizeof( TableNodeDirEntry ); + } + TableNumberingShutdown( numbering ); + return bytes; +} + +// PairsCookFrom: write one cooked file of a pointered graph, in the byte order +// the caller names. The bytes are `schema cook`'s, byte for byte (§7.6). +// +// THE CALLER OWNS THE OUTPUT and nothing is allocated toward it. What is +// allocated is the numbering — the identity map, the entry array and one +// offset per node — through the pair handed in, and released before this +// returns (§6.5, §13.9). A capacity short of the measure writes nothing. +// +// THE HEADER IS WRITTEN LAST. A reference the numbering did not carry is +// found while a body is being written, and a write that refuses there has +// already put bytes in the buffer; with no magic ahead of them, no Open can +// mistake them for a cook. +template +inline bool PairsCookFrom( const Ctx & ctx, const Pairs & root, void * out, uint64_t capacity, TableByteOrder order, TableAllocator allocator ) +{ + if ( out == NULL ) { return false; } + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + TableCookRegion region; + bool ok = PairsNumberFrom( ctx, numbering, root ); + if ( ok ) + { + region.offsets = (int64_t *) allocator.alloc( allocator.context, ( numbering.count + 1 ) * (int64_t) sizeof( int64_t ) ); + ok = region.offsets != NULL && PairsCookLayout( ctx, root, numbering, region ); + } + if ( ok ) + { + const int64_t data_offset = ( kTableCookHeaderBytes + region.align - 1 ) & ~( region.align - 1 ); + const int64_t attribution = region.count * (int64_t) sizeof( TableNodeDirEntry ); + const int64_t need = data_offset + region.bytes + attribution; + ok = (uint64_t) need <= capacity; + if ( ok ) + { + uint8_t * raw = (uint8_t *) out; + memset( raw, 0, (size_t) need ); // EVERY BYTE NO FIELD COVERS IS ZERO (§7.2) + region.base = raw + data_offset; + // the DATA part: the root at the region's base, then every numbered + // node at the offset the layout gave it, each through its own writer + ok = PairsCookNode( ctx, region, region.base, root, order ); + for ( int64_t k = 0; ok && k < numbering.count; k++ ) + { + uint8_t * at = region.base + region.offsets[k + 1]; + const void * node = numbering.entries[k].node; + switch ( numbering.entries[k].type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemCookNode( ctx, region, at, *(const Item *) node, order ); break; // Item + default: ok = false; break; + } + } + // the ATTRIBUTION part: the node directory (§6.3), one entry per node + // in index order, for `schema cook-check` + uint8_t * entry = raw + data_offset + region.bytes; + table_cook_put( entry, 0, 8, order ); + table_cook_put( entry + 8, 0x1404200dab337086ull, 8, order ); // the root: fnv1a64( "Pairs" ) + for ( int64_t k = 0; k < numbering.count; k++ ) + { + entry += sizeof( TableNodeDirEntry ); + table_cook_put( entry, (uint64_t) region.offsets[k + 1], 8, order ); + table_cook_put( entry + 8, numbering.entries[k].type_id, 8, order ); + } + // and the HEADER (§7.1), every word a u64 in the order the file is + // produced in; the two RESERVED words are the memset's zeros + if ( ok ) + { + table_cook_put( raw + 0, TableCookMagic, 8, order ); + table_cook_put( raw + 8, BuildVersion, 8, order ); + table_cook_put( raw + 16, (uint64_t) ( order == TableByteOrder::Big ? 2 : 1 ), 8, order ); + table_cook_put( raw + 24, (uint64_t) region.bytes, 8, order ); + table_cook_put( raw + 32, (uint64_t) attribution, 8, order ); + table_cook_put( raw + 40, (uint64_t) region.align, 8, order ); + } + } + } + allocator.free( allocator.context, region.offsets ); + TableNumberingShutdown( numbering ); + return ok; +} + +// PairsCookMeasure / PairsCook over a REGION root — a locked builder's AsConst, a +// region PairsLoad produced, or an opened cook — with the pair the numbering +// allocates through as an optional last argument, as the wire's own entries +// take it (§13.9). +inline int64_t PairsCookMeasure( const Pairs * root, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return PairsCookMeasureFrom( ctx, *root, allocator ); +} + +inline bool PairsCook( const Pairs * root, void * out, uint64_t capacity, TableByteOrder order, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return false; } + TableRegionCtx ctx; + return PairsCookFrom( ctx, *root, out, capacity, order, allocator ); +} + +// and over a BUILDER, locked or not: the builder's own pair, and the arena +// encoding while it is still mutable (§6.3). +inline int64_t PairsCookMeasure( const PairsBuilder & builder ) +{ + if ( builder.region != NULL ) { return PairsCookMeasure( builder.AsConst(), builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return PairsCookMeasureFrom( ctx, *(const Pairs *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), builder.arena.allocator ); +} + +inline bool PairsCook( const PairsBuilder & builder, void * out, uint64_t capacity, TableByteOrder order ) +{ + if ( builder.region != NULL ) { return PairsCook( builder.AsConst(), out, capacity, order, builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return false; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return PairsCookFrom( ctx, *(const Pairs *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), out, capacity, order, builder.arena.allocator ); +} + +// ---- relocatability, enforced: the wire is a pure length-prefixed +// stream AND the decoded storage is pointer-free — every closure type +// must stay trivially copyable and standard-layout, so instances can be +// memcpy'd, mmap'd, shared across processes, and walked through +// descriptor offsets. A failure here means a pointer, virtual or +// non-trivial member crept into generated storage. +// +// They ask the COMPILER ITSELF, which is what every C++ standard library +// answers the same two questions with — and it costs this header no +// include at all. +// A pointer FIELD is a TableRef — eight bytes and no address — so the +// property holds in BOTH forms: a fixed-size table is one relocatable +// struct, and a packed region is one relocatable block whose references +// are self-relative and therefore survive a plain memcpy. +static_assert( __is_trivially_copyable( PairsSlotsEntry ), "PairsSlotsEntry must stay relocatable" ); +static_assert( __is_standard_layout( PairsSlotsEntry ), "PairsSlotsEntry must stay standard-layout for offsetof" ); +static_assert( __is_trivially_copyable( Pairs ), "Pairs must stay relocatable" ); +static_assert( __is_standard_layout( Pairs ), "Pairs must stay standard-layout for offsetof" ); + +// ---- the cook's layout contract (docs/SPEC-TABLES.md §20.3) ---- +// +// The compiler derived every number below from the declaration and folded it +// into the BUILD VERSION; these asserts are this compiler saying whether it +// agrees. The model is not self-evidently right — on 32-bit System V +// alignof(uint64_t) is 4, not 8 — which is precisely why it is asserted +// rather than assumed. +static_assert( sizeof( PairsSlotsEntry ) == 24, "PairsSlotsEntry's sizeof moved: the build version was taken over 24, so a cook of it would not be this build's file (docs/SPEC-TABLES.md §20.3)" ); +static_assert( alignof( PairsSlotsEntry ) == 8, "PairsSlotsEntry's alignof moved: the build version was taken over 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( PairsSlotsEntry, key ) == 0, "PairsSlotsEntry's field key moved: the build version was taken over offset 0 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( PairsSlotsEntry, value ) == 8, "PairsSlotsEntry's field value moved: the build version was taken over offset 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( sizeof( Pairs ) == 24, "Pairs's sizeof moved: the build version was taken over 24, so a cook of it would not be this build's file (docs/SPEC-TABLES.md §20.3)" ); +static_assert( alignof( Pairs ) == 8, "Pairs's alignof moved: the build version was taken over 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( Pairs, slots ) == 0, "Pairs's field slots moved: the build version was taken over offset 0 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( Pairs, after ) == 16, "Pairs's field after moved: the build version was taken over offset 16 (docs/SPEC-TABLES.md §20.3)" ); + + +// ---- reflection descriptors (tables only, docs/SPEC-TABLES.md) ---- + +inline const TableTypeInfo * PairsSlotsEntryTableType(); +inline const TableTypeInfo * PairsTableType(); +// The descriptors are CONSTANT-INITIALISED data, and a field's target is +// the ADDRESS of another descriptor. These declarations are what let a +// self- or mutually-referential graph — Node naming itself through *Node — +// be expressed as constant data instead of a lazy link, which could not +// have been written race-free OR recursion-safe. The whole reflection +// surface is therefore immutable: read it from any thread, any time. +extern const TableTypeInfo PairsSlotsEntryTableInfo; +extern const TableTypeInfo PairsTableInfo; + +inline const TableFieldInfo PairsSlotsEntryTableFields[] = { + { "key", "key", "uint32", 0x3dc94a19365b10ecull, 8, false, false, NULL, NULL, false, false, 0, (uint32_t) offsetof( PairsSlotsEntry, key ), (uint32_t) sizeof( PairsSlotsEntry::key ), 0xffffffffu, 0xffffffffu, NULL, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, + { "value", "value", "Item", 0x7ce4fd9430e80ceaull, 17, true, true, []( const void * slot ) -> const void * { return (const void *) ItemAt( *(const TableRef *) slot ); }, []( TableWorker & worker, void * slot ) -> void * { return (void *) ItemEmplace( worker, *(TableRef *) slot ); }, false, false, 2, (uint32_t) offsetof( PairsSlotsEntry, value ), (uint32_t) sizeof( PairsSlotsEntry::value[0] ), 0xffffffffu, 0xffffffffu, &ItemTableInfo, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, +}; +inline const TableTypeInfo PairsSlotsEntryTableInfo = { "PairsSlotsEntry", (uint32_t) sizeof( PairsSlotsEntry ), 2, PairsSlotsEntryTableFields, +[]( void * p ) { PairsSlotsEntryReset( *(PairsSlotsEntry *) p ); }, true, TableDocNone, 0, NULL }; +inline const TableTypeInfo * PairsSlotsEntryTableType() { return &PairsSlotsEntryTableInfo; } + +inline const TableFieldInfo PairsTableFields[] = { + { "slots", "slots", "map[uint32]*Item", 0xe68c2e6bb1ee5646ull, 13, true, false, NULL, NULL, true, false, 0, (uint32_t) offsetof( Pairs, slots ), (uint32_t) sizeof( PairsSlotsEntry ), (uint32_t) offsetof( Pairs, slots.count ), 0xffffffffu, &PairsSlotsEntryTableInfo, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, []( TableWorker & worker, void * slot, const char *, int32_t, int64_t key_value ) -> void * { return (void *) TableMapPlace( worker, *(TableMap *) slot, (uint32_t) key_value ); }, "", TableDocNone, 0, NULL }, + { "after", "after", "int32", 0xbf82010f6f71eae9ull, 4, false, false, NULL, NULL, false, false, 0, (uint32_t) offsetof( Pairs, after ), (uint32_t) sizeof( Pairs::after ), 0xffffffffu, 0xffffffffu, NULL, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, +}; +inline const TableTypeInfo PairsTableInfo = { "Pairs", (uint32_t) sizeof( Pairs ), 2, PairsTableFields, +[]( void * p ) { PairsReset( *(Pairs *) p ); }, true, TableDocNone, 0, NULL }; +inline const TableTypeInfo * PairsTableType() { return &PairsTableInfo; } + +// ---- the text form (docs/SPEC-TABLES.md §16) ---- + +// Pairs in and out of a JSON text (docs/SPEC-TABLES.md §16.7): read into a +// builder, written from a region's const root. A node named more than once +// carries `&node` in the text. Defined in PairsTable.cpp; link it to use them. +bool PairsFromJson( PairsBuilder & builder, const char * text, int64_t bytes, TableReport * report ); +int64_t PairsToJsonMeasure( const Pairs * root, TableAllocator allocator = TableDefaultAllocator() ); +int64_t PairsToJson( const Pairs * root, char * buffer, int64_t capacity, TableAllocator allocator = TableDefaultAllocator() ); + +} // namespace mapdemo diff --git a/testdata/golden/tables/maps/RowsTable.h b/testdata/golden/tables/maps/RowsTable.h index e3c970741..6c76612df 100644 --- a/testdata/golden/tables/maps/RowsTable.h +++ b/testdata/golden/tables/maps/RowsTable.h @@ -377,7 +377,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -879,13 +879,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1407,7 +1407,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1425,10 +1425,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1437,6 +1437,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1445,54 +1446,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3430,25 +3440,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -5692,7 +5704,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -7291,7 +7303,7 @@ inline bool RowSaveMessageBody( const Ctx & ctx, const TableNumbering & numberin if ( !order_entries.ok ) { return false; } // the sort could not run if ( order_entries.count > 0 ) { - w.put( 34, kTableMessageRefBitsHere ); + w.put( 38, kTableMessageRefBitsHere ); w.put( (uint64_t) order_entries.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_entries.count; i++ ) { @@ -7302,7 +7314,7 @@ inline bool RowSaveMessageBody( const Ctx & ctx, const TableNumbering & numberin } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body @@ -8004,7 +8016,7 @@ inline bool WideRowSaveMessageBody( const Ctx & ctx, const TableNumbering & numb if ( !order_entries.ok ) { return false; } // the sort could not run if ( order_entries.count > 0 ) { - w.put( 34, kTableMessageRefBitsHere ); + w.put( 38, kTableMessageRefBitsHere ); w.put( (uint64_t) order_entries.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_entries.count; i++ ) { @@ -8015,7 +8027,7 @@ inline bool WideRowSaveMessageBody( const Ctx & ctx, const TableNumbering & numb } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body @@ -8352,7 +8364,7 @@ inline bool EdgeRowNamesEntrySaveMessageBody( TableBitWriter & w, const EdgeRowN if ( value.key_length < 0 || value.key_length > 300 ) { return false; } // storage invariant if ( value.key_length > 0 ) { - w.put( 25, kTableMessageRefBitsHere ); + w.put( 28, kTableMessageRefBitsHere ); w.put( (uint64_t) value.key_length, 9 ); w.align(); // a string or a bytes ALIGNS before its bytes w.putbytes( (const uint8_t *) value.key, value.key_length ); @@ -8593,7 +8605,7 @@ inline bool EdgeRowIdsEntrySaveMessageBody( TableBitWriter & w, const EdgeRowIds { if ( value.key != 0 ) { - w.put( 27, kTableMessageRefBitsHere ); + w.put( 30, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.key ), 64 ); } { @@ -9106,7 +9118,7 @@ inline bool EdgeRowSaveMessageBody( const Ctx & ctx, const TableNumbering & numb if ( !order_names.ok ) { return false; } // the sort could not run if ( order_names.count > 0 ) { - w.put( 24, kTableMessageRefBitsHere ); + w.put( 27, kTableMessageRefBitsHere ); w.put( (uint64_t) order_names.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_names.count; i++ ) { @@ -9120,7 +9132,7 @@ inline bool EdgeRowSaveMessageBody( const Ctx & ctx, const TableNumbering & numb if ( !order_ids.ok ) { return false; } // the sort could not run if ( order_ids.count > 0 ) { - w.put( 26, kTableMessageRefBitsHere ); + w.put( 29, kTableMessageRefBitsHere ); w.put( (uint64_t) order_ids.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_ids.count; i++ ) { @@ -9131,7 +9143,7 @@ inline bool EdgeRowSaveMessageBody( const Ctx & ctx, const TableNumbering & numb } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/RunsTable.h b/testdata/golden/tables/maps/RunsTable.h index 2eb12b0c3..1a17d617e 100644 --- a/testdata/golden/tables/maps/RunsTable.h +++ b/testdata/golden/tables/maps/RunsTable.h @@ -377,7 +377,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -879,13 +879,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1407,7 +1407,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1425,10 +1425,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1437,6 +1437,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1445,54 +1446,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3430,25 +3440,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -3753,6 +3765,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4129,6 +4151,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -5677,7 +5704,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6416,13 +6443,13 @@ inline bool RunsSpansEntrySaveMessageBody( TableBitWriter & w, const RunsSpansEn { if ( value.key != 0 ) { - w.put( 13, kTableMessageRefBitsHere ); + w.put( 14, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.key ), 16 ); } if ( value.value_count < 0 || value.value_count > 4 ) { return false; } // storage invariant if ( value.value_count > 0 ) { - w.put( 36, kTableMessageRefBitsHere ); + w.put( 40, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.value_count ) - 0, 3 ); for ( int32_t i = 0; i < value.value_count; i++ ) { @@ -6807,7 +6834,7 @@ inline bool RunsSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi if ( !order_spans.ok ) { return false; } // the sort could not run if ( order_spans.count > 0 ) { - w.put( 35, kTableMessageRefBitsHere ); + w.put( 39, kTableMessageRefBitsHere ); w.put( (uint64_t) order_spans.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_spans.count; i++ ) { @@ -6818,7 +6845,7 @@ inline bool RunsSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/SlotsTable.h b/testdata/golden/tables/maps/SlotsTable.h index c2c0bed59..c205575b6 100644 --- a/testdata/golden/tables/maps/SlotsTable.h +++ b/testdata/golden/tables/maps/SlotsTable.h @@ -377,7 +377,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -879,13 +879,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1407,7 +1407,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1425,10 +1425,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1437,6 +1437,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1445,54 +1446,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3430,25 +3440,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -3753,6 +3765,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4129,6 +4151,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -5677,7 +5704,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6037,8 +6064,8 @@ inline bool TableEnumSlot( Slot value, uint64_t & slot ) switch ( value ) { case Slot::None: slot = 0; return true; - case Slot::Alpha: slot = 44; return true; - case Slot::Beta: slot = 45; return true; + case Slot::Alpha: slot = 50; return true; + case Slot::Beta: slot = 51; return true; default: return false; // no variant names this value: no wire identity } } @@ -6549,7 +6576,7 @@ inline bool SlotsSeatsEntrySaveMessageBody( TableBitWriter & w, const SlotsSeats { if ( value.key != 0 ) { - w.put( 11, kTableMessageRefBitsHere ); + w.put( 12, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.key ), 32 ); } { @@ -6562,7 +6589,7 @@ inline bool SlotsSeatsEntrySaveMessageBody( TableBitWriter & w, const SlotsSeats } if ( pairs_value > 0 ) { - w.put( 40, kTableMessageRefBitsHere ); + w.put( 44, kTableMessageRefBitsHere ); w.put( (uint64_t) pairs_value, 2 ); // ASCENDING BY VARIANT ORDINAL, which is slot order. It is // this writer's choice and a reader must not rely on it: every @@ -6980,7 +7007,7 @@ inline bool SlotsSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_seats.ok ) { return false; } // the sort could not run if ( order_seats.count > 0 ) { - w.put( 39, kTableMessageRefBitsHere ); + w.put( 43, kTableMessageRefBitsHere ); w.put( (uint64_t) order_seats.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_seats.count; i++ ) { @@ -6991,7 +7018,7 @@ inline bool SlotsSaveMessageBody( const Ctx & ctx, const TableNumbering & number } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/SpansTable.h b/testdata/golden/tables/maps/SpansTable.h index 580da8759..1014e660d 100644 --- a/testdata/golden/tables/maps/SpansTable.h +++ b/testdata/golden/tables/maps/SpansTable.h @@ -377,7 +377,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -879,13 +879,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1407,7 +1407,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1425,10 +1425,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1437,6 +1437,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1445,54 +1446,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3430,25 +3440,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -3753,6 +3765,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4129,6 +4151,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -5677,7 +5704,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6841,7 +6868,7 @@ inline bool SpansSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_tracks.ok ) { return false; } // the sort could not run if ( order_tracks.count > 0 ) { - w.put( 41, kTableMessageRefBitsHere ); + w.put( 45, kTableMessageRefBitsHere ); w.put( (uint64_t) order_tracks.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_tracks.count; i++ ) { @@ -6852,7 +6879,7 @@ inline bool SpansSaveMessageBody( const Ctx & ctx, const TableNumbering & number } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/TextTable.h b/testdata/golden/tables/maps/TextTable.h index e81498914..99a0cf947 100644 --- a/testdata/golden/tables/maps/TextTable.h +++ b/testdata/golden/tables/maps/TextTable.h @@ -376,7 +376,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -878,13 +878,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1406,7 +1406,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1424,10 +1424,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1436,6 +1436,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1444,54 +1445,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3429,25 +3439,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -5691,7 +5703,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6718,7 +6730,7 @@ inline bool TextNamesEntrySaveMessageBody( TableBitWriter & w, const TextNamesEn if ( value.value_length < 0 || value.value_length > 16 ) { return false; } // storage invariant if ( value.value_length > 0 ) { - w.put( 15, kTableMessageRefBitsHere ); + w.put( 16, kTableMessageRefBitsHere ); w.put( (uint64_t) value.value_length, 5 ); w.align(); // a string or a bytes ALIGNS before its bytes w.putbytes( (const uint8_t *) value.value, value.value_length ); @@ -6951,13 +6963,13 @@ inline bool TextWideEntrySaveMessageBody( TableBitWriter & w, const TextWideEntr { if ( value.key != 0 ) { - w.put( 13, kTableMessageRefBitsHere ); + w.put( 14, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.key ), 16 ); } if ( value.value_length < 0 || value.value_length > 6 ) { return false; } // storage invariant if ( value.value_length > 0 ) { - w.put( 14, kTableMessageRefBitsHere ); + w.put( 15, kTableMessageRefBitsHere ); w.put( (uint64_t) value.value_length, 3 ); for ( int32_t i = 0; i < value.value_length; i++ ) { w.put( (uint64_t) (uint16_t) value.value[i], 16 ); } // sixteen bits a unit, no align } @@ -7246,13 +7258,13 @@ inline bool TextBlobsEntrySaveMessageBody( TableBitWriter & w, const TextBlobsEn { if ( value.key != 0 ) { - w.put( 11, kTableMessageRefBitsHere ); + w.put( 12, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.key ), 32 ); } if ( value.value_length < 0 || value.value_length > 10 ) { return false; } // storage invariant if ( value.value_length > 0 ) { - w.put( 12, kTableMessageRefBitsHere ); + w.put( 13, kTableMessageRefBitsHere ); w.put( (uint64_t) value.value_length, 4 ); w.align(); // a string or a bytes ALIGNS before its bytes w.putbytes( (const uint8_t *) value.value, value.value_length ); @@ -7930,7 +7942,7 @@ inline bool TextSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi if ( !order_names.ok ) { return false; } // the sort could not run if ( order_names.count > 0 ) { - w.put( 24, kTableMessageRefBitsHere ); + w.put( 27, kTableMessageRefBitsHere ); w.put( (uint64_t) order_names.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_names.count; i++ ) { @@ -7944,7 +7956,7 @@ inline bool TextSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi if ( !order_wide.ok ) { return false; } // the sort could not run if ( order_wide.count > 0 ) { - w.put( 43, kTableMessageRefBitsHere ); + w.put( 47, kTableMessageRefBitsHere ); w.put( (uint64_t) order_wide.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_wide.count; i++ ) { @@ -7958,7 +7970,7 @@ inline bool TextSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi if ( !order_blobs.ok ) { return false; } // the sort could not run if ( order_blobs.count > 0 ) { - w.put( 18, kTableMessageRefBitsHere ); + w.put( 19, kTableMessageRefBitsHere ); w.put( (uint64_t) order_blobs.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_blobs.count; i++ ) { @@ -7969,7 +7981,7 @@ inline bool TextSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/TrailsTable.cpp b/testdata/golden/tables/maps/TrailsTable.cpp new file mode 100644 index 000000000..37ea897b5 --- /dev/null +++ b/testdata/golden/tables/maps/TrailsTable.cpp @@ -0,0 +1,3448 @@ +// Code generated by the schema compiler from Trails.schema. DO NOT EDIT. +// SPDX-License-Identifier: NONE — this generated output is yours, under terms of +// your choice. See the LICENSE exception in the schema compiler; the compiler is +// AGPL-3.0, its output is not. +// package mapdemo — the TABLE wire's text form (docs/SPEC-TABLES.md §16). +// Compile this file to use FromJson / ToJson; a project that +// never reads or writes a text does not compile it and pays nothing. + +#include "TrailsTable.h" + +#include // the text form: number formatting +#include // the text form: exact number conversion +#include // the text form: the runtime's decimal point + +// The guard is not vestigial. Several mapdemo Table.cpp files may be +// concatenated into ONE translation unit — a unity build — and without it +// each would redefine the walk. It is also why the walk's functions may be +// weak (vague linkage) across separate objects: ODR requires their +// definitions to be token-identical, and the generic-walk gate is what +// proves that, byte for byte, across every generated .cpp. +#ifndef MAPDEMO_SCHEMA_TABLE_JSON +#define MAPDEMO_SCHEMA_TABLE_JSON + +namespace mapdemo { + +// ---- the pointer adapters (docs/SPEC-TABLES.md §16.7) ---- +// +// The walk below is ONE walk, byte-identical in every generated .cpp, and a +// pointer is the one kind it cannot walk alone: reading one needs the +// builder's arena and writing one needs a region's deref, and neither exists +// in a unit that declares no pointer. So the walk calls these three and does +// not define them. A unit with no pointer defines them as stubs no field ever +// reaches; a pointered unit defines them in the graph half that follows the +// walk. + +struct TableJsonIn; +struct TableJsonOut; + +// a pointer field's object, or the `&node` reference standing in for it, into +// the slot; the cursor is on the opening brace +inline bool TableJsonReadPointer( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); +// the node a pointer slot names, in place — or as `&node` when it is shared +inline bool TableJsonWritePointer( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// the FIRST key of an object the walk is skipping begins with `&`: the cursor is +// on its value. A dropped definition still takes its label (§16.7); a fixed reader +// skips the value whole, as it skips everything else it does not place. +inline bool TableJsonSkippedAmpersand( TableJsonIn & in, const char * key, int32_t depth ); + +// ---- the map and list adapters (docs/SPEC-TABLES.md §2.8, §2.9, §16) ---- +// +// A MAP and an UNBOUNDED ARRAY are the other constructs the walk cannot walk +// alone: their arrays live behind a TableMap or a TableList this +// walk has no name for, reading one needs the builder's arena, and neither +// exists in a unit that declares neither construct. Same shape as the +// pointer's three: declared here, defined after the walk by whichever half +// the unit carries. Both are OUT-OF-LINE ARRAYS to the descriptors (§8.1): +// array_bound = 0 is the tell, and the type name says which of the two. + +// a map field: an out-of-line array whose type name spells the map +inline bool TableJsonIsMap( const TableFieldInfo * f ); +// the map as a plain JSON object keyed by the KEY, in ASCENDING key order +inline bool TableJsonWriteMap( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// that object back into the slot, in whatever order the text gives it +inline bool TableJsonReadMap( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); +// an unbounded array: the other out-of-line array +inline bool TableJsonIsList( const TableFieldInfo * f ); +// the list as a JSON array, in INDEX order +inline bool TableJsonWriteList( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// that array back into the slot, every element the text carries +inline bool TableJsonReadList( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); + +// ---- json walk: begin ---- +// +// The TEXT form (docs/SPEC-TABLES.md §16): one table, one text, one walk over the +// reflection descriptors (§8). Reading fills ONE caller-owned instance and +// allocates nothing beyond it; writing targets a caller buffer with the +// wire's measure/write symmetry. Everything AROUND this — which file goes +// with which instance, what key an instance is filed under, how instances +// link into a root table's collections — is a packer's opinion and stays +// with the tool that holds it. +// +// The dialect: trailing commas are accepted on read (the authoring files +// this exists for carry them) and never written; comments are not JSON and +// are refused; unknown keys are skipped and counted; a duplicate key is +// last-wins and counted; a key present with the wrong JSON type is skipped +// and counted, never coerced. + +static const int32_t kTableJsonMaxDepth = 128; + +// A key longer than this cannot name a field, so it is skipped as unknown. +static const int32_t kTableJsonMaxKey = 256; + +// The longest numeric token the walk will convert. Anything longer is a +// value no field can hold and counts as a kind mismatch. +static const int32_t kTableJsonMaxNumber = 512; + +// The decimal point the C runtime is CURRENTLY using. Number conversion is +// the one locale-sensitive corner of the grammar — JSON's point is always +// '.', the runtime's is whatever the program set — so every number crosses +// this one character on the way out and on the way back in. Nothing else in +// the walk consults the locale. +inline char TableJsonDecimalPoint() +{ + const struct lconv * conv = localeconv(); + if ( conv != NULL && conv->decimal_point != NULL && conv->decimal_point[0] != 0 ) + { + return conv->decimal_point[0]; + } + return '.'; +} + +// ---- storage access: the descriptors give an offset and a width, and the +// ---- storage is the HOST's, so every load and store goes through a width +// ---- switch rather than a memcpy into the low bytes of a wider word + +// finite: not a NaN, not an infinity. Written without — the walk's +// runtime surface stays the handful of functions it already names. +// A vocabulary entry the descriptor could not spell. The generated name +// functions answer "???" for a value outside the declared set, and that is +// not a name — writing it would put a spelling in the text that the reader +// then counts as unknown, turning a refusal into a silent loss. +inline bool TableJsonNamed( const char * name ) +{ + return name != NULL && strcmp( name, "???" ) != 0; +} + +inline bool TableJsonFinite( double v ) +{ + return v == v && v <= 1.7976931348623157e308 && v >= -1.7976931348623157e308; +} + +inline uint64_t TableJsonGetRaw( const void * storage, uint32_t width ) +{ + switch ( width ) + { + case 1: { uint8_t v = 0; memcpy( &v, storage, 1 ); return v; } + case 2: { uint16_t v = 0; memcpy( &v, storage, 2 ); return v; } + case 4: { uint32_t v = 0; memcpy( &v, storage, 4 ); return v; } + case 8: { uint64_t v = 0; memcpy( &v, storage, 8 ); return v; } + } + return 0; +} + +inline void TableJsonSetRaw( void * storage, uint32_t width, uint64_t value ) +{ + switch ( width ) + { + case 1: { uint8_t v = (uint8_t) value; memcpy( storage, &v, 1 ); break; } + case 2: { uint16_t v = (uint16_t) value; memcpy( storage, &v, 2 ); break; } + case 4: { uint32_t v = (uint32_t) value; memcpy( storage, &v, 4 ); break; } + case 8: { uint64_t v = value; memcpy( storage, &v, 8 ); break; } + } +} + +inline int64_t TableJsonGetSigned( const void * storage, uint32_t width ) +{ + uint64_t raw = TableJsonGetRaw( storage, width ); + if ( width < 8 ) + { + uint64_t sign = uint64_t( 1 ) << ( width * 8 - 1 ); + if ( ( raw & sign ) != 0 ) + { + raw |= ~( ( sign << 1 ) - 1 ); + } + } + return (int64_t) raw; +} + +// ---- the WIDE kinds (docs/SPEC-TABLES.md §3, §16.2) ---- +// +// The 128-bit integers and the fixed-point family convert EXACTLY, over two +// 64-bit lanes: a 128-bit integer is a decimal integer, a fixed value a +// decimal in WHOLE UNITS (1.0, -0.25, 3.0000152587890625) and nothing +// on either path passes through a double. Nothing here needs a 128-bit type +// either, which is what keeps this walk one text for every unit. +struct TableJsonWide +{ + uint64_t lo; + uint64_t hi; +}; + +inline bool TableJsonKindWide( uint8_t kind ) { return kind >= 18 && kind <= 29; } +inline bool TableJsonKindWideSigned( uint8_t kind ) { return kind == 18 || ( kind >= 20 && kind <= 24 ); } +inline bool TableJsonKindFixed( uint8_t kind ) { return kind >= 20 && kind <= 29; } + +inline bool TableJsonWideZero( TableJsonWide v ) { return v.lo == 0 && v.hi == 0; } +inline bool TableJsonWideNegative( TableJsonWide v ) { return ( v.hi >> 63 ) != 0; } + +inline int TableJsonWideCompare( TableJsonWide a, TableJsonWide b, bool is_signed ) +{ + if ( is_signed && TableJsonWideNegative( a ) != TableJsonWideNegative( b ) ) { return TableJsonWideNegative( a ) ? -1 : 1; } + if ( a.hi != b.hi ) { return a.hi < b.hi ? -1 : 1; } + if ( a.lo != b.lo ) { return a.lo < b.lo ? -1 : 1; } + return 0; +} + +inline TableJsonWide TableJsonWideShl( TableJsonWide v, int n ) +{ + TableJsonWide r = { 0, 0 }; + if ( n <= 0 ) { return v; } + if ( n >= 128 ) { return r; } + if ( n >= 64 ) { r.hi = v.lo << ( n - 64 ); return r; } + r.hi = ( v.hi << n ) | ( v.lo >> ( 64 - n ) ); + r.lo = v.lo << n; + return r; +} + +inline TableJsonWide TableJsonWideShr( TableJsonWide v, int n ) +{ + TableJsonWide r = { 0, 0 }; + if ( n <= 0 ) { return v; } + if ( n >= 128 ) { return r; } + if ( n >= 64 ) { r.lo = v.hi >> ( n - 64 ); return r; } + r.lo = ( v.lo >> n ) | ( v.hi << ( 64 - n ) ); + r.hi = v.hi >> n; + return r; +} + +inline TableJsonWide TableJsonWideNeg( TableJsonWide v ) +{ + TableJsonWide r; + r.lo = ~v.lo + 1; + r.hi = ~v.hi + ( r.lo == 0 ? 1 : 0 ); + return r; +} + +// v = v * m + a; the return is the carry out of 128 bits +inline uint32_t TableJsonWideMulAdd( TableJsonWide * v, uint32_t m, uint32_t a ) +{ + uint64_t limb[4] = { v->lo & 0xffffffffull, v->lo >> 32, v->hi & 0xffffffffull, v->hi >> 32 }; + uint64_t carry = a; + for ( int i = 0; i < 4; i++ ) + { + uint64_t p = limb[i] * m + carry; + limb[i] = p & 0xffffffffull; + carry = p >> 32; + } + v->lo = limb[0] | ( limb[1] << 32 ); + v->hi = limb[2] | ( limb[3] << 32 ); + return (uint32_t) carry; +} + +// v = v / d; the return is the remainder +inline uint32_t TableJsonWideDiv( TableJsonWide * v, uint32_t d ) +{ + uint64_t limb[4] = { v->lo & 0xffffffffull, v->lo >> 32, v->hi & 0xffffffffull, v->hi >> 32 }; + uint64_t rem = 0; + for ( int i = 3; i >= 0; i-- ) + { + uint64_t cur = ( rem << 32 ) | limb[i]; + limb[i] = cur / d; + rem = cur % d; + } + v->lo = limb[0] | ( limb[1] << 32 ); + v->hi = limb[2] | ( limb[3] << 32 ); + return (uint32_t) rem; +} + +// The storage of a wide kind, as lanes. A sixteen-byte storage is serialize's +// pair — native __int128 in the host's byte order, or the emulated struct with +// its low lane first — so the lanes are read in the host's order; a narrower +// storage is one lane, sign-extended for a signed kind. +inline TableJsonWide TableJsonWideLoad( const void * storage, uint32_t width, bool is_signed ) +{ + TableJsonWide v = { 0, 0 }; + if ( width == 16 ) + { + uint64_t half[2]; + memcpy( half, storage, 16 ); + uint16_t probe = 1; + bool little = *(const uint8_t *) &probe == 1; + v.lo = little ? half[0] : half[1]; + v.hi = little ? half[1] : half[0]; + return v; + } + v.lo = is_signed ? (uint64_t) TableJsonGetSigned( storage, width ) : TableJsonGetRaw( storage, width ); + v.hi = ( is_signed && ( v.lo >> 63 ) != 0 ) ? ~uint64_t( 0 ) : 0; + return v; +} + +inline void TableJsonWideStore( void * storage, uint32_t width, TableJsonWide v ) +{ + if ( width == 16 ) + { + uint16_t probe = 1; + bool little = *(const uint8_t *) &probe == 1; + uint64_t half[2]; + half[0] = little ? v.lo : v.hi; + half[1] = little ? v.hi : v.lo; + memcpy( storage, half, 16 ); + return; + } + TableJsonSetRaw( storage, width, v.lo ); +} + +// a counted field's companion: a string's length, a bytes' length, a counted +// array's count. Bounded by the declared extent on the way out, so a storage +// invariant a caller broke cannot walk off the end of the array. +inline int32_t TableJsonCount( const void * base, const TableFieldInfo * f ) +{ + if ( !f->counted ) + { + return f->array_bound; + } + int32_t count = 0; + memcpy( &count, (const uint8_t *) base + f->count_offset, sizeof( count ) ); + if ( count < 0 ) { count = 0; } + if ( count > f->array_bound ) { count = f->array_bound; } + return count; +} + +inline void TableJsonSetCount( void * base, const TableFieldInfo * f, int32_t count ) +{ + if ( f->counted ) + { + memcpy( (uint8_t *) base + f->count_offset, &count, sizeof( count ) ); + } +} + +// ---- what a field's kind expects to see in the text ---- +// +// One classifier, consulted by both directions, so a reader and a writer can +// never disagree about a kind's JSON form. 'o' object, 'a' array, 's' +// string, 'n' number, 'b' boolean. +// +// A vocabulary field is spelled by NAME: an enum is one name, a flags mask +// is the array of the names of its set bits. The two are told apart by the +// id column — an enum variant rides under a wire id, a flags BIT never does +// (docs/SPEC-TABLES.md §4), so a name function with no id function is flags. +// +// bytes(N) is the one kind whose element kind does not decide its form: it +// shares u8 with a plain array of u8, and rides as base64. The schema type +// name settles it, and "bytes" is a keyword no declaration can claim. +inline bool TableJsonIsBytes( const TableFieldInfo * f ) +{ + return f->is_array && f->kind == 6 && strcmp( f->type_name, "bytes" ) == 0; +} + +// An ENUM-KEYED array (docs/SPEC-TABLES.md §2.4): its JSON form is an OBJECT +// keyed by variant name, not a positional array, because that is what the +// storage is — one slot per variant, addressed by the variant. +inline bool TableJsonIsKeyed( const TableFieldInfo * f ) +{ + return f->key_name != NULL; +} + +// THE KEY A STORAGE SLOT HOLDS (§2.4, §8): the storage shifts left, so slot i +// holds the key i + 1 and nothing is stored for None. This is the ONE place +// the walker spells the shift. +inline uint64_t TableJsonKeyedSlotKey( int64_t slot ) +{ + return (uint64_t) ( slot + 1 ); +} + +// A slot whose key names a variant of the keying enum. Every slot in +// [0, array_bound) does, unless the enum carries max-headroom variants outside +// a table closure, where a reserved value names nothing and its key id is 0 — +// the reserved id no declared name can fold to (§5). +inline bool TableJsonKeyedSlotValid( const TableFieldInfo * f, int64_t slot ) +{ + return f->key_id( TableJsonKeyedSlotKey( slot ) ) != 0; +} + +inline bool TableJsonIsFlags( const TableFieldInfo * f ) +{ + return f->enum_name != NULL && f->variant_id == NULL; +} + +inline bool TableJsonIsEnum( const TableFieldInfo * f ) +{ + return f->variant_id != NULL && f->arms == NULL; +} + +inline char TableJsonShape( const TableFieldInfo * f ) +{ + if ( TableJsonIsMap( f ) ) return 'o'; // a MAP: an object keyed by the KEY (§2.8) + if ( f->kind == 12 ) return 's'; // string + if ( f->kind == 33 ) return 's'; // wstring: the same text, transcoded (§16.2) + if ( TableJsonIsBytes( f ) ) return 's'; // bytes: base64 + if ( TableJsonIsKeyed( f ) ) return 'o'; // an object keyed by variant NAME + if ( f->is_array ) return 'a'; + if ( f->arms != NULL ) return 'o'; // union: an object with ONE key + if ( f->kind == 13 ) return 'o'; // nested table or type + if ( f->kind == 17 ) return f->table != NULL ? 'o' : 's'; // a pointer: the pointee's object in place, or null (§16.7); a byte buffer's string (§2.5) + if ( TableJsonIsEnum( f ) ) return 's'; + if ( TableJsonIsFlags( f ) ) return 'a'; + if ( f->kind == 1 ) return 'b'; + return 'n'; +} + +// the ELEMENT shape of an array field — the same classifier one level down +inline char TableJsonElementShape( const TableFieldInfo * f ) +{ + if ( f->arms != NULL ) return 'o'; // an element of an array of unions: one key, the arm (§2.6) + if ( f->kind == 13 ) return 'o'; + if ( TableJsonIsEnum( f ) ) return 's'; + if ( TableJsonIsFlags( f ) ) return 'a'; + if ( f->kind == 1 ) return 'b'; + return 'n'; +} + +// A guarded group rides only when its guard reads true — the wire's own +// elision (§4), carried into the text so a text and a wire written from one +// instance say the same thing. The guard is spelled as its branch condition +// over bool fields of the SAME type ("at_rest", "!at_rest", +// "active && has_target"), so evaluating it is a walk of the same +// descriptor. Nothing is inferred in the other direction: reading places +// every key it can name, and the guard is a plain bool key (§16.2). +inline bool TableJsonGuardHolds( const void * base, const TableTypeInfo * info, const char * guard ) +{ + const char * p = guard; + for ( ;; ) + { + while ( *p == ' ' || *p == '&' ) { p++; } + if ( *p == 0 ) { return true; } + bool want = true; + if ( *p == '!' ) { want = false; p++; } + const char * start = p; + while ( *p != 0 && *p != ' ' && *p != '&' ) { p++; } + size_t length = (size_t) ( p - start ); + bool value = false; + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + const TableFieldInfo * f = &info->fields[i]; + if ( strlen( f->name ) == length && strncmp( f->name, start, length ) == 0 ) + { + value = TableJsonGetRaw( (const uint8_t *) base + f->offset, f->elem_size ) != 0; + break; + } + } + if ( value != want ) { return false; } + } +} + +// ---- writing ---- + +// The writer sink MEASURES when the buffer is NULL and WRITES when it is +// not, over one code path — so measure and write agree byte for byte, the +// wire's invariant (§9) carried across. +struct TableJsonOut +{ + char * buffer; + int64_t capacity; + int64_t offset; + bool overflow; + void * graph; // the pointered write's identity map (§16.7); NULL for a fixed table + + void raw( const char * data, int64_t count ) + { + if ( buffer != NULL ) + { + if ( offset + count > capacity ) { overflow = true; return; } + memcpy( buffer + offset, data, (size_t) count ); + } + offset += count; + } + void put( char c ) { raw( &c, 1 ); } + void text( const char * s ) { raw( s, (int64_t) strlen( s ) ); } + void line( int32_t depth ) + { + put( '\n' ); + for ( int32_t i = 0; i < depth; i++ ) { raw( " ", 2 ); } + } +}; + +inline const char * TableJsonBase64Alphabet() +{ + return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +} + +inline void TableJsonWriteBase64( TableJsonOut & out, const uint8_t * data, int32_t length ) +{ + const char * alphabet = TableJsonBase64Alphabet(); + out.put( '"' ); + int32_t i = 0; + for ( ; i + 3 <= length; i += 3 ) + { + uint32_t triple = ( uint32_t( data[i] ) << 16 ) | ( uint32_t( data[i+1] ) << 8 ) | uint32_t( data[i+2] ); + char quad[4] = { alphabet[ ( triple >> 18 ) & 0x3f ], alphabet[ ( triple >> 12 ) & 0x3f ], + alphabet[ ( triple >> 6 ) & 0x3f ], alphabet[ triple & 0x3f ] }; + out.raw( quad, 4 ); + } + if ( i < length ) + { + int32_t left = length - i; + uint32_t triple = uint32_t( data[i] ) << 16; + if ( left == 2 ) { triple |= uint32_t( data[i+1] ) << 8; } + char quad[4] = { alphabet[ ( triple >> 18 ) & 0x3f ], alphabet[ ( triple >> 12 ) & 0x3f ], '=', '=' }; + if ( left == 2 ) { quad[2] = alphabet[ ( triple >> 6 ) & 0x3f ]; } + out.raw( quad, 4 ); + } + out.put( '"' ); +} + +// One UTF-8 sequence at s, or -1 when the bytes there are not one. Rejects +// the lot: a stray continuation, an overlong form, a surrogate half, and +// anything past U+10FFFF. +inline int32_t TableJsonUtf8( const char * s, int32_t remaining, int32_t * width ) +{ + unsigned char lead = (unsigned char) s[0]; + int32_t want = 0; + int32_t code = 0; + if ( lead < 0x80 ) { *width = 1; return lead; } + else if ( lead >= 0xc2 && lead <= 0xdf ) { want = 2; code = lead & 0x1f; } + else if ( lead >= 0xe0 && lead <= 0xef ) { want = 3; code = lead & 0x0f; } + else if ( lead >= 0xf0 && lead <= 0xf4 ) { want = 4; code = lead & 0x07; } + else { return -1; } + if ( remaining < want ) { return -1; } + for ( int32_t i = 1; i < want; i++ ) + { + unsigned char next = (unsigned char) s[i]; + if ( ( next & 0xc0 ) != 0x80 ) { return -1; } + code = ( code << 6 ) | ( next & 0x3f ); + } + if ( want == 3 && code < 0x800 ) { return -1; } // overlong + if ( want == 4 && code < 0x10000 ) { return -1; } // overlong + if ( code >= 0xd800 && code <= 0xdfff ) { return -1; } // a surrogate half + if ( code > 0x10ffff ) { return -1; } + *width = want; + return code; +} + +// The inverse: one code point encoded as UTF-8 into unit, its length +// answered. Both text kinds' writers reach it, and so does the escape +// grammar's U+FFFD replacement. +inline int32_t TableJsonEncodeUtf8( uint32_t code, char * unit ) +{ + if ( code < 0x80 ) { unit[0] = (char) code; return 1; } + if ( code < 0x800 ) + { + unit[0] = (char) ( 0xc0 | ( code >> 6 ) ); + unit[1] = (char) ( 0x80 | ( code & 0x3f ) ); + return 2; + } + if ( code < 0x10000 ) + { + unit[0] = (char) ( 0xe0 | ( code >> 12 ) ); + unit[1] = (char) ( 0x80 | ( ( code >> 6 ) & 0x3f ) ); + unit[2] = (char) ( 0x80 | ( code & 0x3f ) ); + return 3; + } + unit[0] = (char) ( 0xf0 | ( code >> 18 ) ); + unit[1] = (char) ( 0x80 | ( ( code >> 12 ) & 0x3f ) ); + unit[2] = (char) ( 0x80 | ( ( code >> 6 ) & 0x3f ) ); + unit[3] = (char) ( 0x80 | ( code & 0x3f ) ); + return 4; +} + +// A JSON text MUST be valid UTF-8 (RFC 8259 §8.1). The read path is +// byte-transparent — the wire imposes no encoding (§3) and a string may hold +// anything — so the WRITER is where that obligation is met: a byte that is +// not part of a well-formed sequence is written as U+FFFD, one per bad byte, +// and never raw. A text this walk writes is therefore readable by any +// conforming parser, which a raw byte would not be. The cost is stated +// plainly: for a string holding invalid UTF-8, the round trip is NOT +// byte-identical, because the alternative is emitting a text that is not +// JSON. +inline void TableJsonWriteString( TableJsonOut & out, const char * s, int32_t length ) +{ + static const char hex[] = "0123456789abcdef"; + out.put( '"' ); + for ( int32_t i = 0; i < length; i++ ) + { + unsigned char c = (unsigned char) s[i]; + switch ( c ) + { + case '"': out.raw( "\\\"", 2 ); break; + case '\\': out.raw( "\\\\", 2 ); break; + case '\b': out.raw( "\\b", 2 ); break; + case '\f': out.raw( "\\f", 2 ); break; + case '\n': out.raw( "\\n", 2 ); break; + case '\r': out.raw( "\\r", 2 ); break; + case '\t': out.raw( "\\t", 2 ); break; + default: + if ( c < 0x20 ) + { + char escape[6] = { '\\', 'u', '0', '0', hex[ c >> 4 ], hex[ c & 0xf ] }; + out.raw( escape, 6 ); + } + else if ( c < 0x80 ) + { + out.put( (char) c ); + } + else + { + int32_t width = 0; + if ( TableJsonUtf8( s + i, length - i, &width ) < 0 ) + { + out.raw( "\xef\xbf\xbd", 3 ); // U+FFFD, one per bad byte + } + else + { + out.raw( s + i, width ); + i += width - 1; + } + } + break; + } + } + out.put( '"' ); +} + +// A WIDE field's text: the code units transcoded back to UTF-8 (§16.2). A +// SURROGATE PAIR is one code point; an UNPAIRED SURROGATE is not a code point +// at all, encodes to nothing, and writes one U+FFFD per ill-formed unit; a +// ZERO UNIT is U+0000, which JSON has an escape for, and writes \u0000 +// (§16.3). No wire can put either into storage (§3), so both answer for +// storage a PROGRAM built. +inline void TableJsonWriteWString( TableJsonOut & out, const char16_t * s, int32_t length ) +{ + static const char hex[] = "0123456789abcdef"; + out.put( '"' ); + for ( int32_t i = 0; i < length; i++ ) + { + uint32_t code = (uint32_t) (uint16_t) s[i]; + if ( code >= 0xd800 && code <= 0xdbff && i + 1 < length ) + { + const uint32_t low = (uint32_t) (uint16_t) s[i + 1]; + if ( low >= 0xdc00 && low <= 0xdfff ) + { + code = 0x10000 + ( ( code - 0xd800 ) << 10 ) + ( low - 0xdc00 ); + i++; + } + } + if ( code >= 0xd800 && code <= 0xdfff ) { code = 0xfffd; } // an unpaired surrogate + switch ( code ) + { + case '"': out.raw( "\\\"", 2 ); continue; + case '\\': out.raw( "\\\\", 2 ); continue; + case '\b': out.raw( "\\b", 2 ); continue; + case '\f': out.raw( "\\f", 2 ); continue; + case '\n': out.raw( "\\n", 2 ); continue; + case '\r': out.raw( "\\r", 2 ); continue; + case '\t': out.raw( "\\t", 2 ); continue; + default: break; + } + if ( code < 0x20 ) + { + char escape[6] = { '\\', 'u', '0', '0', hex[ code >> 4 ], hex[ code & 0xf ] }; + out.raw( escape, 6 ); + continue; + } + char encoded[4]; + const int32_t encoded_length = TableJsonEncodeUtf8( code, encoded ); + out.raw( encoded, encoded_length ); + } + out.put( '"' ); +} + +inline void TableJsonWriteUnsigned( TableJsonOut & out, uint64_t value ) +{ + char digits[24]; + int32_t n = 0; + do + { + digits[n++] = (char) ( '0' + (int) ( value % 10 ) ); + value /= 10; + } while ( value != 0 ); + char text[24]; + for ( int32_t i = 0; i < n; i++ ) { text[i] = digits[n - 1 - i]; } + out.raw( text, n ); +} + +inline void TableJsonWriteSigned( TableJsonOut & out, int64_t value ) +{ + if ( value < 0 ) + { + out.put( '-' ); + TableJsonWriteUnsigned( out, uint64_t( 0 ) - (uint64_t) value ); + return; + } + TableJsonWriteUnsigned( out, (uint64_t) value ); +} + +// A wide kind writes its raw storage as §16.2's text: a 128-bit integer as a +// decimal integer; a fixed value in WHOLE UNITS as the shortest exact decimal +// with at least one fractional digit (1.0, -0.25), the spelling the schema text +// gives a fixed default. The fraction terminates because a dyadic fraction has +// a finite decimal expansion — at most F digits. +inline void TableJsonWriteWide( TableJsonOut & out, const void * storage, const TableFieldInfo * f ) +{ + bool is_signed = TableJsonKindWideSigned( f->kind ); + TableJsonWide v = TableJsonWideLoad( storage, f->elem_size, is_signed ); + if ( is_signed && TableJsonWideNegative( v ) ) + { + out.put( '-' ); + v = TableJsonWideNeg( v ); + } + int frac = f->frac_bits; + TableJsonWide whole = TableJsonWideShr( v, frac ); + char digits[40]; + int32_t n = 0; + do + { + digits[n++] = (char) ( '0' + (int) TableJsonWideDiv( &whole, 10 ) ); + } while ( !TableJsonWideZero( whole ) ); + char text[40]; + for ( int32_t i = 0; i < n; i++ ) { text[i] = digits[n - 1 - i]; } + out.raw( text, n ); + if ( !TableJsonKindFixed( f->kind ) ) { return; } + out.put( '.' ); + // the fraction bits alone: v with everything at and above bit F cleared + TableJsonWide fraction = v; + if ( frac < 64 ) { fraction.hi = 0; fraction.lo &= ( uint64_t( 1 ) << frac ) - 1; } + else { fraction.hi &= ( uint64_t( 1 ) << ( frac - 64 ) ) - 1; } + if ( frac == 0 ) { fraction.lo = 0; } + if ( TableJsonWideZero( fraction ) ) + { + out.put( '0' ); + return; + } + while ( !TableJsonWideZero( fraction ) ) + { + // ×10: the digit is what lands at and above bit F, including the + // carry out of 128 bits when F leaves no room for it below + uint32_t carry = TableJsonWideMulAdd( &fraction, 10, 0 ); + uint64_t digit = TableJsonWideShr( fraction, frac ).lo; + if ( frac > 64 ) { digit |= uint64_t( carry ) << ( 128 - frac ); } + out.put( (char) ( '0' + (int) digit ) ); + if ( frac < 64 ) { fraction.hi = 0; fraction.lo &= ( uint64_t( 1 ) << frac ) - 1; } + else { fraction.hi &= ( uint64_t( 1 ) << ( frac - 64 ) ) - 1; } + } +} + +// A float writes at the SHORTEST precision that reads back as the same value +// at the field's own width, so a round trip is exact and a text stays +// readable. Non-finite values have no JSON spelling at all, and the writer +// REFUSES rather than losing one silently — the same rule measure and save +// already apply to an enum value no variant names (§5). +inline bool TableJsonWriteFloat( TableJsonOut & out, double value, bool single ) +{ + if ( !TableJsonFinite( value ) ) { return false; } + char text[64]; + int low = single ? 6 : 15; + int high = single ? 9 : 17; + int length = 0; + for ( int digits = low; ; digits++ ) + { + length = snprintf( text, sizeof( text ), "%.*g", digits, value ); + if ( length <= 0 || length >= (int) sizeof( text ) ) { return false; } + if ( digits >= high ) { break; } + // the round-trip check runs BEFORE the decimal point is normalised: + // the token still carries whatever point snprintf just produced + if ( single ) + { + if ( (double) strtof( text, NULL ) == value ) { break; } + } + else + { + if ( strtod( text, NULL ) == value ) { break; } + } + } + char point = TableJsonDecimalPoint(); + if ( point != '.' ) + { + for ( int i = 0; i < length; i++ ) + { + if ( text[i] == point ) { text[i] = '.'; } + } + } + out.raw( text, length ); + return true; +} + +inline bool TableJsonWriteValue( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth ); +// a UNION ARM that names no declaration writes through the field walk one key +// down (docs/SPEC-TABLES.md §2.6, §16.2), which is defined below +inline bool TableJsonWriteField( TableJsonOut & out, const void * base, const TableFieldInfo * f, int32_t depth ); + +// one scalar, at one storage address: a nested object, a union, a +// vocabulary, or a number +inline bool TableJsonWriteScalar( TableJsonOut & out, const void * storage, const TableFieldInfo * f, int32_t depth ) +{ + if ( f->arms != NULL ) + { + // a union is an object with ONE key, the arm's name; None is {} + const TableUnionInfo * arms = f->arms(); + uint64_t tag = TableJsonGetRaw( (const uint8_t *) storage + arms->tag_offset, arms->tag_size ); + if ( tag == 0 ) + { + out.raw( "{}", 2 ); + return true; + } + if ( (int64_t) tag > f->enum_max ) + { + return false; // a tag no arm names, exactly as measure refuses it + } + const char * arm = f->enum_name( tag ); + // and refuse on the NAME, not merely on the bound: §16.2 says a value + // no variant NAMES is refused, so the check is the name. Writing + // whatever came back would emit "???", a spelling the reader counts + // as unknown — a silent round-trip loss in place of a refusal. + if ( !TableJsonNamed( arm ) ) { return false; } + out.put( '{' ); + out.line( depth + 1 ); + TableJsonWriteString( out, arm, (int32_t) strlen( arm ) ); + out.raw( ": ", 2 ); + // THE ARM'S VALUE TAKES THE ARM'S OWN ROW (§16.2): an arm that names + // no declaration carries the FIELD descriptor a field of its type + // would carry, offsets taken inside the union storage (§2.6), so the + // value walks through the field writer one key down. + if ( arms->arms[tag].field != NULL ) + { + if ( !TableJsonWriteField( out, storage, arms->arms[tag].field, depth + 1 ) ) + { + return false; + } + } + else if ( arms->arms[tag].table == NULL ) + { + out.raw( "null", 4 ); // a payload-free arm: the name selects it (§2.6) + } + else if ( !TableJsonWriteValue( out, (const uint8_t *) storage + arms->arms[tag].offset, arms->arms[tag].table, depth + 1 ) ) + { + return false; + } + out.line( depth ); + out.put( '}' ); + return true; + } + if ( f->kind == 13 ) + { + return TableJsonWriteValue( out, storage, f->table, depth ); + } + if ( TableJsonIsEnum( f ) ) + { + uint64_t value = TableJsonGetRaw( storage, f->elem_size ); + // a value no variant names has no text spelling, exactly as it has no + // wire identity: the writer REFUSES rather than writing None over it, + // the rule measure and save already apply (docs/SPEC-TABLES.md §5) + if ( (int64_t) value > f->enum_max ) { return false; } + if ( value != 0 && f->variant_id( value ) == 0 ) { return false; } + const char * name = f->enum_name( value ); + if ( !TableJsonNamed( name ) ) { return false; } + TableJsonWriteString( out, name, (int32_t) strlen( name ) ); + return true; + } + if ( TableJsonIsFlags( f ) ) + { + uint64_t bits = TableJsonGetRaw( storage, f->elem_size ); + if ( bits == 0 ) + { + out.raw( "[]", 2 ); + return true; + } + out.put( '[' ); + bool first = true; + for ( int64_t bit = 0; bit < 64; bit++ ) + { + if ( ( bits & ( uint64_t( 1 ) << bit ) ) == 0 ) { continue; } + if ( bit > f->enum_max ) + { + return false; // a bit no variant names has no text spelling + } + const char * name = f->enum_name( (uint64_t) bit ); + if ( !TableJsonNamed( name ) ) { return false; } + if ( !first ) { out.put( ',' ); } + first = false; + out.line( depth + 1 ); + TableJsonWriteString( out, name, (int32_t) strlen( name ) ); + } + out.line( depth ); + out.put( ']' ); + return true; + } + switch ( f->kind ) + { + case 1: + out.text( TableJsonGetRaw( storage, f->elem_size ) != 0 ? "true" : "false" ); + return true; + case 10: + { + float v = 0.0f; + memcpy( &v, storage, sizeof( v ) ); + return TableJsonWriteFloat( out, (double) v, true ); + } + case 11: + { + double v = 0.0; + memcpy( &v, storage, sizeof( v ) ); + return TableJsonWriteFloat( out, v, false ); + } + case 2: case 3: case 4: case 5: + TableJsonWriteSigned( out, TableJsonGetSigned( storage, f->elem_size ) ); + return true; + default: + if ( TableJsonKindWide( f->kind ) ) + { + TableJsonWriteWide( out, storage, f ); + return true; + } + TableJsonWriteUnsigned( out, TableJsonGetRaw( storage, f->elem_size ) ); + return true; + } +} + +inline bool TableJsonWriteField( TableJsonOut & out, const void * base, const TableFieldInfo * f, int32_t depth ) +{ + const uint8_t * storage = (const uint8_t *) base + f->offset; + if ( TableJsonIsMap( f ) ) + { + return TableJsonWriteMap( out, (const void *) storage, f, depth ); + } + if ( TableJsonIsList( f ) ) + { + return TableJsonWriteList( out, (const void *) storage, f, depth ); + } + if ( f->kind == 17 && !f->is_array ) + { + return TableJsonWritePointer( out, storage, f, depth ); + } + if ( f->kind == 17 ) + { + // an ARRAY OF POINTERS (§2.1): the pointer row per element — the + // pointee's object in place, null, or `&node` for a shared one (§16.7) + int32_t count = TableJsonCount( base, f ); + if ( count == 0 ) { out.raw( "[]", 2 ); return true; } + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + if ( !TableJsonWritePointer( out, storage + (int64_t) i * f->elem_size, f, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( ']' ); + return true; + } + if ( f->kind == 12 ) + { + TableJsonWriteString( out, (const char *) storage, TableJsonCount( base, f ) ); + return true; + } + if ( f->kind == 33 ) + { + TableJsonWriteWString( out, (const char16_t *) (const void *) storage, TableJsonCount( base, f ) ); + return true; + } + if ( TableJsonIsBytes( f ) ) + { + TableJsonWriteBase64( out, storage, TableJsonCount( base, f ) ); + return true; + } + if ( TableJsonIsKeyed( f ) ) + { + // one entry per SLOT, keyed by the variant that owns it, so inserting + // a variant next season moves nothing in the text either. Slot i holds + // the key i + 1: nothing is stored for None, so nothing is written for it. + out.put( '{' ); + bool first = true; + for ( int64_t slot = 0; slot < f->array_bound; slot++ ) + { + if ( !TableJsonKeyedSlotValid( f, slot ) ) { continue; } + if ( !first ) { out.put( ',' ); } + first = false; + out.line( depth + 1 ); + const char * key = f->key_name( TableJsonKeyedSlotKey( slot ) ); + TableJsonWriteString( out, key, (int32_t) strlen( key ) ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteScalar( out, storage + slot * f->elem_size, f, depth + 1 ) ) + { + return false; + } + } + if ( first ) { out.raw( "}", 1 ); return true; } + out.line( depth ); + out.put( '}' ); + return true; + } + if ( f->is_array ) + { + int32_t count = TableJsonCount( base, f ); + if ( count == 0 ) + { + out.raw( "[]", 2 ); + return true; + } + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + if ( !TableJsonWriteScalar( out, storage + (int64_t) i * f->elem_size, f, depth + 1 ) ) + { + return false; + } + } + out.line( depth ); + out.put( ']' ); + return true; + } + return TableJsonWriteScalar( out, storage, f, depth ); +} + +// One instance's fields, in DECLARATION ORDER, defaults included — a text is +// for people and tools, and a text that elides is a text a reader has to know +// the schema to complete. `any` says whether the object is already open on +// entry — a shared node's `&node` opens it before the fields (§16.7) — and +// whether it is open on return. +inline bool TableJsonWriteFields( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth, bool & any ) +{ + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + const TableFieldInfo * f = &info->fields[i]; + if ( f->guard[0] != 0 && !TableJsonGuardHolds( base, info, f->guard ) ) { continue; } + // an ABSENT optional writes no key: presence of the key IS the + // presence (§16.2), so an absent field is an absent key and nothing + // else would read back as absent + if ( f->optional && + TableJsonGetRaw( (const uint8_t *) base + f->present_offset, 1 ) == 0 ) + { + continue; + } + if ( !any ) { out.put( '{' ); } + else { out.put( ',' ); } + any = true; + out.line( depth + 1 ); + TableJsonWriteString( out, f->json, (int32_t) strlen( f->json ) ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteField( out, base, f, depth + 1 ) ) { return false; } + } + return true; +} + +// One instance as one object. The writer carries the reader's depth cap +// (§16.2): a pointer chain nests as deep as it is long (§16.7), and a text the +// writer produced past the cap would be a text the reader refuses. +inline bool TableJsonWriteValue( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { return false; } + bool any = false; + if ( !TableJsonWriteFields( out, base, info, depth, any ) ) { return false; } + if ( !any ) + { + out.raw( "{}", 2 ); + return true; + } + out.line( depth ); + out.put( '}' ); + return true; +} + +// ---- reading ---- + +struct TableJsonIn +{ + const char * text; + int64_t size; + int64_t pos; + TableReport * report; + bool bad; // the text is not JSON: the walk stops and keeps what it placed + void * graph; // the pointered read's builder and label map (§16.7); NULL for a fixed table +}; + +inline void TableJsonSpace( TableJsonIn & in ) +{ + while ( in.pos < in.size ) + { + char c = in.text[in.pos]; + if ( c == ' ' || c == '\t' || c == '\n' || c == '\r' ) { in.pos++; continue; } + // COMMENTS ARE ACCEPTED ON READ AND NEVER WRITTEN (docs/SPEC-TABLES.md + // §16.2): a line comment runs to the end of the line or of the input, + // a block comment to its closing delimiter, which does not nest, and + // an UNCLOSED block comment is malformed on the terms an unclosed + // string is. Both are legal wherever whitespace is; a lone slash is not JSON. + if ( c == '/' && in.pos + 1 < in.size && in.text[in.pos + 1] == '/' ) + { + in.pos += 2; + while ( in.pos < in.size && in.text[in.pos] != '\n' ) { in.pos++; } + continue; + } + if ( c == '/' && in.pos + 1 < in.size && in.text[in.pos + 1] == '*' ) + { + int64_t at = in.pos + 2; + while ( at + 1 < in.size && !( in.text[at] == '*' && in.text[at + 1] == '/' ) ) { at++; } + if ( at + 1 >= in.size ) { in.bad = true; in.pos = in.size; return; } + in.pos = at + 2; + continue; + } + if ( c == '/' ) { in.bad = true; } + return; + } +} + +inline char TableJsonPeek( TableJsonIn & in ) +{ + TableJsonSpace( in ); + return in.pos < in.size ? in.text[in.pos] : 0; +} + +// the shape of the value sitting at the cursor, without consuming it +inline char TableJsonValueShape( TableJsonIn & in ) +{ + char c = TableJsonPeek( in ); + switch ( c ) + { + case '{': return 'o'; + case '[': return 'a'; + case '"': return 's'; + case 't': case 'f': return 'b'; + case 'n': return 'z'; + case 0: return 0; + default: return 'n'; + } +} + +inline bool TableJsonLiteral( TableJsonIn & in, const char * word ) +{ + int64_t length = (int64_t) strlen( word ); + if ( in.pos + length > in.size || memcmp( in.text + in.pos, word, (size_t) length ) != 0 ) + { + in.bad = true; + return false; + } + in.pos += length; + return true; +} + +// one \uXXXX escape body; -1 when the four hex digits are not there +inline int TableJsonHex4( TableJsonIn & in ) +{ + if ( in.pos + 4 > in.size ) { return -1; } + int value = 0; + for ( int i = 0; i < 4; i++ ) + { + char c = in.text[in.pos + i]; + int digit; + if ( c >= '0' && c <= '9' ) { digit = c - '0'; } + else if ( c >= 'a' && c <= 'f' ) { digit = c - 'a' + 10; } + else if ( c >= 'A' && c <= 'F' ) { digit = c - 'A' + 10; } + else { return -1; } + value = ( value << 4 ) | digit; + } + in.pos += 4; + return value; +} + + +// One STRING BODY CHARACTER at the cursor, encoded into unit as UTF-8 and +// its length answered: an escape's code point, or a UTF-8 sequence read +// whole. It is ONE grammar serving both text kinds — the narrow scan places +// these bytes and the wide scan converts them back to code units — so the +// escape table, the lone-surrogate rule and the U+FFFD replacement are stated +// once. false means the text is not JSON and in.bad says so; a returned +// length of 0 means the closing quote was consumed and the string is done. +inline bool TableJsonScanUnit( TableJsonIn & in, char * unit, int32_t * unit_length_out ) +{ + int32_t unit_length = 0; + *unit_length_out = 0; + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos]; + if ( c == '"' ) { in.pos++; return true; } + if ( c == '\\' ) + { + in.pos++; + if ( in.pos >= in.size ) { in.bad = true; return false; } + char escape = in.text[in.pos++]; + switch ( escape ) + { + case '"': unit[0] = '"'; unit_length = 1; break; + case '\\': unit[0] = '\\'; unit_length = 1; break; + case '/': unit[0] = '/'; unit_length = 1; break; + case 'b': unit[0] = '\b'; unit_length = 1; break; + case 'f': unit[0] = '\f'; unit_length = 1; break; + case 'n': unit[0] = '\n'; unit_length = 1; break; + case 'r': unit[0] = '\r'; unit_length = 1; break; + case 't': unit[0] = '\t'; unit_length = 1; break; + case 'u': + { + int high = TableJsonHex4( in ); + if ( high < 0 ) { in.bad = true; return false; } + uint32_t code = (uint32_t) high; + if ( high >= 0xd800 && high <= 0xdbff && in.pos + 2 <= in.size && + in.text[in.pos] == '\\' && in.text[in.pos + 1] == 'u' ) + { + int64_t mark = in.pos; + in.pos += 2; + int low = TableJsonHex4( in ); + if ( low >= 0xdc00 && low <= 0xdfff ) + { + code = 0x10000 + ( ( (uint32_t) high - 0xd800 ) << 10 ) + ( (uint32_t) low - 0xdc00 ); + } + else + { + in.pos = mark; // a lone lead surrogate rides as itself + } + } + // a surrogate half that never found its partner has no + // UTF-8 encoding: encoding it anyway would manufacture + // CESU-8 — invalid UTF-8 — out of input that was valid + // JSON, so it reads as the replacement character + if ( code >= 0xd800 && code <= 0xdfff ) { code = 0xfffd; } + unit_length = TableJsonEncodeUtf8( code, unit ); + break; + } + default: in.bad = true; return false; + } + } + else if ( (unsigned char) c < 0x20 ) + { + in.bad = true; // a raw control character is not a JSON string body + return false; + } + else + { + // a UTF-8 sequence read WHOLE, so the clamp below can only land + // between code points. Only bytes that ACTUALLY look like + // continuations are taken: the wire imposes no encoding (§3), so + // a string may legitimately hold a stray lead byte, and one at + // the end of a text must not swallow the closing quote. + unsigned char lead = (unsigned char) c; + int32_t want = 1; + if ( ( lead & 0xe0 ) == 0xc0 ) { want = 2; } + else if ( ( lead & 0xf0 ) == 0xe0 ) { want = 3; } + else if ( ( lead & 0xf8 ) == 0xf0 ) { want = 4; } + unit[0] = c; + in.pos++; + unit_length = 1; + while ( unit_length < want && in.pos < in.size && + ( (unsigned char) in.text[in.pos] & 0xc0 ) == 0x80 ) + { + unit[unit_length++] = in.text[in.pos++]; + } + // A SEQUENCE THAT IS NOT A CODE POINT READS AS U+FFFD, which is + // §16.3's rule at the point the defect ENTERS rather than at the + // point it leaves: a lone surrogate escape already reads that way + // above, RFC 8259 requires a JSON text to be valid UTF-8, and a + // kind 12 payload is well-formed UTF-8 (§3), so storage the text + // form built has to be storage the wire can carry (§5). + if ( !TableUtf8Valid( (const uint8_t *) unit, unit_length ) ) + { + unit_length = TableJsonEncodeUtf8( 0xfffd, unit ); + } + } + } + *unit_length_out = unit_length; + return true; +} + +// Scan one JSON string into a caller buffer. Bytes are appended ONE CODE +// POINT AT A TIME — an escape's encoding, or a UTF-8 sequence read whole — +// so a string longer than the field is clamped AT A CODE POINT BOUNDARY and +// never cut through a multi-byte character. Clamping is counted, never +// fatal, exactly as it is on the wire (§4). A NULL destination scans past a +// string without keeping it. +// +// A CALLER THAT TAKES clamped_out OWNS THE COUNTER. The value paths leave it +// NULL, and the clamp is a value's clamp, counted here. A MAP KEY takes it, +// because a key never clamps: a key this buffer could not hold whole is not a +// shorter key, and its entry drops instead (§2.8). +inline bool TableJsonScanString( TableJsonIn & in, char * out, int32_t capacity, int32_t * length, + bool * clamped_out = NULL ) +{ + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + int32_t placed = 0; + bool clamped = false; + for ( ;; ) + { + char unit[4]; + int32_t unit_length = 0; + if ( !TableJsonScanUnit( in, unit, &unit_length ) ) { return false; } + if ( unit_length == 0 ) { break; } + if ( out == NULL ) + { + placed += unit_length; // measured and not kept: a byte buffer's read sizes its node this way (§2.5) + } + else if ( !clamped && placed + unit_length <= capacity ) + { + memcpy( out + placed, unit, (size_t) unit_length ); + placed += unit_length; + } + else + { + // A CLAMP IS A PREFIX. Once one code point does not fit, the scan + // stops placing: a later SHORTER code point sliding into the room + // the long one left would store a string the input never spelled, + // and one clamped count cannot tell the two apart. + clamped = true; + } + } + if ( clamped_out != NULL ) { *clamped_out = clamped; } + else if ( clamped ) { in.report->clamped++; } + if ( length != NULL ) { *length = placed; } + return true; +} + +// One UTF-8 sequence back to its CODE POINT, over bytes TableJsonScanUnit +// produced and therefore already well formed. It is the inverse of +// TableJsonEncodeUtf8 and nothing more. +inline uint32_t TableJsonDecodeUtf8( const char * unit, int32_t unit_length ) +{ + const unsigned char lead = (unsigned char) unit[0]; + if ( unit_length == 1 ) { return lead; } + uint32_t code = lead & ( unit_length == 2 ? 0x1fu : ( unit_length == 3 ? 0x0fu : 0x07u ) ); + for ( int32_t i = 1; i < unit_length; i++ ) + { + code = ( code << 6 ) | (uint32_t) ( (unsigned char) unit[i] & 0x3f ); + } + return code; +} + +// Scan one JSON string into a caller buffer of UTF-16 CODE UNITS: the wstring +// row of §16.2, the text TRANSCODED at the boundary. It shares +// TableJsonScanUnit with the narrow scan, so the escape grammar and the +// lone-surrogate rule are one grammar, and appends ONE CODE POINT AT A TIME — +// one unit below U+10000 and a surrogate PAIR above it. A string longer than +// the field is therefore clamped at N code units with a pair never split, and +// a high surrogate left without its low half is dropped with it, which is the +// same sentence the wire's clamp takes (§3). Clamping is counted, never fatal. +inline bool TableJsonScanWString( TableJsonIn & in, char16_t * out, int32_t capacity, int32_t * length ) +{ + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + int32_t placed = 0; + bool clamped = false; + for ( ;; ) + { + char unit[4]; + int32_t unit_length = 0; + if ( !TableJsonScanUnit( in, unit, &unit_length ) ) { return false; } + if ( unit_length == 0 ) { break; } + const uint32_t code = TableJsonDecodeUtf8( unit, unit_length ); + char16_t units[2]; + int32_t units_length = 1; + if ( code < 0x10000 ) + { + units[0] = (char16_t) code; + } + else + { + const uint32_t rest = code - 0x10000; + units[0] = (char16_t) ( 0xd800 + ( rest >> 10 ) ); + units[1] = (char16_t) ( 0xdc00 + ( rest & 0x3ff ) ); + units_length = 2; + } + if ( out == NULL ) + { + placed += units_length; + } + else if ( !clamped && placed + units_length <= capacity ) + { + for ( int32_t i = 0; i < units_length; i++ ) { out[placed + i] = units[i]; } + placed += units_length; + } + else + { + // A CLAMP IS A PREFIX, the narrow scan's own rule: once one code + // point does not fit, the scan stops placing. A pair is placed + // whole or not at all, so no clamp can leave an unpaired + // surrogate in storage. + clamped = true; + } + } + if ( clamped ) { in.report->clamped++; } + if ( length != NULL ) { *length = placed; } + return true; +} + + +// the numeric token at the cursor, copied out whole; false = not a number +// Scan one number, to JSON's OWN grammar (RFC 8259 §6) and not to a run of +// number-ish characters: +// +// number = [ "-" ] int [ frac ] [ exp ] +// int = "0" / ( digit1-9 *digit ) +// frac = "." 1*digit +// exp = ( "e" / "E" ) [ "-" / "+" ] 1*digit +// +// Scanning the production is what makes a typo in an authoring file a +// DIAGNOSTIC rather than a value: "1-2" scans as 1 and leaves "-2" where the +// object expects a comma, so the text is malformed — which is what §16.2 +// already promises. A permissive scan would hand "1-2" to a digit loop and +// report a clamp, and a config pipeline would never hear about it. Leading +// "+", leading zeros, ".5" and "3." are not JSON either. +inline bool TableJsonWalkNumber( TableJsonIn & in, bool * integral ) +{ + TableJsonSpace( in ); + bool whole = true; + if ( in.pos < in.size && in.text[in.pos] == '-' ) { in.pos++; } + // int: a lone zero, or a non-zero digit and any digits after it + if ( in.pos >= in.size ) { return false; } + if ( in.text[in.pos] == '0' ) + { + in.pos++; + } + else if ( in.text[in.pos] >= '1' && in.text[in.pos] <= '9' ) + { + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + } + else + { + return false; + } + // frac + if ( in.pos < in.size && in.text[in.pos] == '.' ) + { + in.pos++; + if ( in.pos >= in.size || in.text[in.pos] < '0' || in.text[in.pos] > '9' ) { return false; } + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + whole = false; + } + // exp + if ( in.pos < in.size && ( in.text[in.pos] == 'e' || in.text[in.pos] == 'E' ) ) + { + in.pos++; + if ( in.pos < in.size && ( in.text[in.pos] == '-' || in.text[in.pos] == '+' ) ) { in.pos++; } + if ( in.pos >= in.size || in.text[in.pos] < '0' || in.text[in.pos] > '9' ) { return false; } + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + whole = false; + } + *integral = whole; + return true; +} + +// the same production, with the token kept for conversion +inline bool TableJsonScanNumber( TableJsonIn & in, char * token, int32_t capacity, int32_t * length, bool * integral ) +{ + TableJsonSpace( in ); + int64_t start = in.pos; + if ( !TableJsonWalkNumber( in, integral ) ) { return false; } + int64_t count = in.pos - start; + if ( count <= 0 || count >= capacity ) { return false; } + memcpy( token, in.text + start, (size_t) count ); + token[count] = 0; + *length = (int32_t) count; + return true; +} + +// the token's exact double, through the runtime's own converter — which +// speaks the LOCALE's decimal point, so the token crosses back over that +// character on its way in +inline double TableJsonTokenDouble( const char * token, int32_t length, bool single ) +{ + char work[kTableJsonMaxNumber]; + memcpy( work, token, (size_t) length ); + work[length] = 0; + char point = TableJsonDecimalPoint(); + if ( point != '.' ) + { + for ( int32_t i = 0; i < length; i++ ) + { + if ( work[i] == '.' ) { work[i] = point; } + } + } + if ( single ) { return (double) strtof( work, NULL ); } + return strtod( work, NULL ); +} + +// ---- ONE CHECKED NUMERIC INTERPRETATION (docs/SPEC-TABLES.md §16.2) ---- +// +// JSON HAS ONE NUMBER TYPE, so every integer target reads a token the same way +// and the VALUE decides rather than the spelling: 2, 2.0 and 1e3 are the +// integers 2, 2 and 1000. What comes out of a token is a SIGN, a MAGNITUDE and +// a STATUS, and nothing on the way is cast through a type that cannot hold what +// it is handed. A uint64 magnitude past INT64_MAX is a magnitude and never a +// negative, and a double is consulted only for a spelling the digit path cannot +// read exactly. +// +// TWO POLICIES SIT ON TOP OF THE ONE VALUE and neither reinterprets the token: +// an ordinary FIELD clamps to its domain and counts, and a MAP KEY rejects the +// whole entry, because a key is an identity and a clamped one is two entries +// merged. That difference is the only difference between them. +// +// THE KEY READS ITS TOKEN EXACTLY AND A FIELD READS IT THROUGH THE DOUBLE, and +// that is the one place the two interpretations part. A key is an IDENTITY, so +// two spellings a 53-bit mantissa cannot tell apart are two keys and the key +// path carries TableJsonInterpretExact below. A field's value is a quantity +// under a clamp, and its interpretation is the one the C, Go and Rust ports +// read the same texts with, so it lives here and reads as they read. +struct TableJsonInteger +{ + uint64_t magnitude; // |value|, exact for every integral token 64 bits hold + bool negative; + bool fractional; // a genuinely fractional VALUE: the wrong shape for an integer + bool saturated; // a magnitude past what 64 bits hold, held at that edge + bool finite; // false: no integer target holds it at all +}; + +// THE FIELD'S INTERPRETATION: the token, parsed digit by digit so no width and +// no locale can move it, and through the runtime's converter only where the +// spelling carries a fraction or an exponent +inline TableJsonInteger TableJsonInterpret( const char * token, int32_t length, bool integral ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = false; + out.fractional = false; + out.saturated = false; + out.finite = true; + if ( integral ) + { + int32_t i = 0; + if ( i < length && token[i] == '-' ) // WalkNumber refuses a leading plus + { + out.negative = true; + i++; + } + for ( ; i < length; i++ ) + { + const uint64_t digit = (uint64_t) ( token[i] - '0' ); + if ( out.magnitude > ( UINT64_MAX - digit ) / 10 ) + { + out.magnitude = UINT64_MAX; + out.saturated = true; + break; + } + out.magnitude = out.magnitude * 10 + digit; + } + if ( out.magnitude == 0 ) { out.negative = false; } // -0 IS zero + return out; + } + const double d = TableJsonTokenDouble( token, length, false ); + if ( !TableJsonFinite( d ) ) { out.finite = false; return out; } + out.negative = d < 0; + const double whole = out.negative ? -d : d; + // THE DOMAIN IS ESTABLISHED BEFORE THE CAST: a magnitude past what sixty-four + // bits hold is answered here, so no value ever reaches a conversion that is + // undefined for it + if ( whole >= 18446744073709551616.0 ) + { + out.magnitude = UINT64_MAX; + out.saturated = true; + return out; + } + const uint64_t truncated = (uint64_t) whole; + if ( (double) truncated != whole ) { out.fractional = true; return out; } + out.magnitude = truncated; + if ( out.magnitude == 0 ) { out.negative = false; } + return out; +} + +// THE DECIMAL BAND a token is answered in without arithmetic: 10^20 is above +// UINT64_MAX whatever the digits are, so a point past it saturates and a token +// spelling 1e999999999 costs nothing to refuse. +const int64_t kTableJsonDecimalBand = 20; + +// THE MAP KEY'S INTERPRETATION, and no other path's: the token's own digits, +// read where they stand, so no 53-bit mantissa decides the identity of a 64-bit +// key. The int and frac runs are one digit string with the point after "point" +// of them, and the exponent moves the point rather than the digits, which is +// the normalization the wide kinds already use over one 64-bit lane. A zero +// fraction is the integer the token spells at every magnitude the kind holds, +// so 9007199254740993.0 is that key rather than the one a double rounds it to. +// No exact reader has an infinity, so finite is true here always. +inline TableJsonInteger TableJsonInterpretExact( const char * token, int32_t length ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = false; + out.fractional = false; + out.saturated = false; + out.finite = true; + int32_t i = 0; + if ( i < length && token[i] == '-' ) { out.negative = true; i++; } // WalkNumber refuses a leading plus + const char * int_digits = token + i; + int32_t int_len = 0; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { int_len++; i++; } + const char * frac_digits = token + i; + int32_t frac_len = 0; + if ( i < length && token[i] == '.' ) + { + i++; + frac_digits = token + i; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { frac_len++; i++; } + } + int64_t exp = 0; + if ( i < length && ( token[i] == 'e' || token[i] == 'E' ) ) + { + i++; + bool exp_negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { exp_negative = token[i] == '-'; i++; } + while ( i < length && token[i] >= '0' && token[i] <= '9' ) + { + if ( exp < 100000 ) { exp = exp * 10 + ( token[i] - '0' ); } + i++; + } + if ( exp_negative ) { exp = -exp; } + } + // leading and trailing zeros stripped, so the last digit kept is significant + int32_t start = 0, end = int_len + frac_len; + int64_t point = (int64_t) int_len + exp; + while ( start < end && ( start < int_len ? int_digits[start] : frac_digits[start - int_len] ) == '0' ) { start++; point--; } + while ( end > start && ( end - 1 < int_len ? int_digits[end - 1] : frac_digits[end - 1 - int_len] ) == '0' ) { end--; } + const int64_t digits = end - start; + if ( digits == 0 ) { out.negative = false; return out; } // the value is zero, and -0 IS zero + if ( point < digits ) { out.fractional = true; return out; } // a significant digit below the point + if ( point > kTableJsonDecimalBand ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + for ( int32_t k = start; k < end; k++ ) + { + const uint64_t digit = (uint64_t) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ); + if ( out.magnitude > ( UINT64_MAX - digit ) / 10 ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + out.magnitude = out.magnitude * 10 + digit; + } + for ( int64_t k = digits; k < point; k++ ) // the point's own zeros, which no digit spells + { + if ( out.magnitude > UINT64_MAX / 10 ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + out.magnitude *= 10; + } + return out; +} + +// a declared range bound as the same value. A bound is inside the field's own +// domain by construction, so nothing here saturates. +inline TableJsonInteger TableJsonIntegerOf( double bound ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = bound < 0; + out.fractional = false; + out.saturated = false; + out.finite = true; + const double whole = out.negative ? -bound : bound; + out.magnitude = whole >= 18446744073709551616.0 ? UINT64_MAX : (uint64_t) whole; + if ( out.magnitude == 0 ) { out.negative = false; } + return out; +} + +// THE TARGET DOMAIN, established before the value reaches storage: the bytes of +// storage, signed or not. Answers what the target holds and whether the domain +// MOVED it. An unsigned magnitude above INT64_MAX rides out as its bit pattern, +// which is the storage's own image of it and not a negative number. +inline int64_t TableJsonIntegerInDomain( const TableJsonInteger & number, bool is_signed, int32_t bytes, bool & moved ) +{ + moved = false; + uint64_t magnitude = number.magnitude; + if ( is_signed ) + { + const uint64_t high = bytes >= 8 ? (uint64_t) INT64_MAX : ( ( uint64_t( 1 ) << ( bytes * 8 - 1 ) ) - 1 ); + if ( number.negative ) + { + const uint64_t low = high + 1; // the floor's magnitude + if ( magnitude > low ) { magnitude = low; moved = true; } + return (int64_t) ( ~magnitude + 1 ); // two's complement, INT64_MIN included + } + if ( magnitude > high ) { magnitude = high; moved = true; } + return (int64_t) magnitude; + } + // A NEGATIVE TOKEN IN AN UNSIGNED FIELD CLAMPS TO ZERO, and -0 is zero, + // which is why the sign is dropped at a zero magnitude above + if ( number.negative ) { moved = true; return 0; } + const uint64_t high = bytes >= 8 ? UINT64_MAX : ( ( uint64_t( 1 ) << ( bytes * 8 ) ) - 1 ); + if ( magnitude > high ) { magnitude = high; moved = true; } + return (int64_t) magnitude; +} + +// A number token into a wide kind's raw storage (docs/SPEC-TABLES.md §16.2). A +// 128-bit integer takes any token whose VALUE is integral; a fixed field any +// token whose value is EXACTLY representable in its Q I.F — a finer fraction +// is the wrong shape for the field, counted as a kind mismatch and never +// rounded, the rule SPEC.md §4.6 gives a fixed default. A magnitude past 128 +// bits saturates and counts as a clamp, as an int64 field saturates at +// INT64_MAX; the declared range clamps after it, on the RAW scale, as it does +// for every bounded scalar. +// +// The token is normalized to its digits with the decimal point after "point" +// of them. An integer part past 40 digits is above 2^128 whatever the digits +// are, and a value below 10^-40 is finer than 2^-127, the finest fraction any +// F can spell — so outside that band the answer is known without the +// arithmetic, and a token spelling 1e999999999 costs nothing to refuse. +inline bool TableJsonReadWide( TableJsonIn & in, const char * token, int32_t length, void * storage, const TableFieldInfo * f ) +{ + bool is_signed = TableJsonKindWideSigned( f->kind ); + int frac = f->frac_bits; + int32_t i = 0; + bool negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { negative = token[i] == '-'; i++; } + const char * int_digits = token + i; + int32_t int_len = 0; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { int_len++; i++; } + const char * frac_digits = token + i; + int32_t frac_len = 0; + if ( i < length && token[i] == '.' ) + { + i++; + frac_digits = token + i; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { frac_len++; i++; } + } + int64_t exp = 0; + if ( i < length && ( token[i] == 'e' || token[i] == 'E' ) ) + { + i++; + bool exp_negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { exp_negative = token[i] == '-'; i++; } + while ( i < length && token[i] >= '0' && token[i] <= '9' ) + { + if ( exp < 100000 ) { exp = exp * 10 + ( token[i] - '0' ); } + i++; + } + if ( exp_negative ) { exp = -exp; } + } + // the digits, with the point after "point" of them; leading and trailing + // zeros stripped. digit( k ) reads the k-th of the int and frac runs. + int32_t start = 0, end = int_len + frac_len; + int64_t point = int_len + exp; + while ( start < end && ( start < int_len ? int_digits[start] : frac_digits[start - int_len] ) == '0' ) { start++; point--; } + while ( end > start && ( end - 1 < int_len ? int_digits[end - 1] : frac_digits[end - 1 - int_len] ) == '0' ) { end--; } + + TableJsonWide raw = { 0, 0 }; + bool saturated = false; + TableJsonWide signed_max = { ~uint64_t( 0 ), ~uint64_t( 0 ) >> 1 }; + TableJsonWide signed_min = { 0, uint64_t( 1 ) << 63 }; + TableJsonWide unsigned_max = { ~uint64_t( 0 ), ~uint64_t( 0 ) }; + if ( start == end ) + { + // zero, and -0 IS zero + } + else if ( point > 40 ) + { + saturated = true; + if ( !negative ) { raw = is_signed ? signed_max : unsigned_max; } + else if ( is_signed ) { raw = signed_min; } + } + else if ( point < -40 ) + { + in.report->kind_mismatch++; // finer than any F can spell + return true; + } + else + { + // the fraction FIRST, so an inexact value is the wrong shape whatever + // its magnitude: its digits, with the zeros a negative point puts in + // front, doubled F times; each doubling's carry is the next bit, and + // the value is exact iff nothing is left after the last one + char fd[kTableJsonMaxNumber + 48]; + int32_t fn = 0; + for ( int64_t z = point; z < 0; z++ ) { fd[fn++] = 0; } + for ( int32_t k = (int32_t) ( point > 0 ? point : 0 ) + start; k < end; k++ ) + { + fd[fn++] = (char) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ); + } + TableJsonWide fraction = { 0, 0 }; + for ( int b = 0; b < frac; b++ ) + { + int carry = 0; + for ( int32_t k = fn - 1; k >= 0; k-- ) + { + int d = fd[k] * 2 + carry; + fd[k] = (char) ( d % 10 ); + carry = d / 10; + } + fraction = TableJsonWideShl( fraction, 1 ); + fraction.lo |= (uint64_t) carry; + } + for ( int32_t k = 0; k < fn; k++ ) + { + if ( fd[k] != 0 ) + { + in.report->kind_mismatch++; + return true; + } + } + // then the whole part, saturating past 128 bits + TableJsonWide whole = { 0, 0 }; + for ( int64_t k = start; k < start + point && !saturated; k++ ) + { + uint32_t digit = k < end ? (uint32_t) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ) : 0; + if ( TableJsonWideMulAdd( &whole, 10, digit ) != 0 ) { saturated = true; } + } + if ( !saturated && frac > 0 && !TableJsonWideZero( TableJsonWideShr( whole, 128 - frac ) ) ) { saturated = true; } + if ( !saturated ) + { + raw = TableJsonWideShl( whole, frac ); + raw.lo |= fraction.lo; + raw.hi |= fraction.hi; + } + if ( is_signed ) + { + if ( !saturated && !negative && TableJsonWideNegative( raw ) ) { saturated = true; } + if ( !saturated && negative && TableJsonWideCompare( raw, signed_min, false ) > 0 ) { saturated = true; } + if ( saturated ) { raw = negative ? signed_min : signed_max; } + else if ( negative ) { raw = TableJsonWideNeg( raw ); } + } + else + { + if ( saturated ) { raw = unsigned_max; } + if ( negative && !TableJsonWideZero( raw ) ) { raw.lo = 0; raw.hi = 0; saturated = true; } + } + } + if ( saturated ) { in.report->clamped++; } + if ( f->wide != NULL ) + { + TableJsonWide lo = { f->wide->lo[0], f->wide->lo[1] }; + TableJsonWide hi = { f->wide->hi[0], f->wide->hi[1] }; + if ( TableJsonWideCompare( raw, lo, is_signed ) < 0 ) { raw = lo; in.report->clamped++; } + else if ( TableJsonWideCompare( raw, hi, is_signed ) > 0 ) { raw = hi; in.report->clamped++; } + } + TableJsonWideStore( storage, f->elem_size, raw ); + return true; +} + +inline bool TableJsonSkipValue( TableJsonIn & in, int32_t depth ); + +inline bool TableJsonSkipContainer( TableJsonIn & in, char close, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; // the opening bracket + bool first = true; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == close ) { in.pos++; return true; } + if ( c == 0 ) { in.bad = true; return false; } + if ( close == '}' ) + { + // the key is kept, because a skipped OBJECT may still be a + // pointer's: an `&node` opening it names a node the storage could + // not hold, and the numbering has to survive the drop (§16.7). + // Anywhere but first, the prefix is the reserved key out of place + // — in a pointered unit; a fixed unit skips the value whole. + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + if ( key[0] == '&' && in.graph != NULL ) + { + if ( !first ) { in.report->malformed = true; in.bad = true; return false; } + if ( !TableJsonSkippedAmpersand( in, key, depth ) ) { return false; } + first = false; + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == close ) { in.pos++; return true; } + in.bad = true; + return false; + } + } + first = false; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == close ) { in.pos++; return true; } + in.bad = true; + return false; + } +} + +inline bool TableJsonSkipValue( TableJsonIn & in, int32_t depth ) +{ + char c = TableJsonPeek( in ); + switch ( c ) + { + case '{': return TableJsonSkipContainer( in, '}', depth ); + case '[': return TableJsonSkipContainer( in, ']', depth ); + case '"': return TableJsonScanString( in, NULL, 0, NULL ); + case 't': return TableJsonLiteral( in, "true" ); + case 'f': return TableJsonLiteral( in, "false" ); + case 'n': return TableJsonLiteral( in, "null" ); + case 0: in.bad = true; return false; + default: + { + // consumed, never converted: skipping needs no buffer, and this + // is the one walk a hostile text drives to the depth cap. It is + // the SAME production the value path scans, so an unknown key + // cannot smuggle past a number a named key would refuse. + bool integral = false; + if ( !TableJsonWalkNumber( in, &integral ) ) { in.bad = true; return false; } + return true; + } + } +} + +inline bool TableJsonReadTable( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth ); +// a UNION ARM that names no declaration reads through the field walk one key +// down (docs/SPEC-TABLES.md §2.6, §16.2), which is defined below +inline bool TableJsonReadField( TableJsonIn & in, void * base, const TableFieldInfo * f, int32_t depth ); + +// place one scalar at one storage address +inline bool TableJsonReadScalar( TableJsonIn & in, void * storage, const TableFieldInfo * f, int32_t depth ) +{ + if ( f->arms != NULL ) + { + // a union is an object with ONE key, the arm's name; {} is None, and + // two keys is a text this walk will not guess at + const TableUnionInfo * arms = f->arms(); + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + TableJsonSetRaw( (uint8_t *) storage + arms->tag_offset, arms->tag_size, 0 ); + if ( TableJsonPeek( in ) == '}' ) { in.pos++; return true; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t tag = 0; + for ( int64_t t = 1; t <= f->enum_max; t++ ) + { + if ( strcmp( f->enum_name( (uint64_t) t ), key ) == 0 ) { tag = t; break; } + } + if ( tag == 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + void * payload = (uint8_t *) storage + arms->arms[tag].offset; + const TableFieldInfo * arm = arms->arms[tag].field; + bool placed = true; + if ( arm != NULL ) + { + // THE ARM'S VALUE TAKES THE ARM'S OWN ROW (§16.2). A value of + // the wrong shape for that row is a KIND MISMATCH: the union + // reads None, the event is counted, and the enclosing object + // continues — the rule a FIELD's value lives under, one key + // down. A pointer arm's null is a null pointer, not a shape + // error, exactly as a pointer field's is (§16.7). + char got = TableJsonValueShape( in ); + if ( arm->kind == 17 && !arm->is_array && got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + memset( payload, 0, (size_t) arms->arms[tag].size ); + } + else if ( got != TableJsonShape( arm ) ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else if ( arm->kind == 17 && !arm->is_array ) + { + // A POINTER ARM'S VALUE IS THE POINTEE IN PLACE, or a + // node reference to one (§16.7) — the read a pointer + // FIELD takes, which is not the scalar walk + memset( payload, 0, (size_t) arms->arms[tag].size ); + if ( !TableJsonReadPointer( in, payload, arm, depth + 1 ) ) { return false; } + } + else + { + // SELECTION ZERO-ESTABLISHES THE ARM (SPEC §5): an arm + // takes no specified default, so zero is the establish + memset( payload, 0, (size_t) arms->arms[tag].size ); + if ( !TableJsonReadField( in, storage, arm, depth + 1 ) ) { return false; } + } + } + else if ( arms->arms[tag].table != NULL ) + { + if ( TableJsonValueShape( in ) != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else + { + arms->arms[tag].table->reset( payload ); + if ( !TableJsonReadTable( in, payload, arms->arms[tag].table, depth + 1 ) ) { return false; } + } + } + else + { + // A PAYLOAD-FREE ARM'S VALUE IS null (§2.6): the arm name + // selects it and there is nothing to place + if ( TableJsonValueShape( in ) != 'z' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else if ( !TableJsonLiteral( in, "null" ) ) + { + return false; + } + } + if ( placed ) + { + TableJsonSetRaw( (uint8_t *) storage + arms->tag_offset, arms->tag_size, (uint64_t) tag ); + } + } + char c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; c = TableJsonPeek( in ); } + if ( c == '}' ) { in.pos++; return true; } + in.bad = true; // a second key: a one-of with two arms is not a value + return false; + } + if ( f->kind == 13 ) + { + f->table->reset( storage ); + return TableJsonReadTable( in, storage, f->table, depth + 1 ); + } + if ( TableJsonIsEnum( f ) ) + { + char name[kTableJsonMaxKey]; + int32_t name_length = 0; + if ( !TableJsonScanString( in, name, kTableJsonMaxKey - 1, &name_length ) ) { return false; } + name[name_length] = 0; + for ( int64_t v = 0; v <= f->enum_max; v++ ) + { + if ( strcmp( f->enum_name( (uint64_t) v ), name ) == 0 ) + { + TableJsonSetRaw( storage, f->elem_size, (uint64_t) v ); + return true; + } + } + // a name this build cannot name reads as None and counts as unknown, + // exactly as an unknown variant id does on the wire (§4) + TableJsonSetRaw( storage, f->elem_size, 0 ); + in.report->unknown++; + return true; + } + if ( TableJsonIsFlags( f ) ) + { + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + in.pos++; + uint64_t bits = 0; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + if ( c != '"' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + char name[kTableJsonMaxKey]; + int32_t name_length = 0; + if ( !TableJsonScanString( in, name, kTableJsonMaxKey - 1, &name_length ) ) { return false; } + name[name_length] = 0; + bool found = false; + for ( int64_t bit = 0; bit <= f->enum_max; bit++ ) + { + if ( strcmp( f->enum_name( (uint64_t) bit ), name ) == 0 ) + { + bits |= uint64_t( 1 ) << bit; + found = true; + break; + } + } + if ( !found ) { in.report->unknown++; } + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + TableJsonSetRaw( storage, f->elem_size, bits ); + return true; + } + if ( f->kind == 1 ) + { + char c = TableJsonPeek( in ); + if ( c == 't' ) { if ( !TableJsonLiteral( in, "true" ) ) { return false; } TableJsonSetRaw( storage, f->elem_size, 1 ); return true; } + if ( !TableJsonLiteral( in, "false" ) ) { return false; } + TableJsonSetRaw( storage, f->elem_size, 0 ); + return true; + } + char token[kTableJsonMaxNumber]; + int32_t length = 0; + bool integral = false; + if ( !TableJsonScanNumber( in, token, kTableJsonMaxNumber, &length, &integral ) ) + { + in.bad = true; + return false; + } + if ( TableJsonKindWide( f->kind ) ) + { + return TableJsonReadWide( in, token, length, storage, f ); + } + if ( f->kind == 10 || f->kind == 11 ) + { + bool single = f->kind == 10; + double value = TableJsonTokenDouble( token, length, single ); + // A magnitude the field's format cannot hold is the WRONG SHAPE for + // the kind, and it never reaches storage: 1e400 is not a float64 and + // 1e300 is not a float32. Storing the infinity the conversion + // produced would leave an instance this walk called CLEAN that + // ToJsonMeasure then refuses forever (a non-finite float has no JSON + // spelling), and §16.1's one invariant is that a text which reads + // clean writes back. + if ( !TableJsonFinite( value ) ) + { + in.report->kind_mismatch++; + return true; + } + if ( f->has_range ) + { + if ( value < f->range_min ) { value = f->range_min; in.report->clamped++; } + else if ( value > f->range_max ) { value = f->range_max; in.report->clamped++; } + } + if ( single ) + { + float narrow = (float) value; + if ( !TableJsonFinite( (double) narrow ) ) + { + in.report->kind_mismatch++; + return true; + } + memcpy( storage, &narrow, sizeof( narrow ) ); + } + else + { + memcpy( storage, &value, sizeof( value ) ); + } + return true; + } + // AN ORDINARY FIELD'S POLICY over the one interpreted value: it CLAMPS to + // its domain and counts. JSON has one number type, so 2.0 IS the integer 2 + // and 1e3 IS 1000. A library that round-trips numbers through a double + // emits them that way, and this walker's own float writer emits 1e+21. Only + // a genuinely fractional value is the wrong shape for the kind. + const bool is_signed = f->kind >= 2 && f->kind <= 5; + const TableJsonInteger number = TableJsonInterpret( token, length, integral ); + if ( !number.finite || number.fractional ) + { + in.report->kind_mismatch++; + return true; + } + if ( number.saturated ) { in.report->clamped++; } // past what sixty-four bits hold + // THE DECLARED RANGE FIRST, THEN THE STORAGE WIDTH, the wire's order (§4), + // so a text and a wire loaded from the same data land the same instance. + // The comparison is on the value's OWN scale, correctly signed past + // INT64_MAX, where the storage's bit pattern is not a number to compare. + TableJsonInteger bounded = number; + if ( f->has_range ) + { + const double scale = number.negative ? -(double) number.magnitude : (double) number.magnitude; + if ( scale < f->range_min ) { bounded = TableJsonIntegerOf( f->range_min ); in.report->clamped++; } + else if ( scale > f->range_max ) { bounded = TableJsonIntegerOf( f->range_max ); in.report->clamped++; } + } + bool moved = false; + const int64_t value = TableJsonIntegerInDomain( bounded, is_signed, (int32_t) f->elem_size, moved ); + if ( moved ) { in.report->clamped++; } + TableJsonSetRaw( storage, f->elem_size, (uint64_t) value ); + return true; +} + +inline bool TableJsonReadField( TableJsonIn & in, void * base, const TableFieldInfo * f, int32_t depth ) +{ + uint8_t * storage = (uint8_t *) base + f->offset; + if ( TableJsonIsMap( f ) ) + { + return TableJsonReadMap( in, (void *) storage, f, depth ); + } + if ( TableJsonIsList( f ) ) + { + return TableJsonReadList( in, (void *) storage, f, depth ); + } + + if ( f->kind == 12 ) + { + int32_t length = 0; + if ( !TableJsonScanString( in, (char *) storage, f->array_bound, &length ) ) { return false; } + storage[length] = 0; + TableJsonSetCount( base, f, length ); + return true; + } + if ( f->kind == 33 ) + { + char16_t * units = (char16_t *) (void *) storage; + int32_t length = 0; + if ( !TableJsonScanWString( in, units, (int32_t) f->array_bound, &length ) ) { return false; } + units[length] = 0; // the terminating zero UNIT at index length (§7.2, SPEC.md §4.12) + TableJsonSetCount( base, f, length ); + return true; + } + if ( TableJsonIsBytes( f ) ) + { + // base64 decodes STRAIGHT INTO the field's storage, six bits at a + // time — no window, no temporary, so a bytes(N) of any declared + // extent reads the same way. A base64 body carries no escapes, so a + // backslash in one is simply not an alphabet character. + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + memset( storage, 0, (size_t) f->array_bound ); + TableJsonSetCount( base, f, 0 ); + const char * alphabet = TableJsonBase64Alphabet(); + int32_t placed = 0; + uint32_t accumulator = 0; + int32_t held = 0; + bool clamped = false; + bool malformed = false; + for ( ;; ) + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos++]; + if ( c == '"' ) { break; } + if ( c == '=' || malformed ) { continue; } + const char * at = c != 0 ? strchr( alphabet, c ) : NULL; + if ( at == NULL ) { malformed = true; continue; } + accumulator = ( accumulator << 6 ) | (uint32_t) ( at - alphabet ); + held += 6; + if ( held >= 8 ) + { + held -= 8; + if ( placed < f->array_bound ) + { + storage[placed++] = (uint8_t) ( ( accumulator >> held ) & 0xff ); + } + else + { + clamped = true; + } + } + } + if ( malformed ) + { + // a body that is not base64 is the wrong shape for the kind: the + // field keeps its default and the event is counted + in.report->kind_mismatch++; + return true; + } + if ( clamped ) { in.report->clamped++; } + TableJsonSetCount( base, f, placed ); + return true; + } + if ( TableJsonIsKeyed( f ) ) + { + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + // every slot back to its declared defaults first, so a key the text + // omits keeps them and a repeated field key cannot leave an earlier + // occurrence's slots standing + for ( int32_t i = 0; i < f->array_bound; i++ ) + { + void * slot = storage + (int64_t) i * f->elem_size; + if ( f->kind == 13 ) { f->table->reset( slot ); } + else { memset( slot, 0, (size_t) f->elem_size ); } + } + char shape = TableJsonElementShape( f ); + // A KEYED OBJECT'S KEYS ARE KEYS: a variant named twice is a duplicate + // key like any other, last-wins and counted (§16.2). Tracked the way + // a table's own field keys are — a bounded, allocation-free bitmask; + // a vocabulary wider than this still reads, its repeats simply stop + // being counted. + uint64_t seen[8] = {}; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t slot = -1; + for ( int64_t v = 0; v < f->array_bound; v++ ) + { + // nothing is stored for None, so "None" finds no slot and is + // an unknown key like any other name this reader cannot place + if ( !TableJsonKeyedSlotValid( f, v ) ) { continue; } + if ( strcmp( f->key_name( TableJsonKeyedSlotKey( v ) ), key ) == 0 ) { slot = v; break; } + } + if ( slot >= 0 && slot < 512 ) + { + uint64_t bit = uint64_t( 1 ) << ( slot & 63 ); + if ( ( seen[slot >> 6] & bit ) != 0 ) { in.report->duplicate++; } + seen[slot >> 6] |= bit; + } + if ( slot < 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( TableJsonValueShape( in ) != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadScalar( in, storage + slot * f->elem_size, f, depth + 1 ) ) + { + return false; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; + } + if ( f->is_array ) + { + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + in.pos++; + // LAST WINS has to be true of a repeated ARRAY key too, and it is + // wire-visible: a fixed array writes every slot, so a second, shorter + // occurrence overlaying a prefix would leave the first occurrence's + // tail standing. The field goes back to its declared defaults before + // this occurrence's elements are placed — the re-establishment a nested + // table and a union arm already get. A table element's defaults are + // its own (the reset hook); every other element kind's storage + // default is zero, which is what the generated array declares. + if ( f->kind == 13 ) + { + for ( int32_t i = 0; i < f->array_bound; i++ ) + { + f->table->reset( storage + (int64_t) i * f->elem_size ); + } + } + else + { + memset( storage, 0, (size_t) f->array_bound * (size_t) f->elem_size ); + } + TableJsonSetCount( base, f, 0 ); + int32_t placed = 0; + char shape = TableJsonElementShape( f ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + if ( placed >= f->array_bound ) + { + // more elements than the reader's bound: the bounded prefix + // is kept and the excess counts, the wire's rule (§4) + in.report->clamped++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( f->kind == 17 ) + { + // an element of an ARRAY OF POINTERS (§2.1): null is a null slot, an + // object is the pointee in place or an `&node` reference (§16.7) + char got = TableJsonValueShape( in ); + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( storage + (int64_t) placed * f->elem_size, f->elem_size, 0 ); + } + else if ( got != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, storage + (int64_t) placed * f->elem_size, f, depth + 1 ) ) { return false; } + placed++; + } + else if ( TableJsonValueShape( in ) != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed++; + } + else + { + if ( !TableJsonReadScalar( in, storage + (int64_t) placed * f->elem_size, f, depth + 1 ) ) { return false; } + placed++; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + // a fixed array's tail keeps the defaults the prefill left there, + // exactly as a short wire count does + TableJsonSetCount( base, f, placed ); + return true; + } + return TableJsonReadScalar( in, storage, f, depth ); +} + +inline bool TableJsonReadTableKeys( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth, const char * first_key ); + +// ONE table object: keys are field keys, unknown ones are skipped and +// counted, a repeated key is last-wins and counted. The instance is already +// at its declared defaults when this is entered, so a key the text never +// mentions keeps the default an absent field takes on the wire (§4). +inline bool TableJsonReadTable( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + return TableJsonReadTableKeys( in, base, info, depth, NULL ); +} + +// The keys of an object whose brace is already consumed. A pointer's object +// opens the same way a table's does, but its FIRST key may be `&node` (§16.7) +// and the adapter that reads it has to scan the key to know — so it hands the +// key it scanned in as `first_key`, with the colon consumed, and this places +// it before scanning the rest. +inline bool TableJsonReadTableKeys( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth, const char * first_key ) +{ + // duplicate tracking, bounded and allocation-free: a table with more + // fields than this still reads, its repeats simply stop being counted + uint64_t seen[8] = {}; + for ( ;; ) + { + char key[kTableJsonMaxKey]; + char c = 0; + if ( first_key != NULL ) + { + memcpy( key, first_key, strlen( first_key ) + 1 ); // scanned into a buffer this size by the caller + first_key = NULL; + } + else + { + c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; return true; } + if ( c == 0 ) { in.bad = true; return false; } + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + } + int32_t index = -1; + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + if ( strcmp( info->fields[i].json, key ) == 0 ) { index = i; break; } + } + if ( key[0] == '&' ) + { + // THE AMPERSAND PREFIX IS RESERVED TO THE FORM (docs/SPEC-TABLES.md + // §16.7). No declaration may take a key beginning with it, so this + // is never a field this build lacks — it is the sharing construct + // somewhere it cannot stand: `&node` is the FIRST key of a pointer's + // object and nothing else, and the adapter that reads a pointer + // has consumed it before these keys are read. MALFORMED, refused + // and counted; never counted as unknown, never skipped. + in.report->malformed = true; + in.bad = true; + return false; + } + if ( index < 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + const TableFieldInfo * f = &info->fields[index]; + if ( index < 512 ) + { + uint64_t bit = uint64_t( 1 ) << ( index & 63 ); + if ( ( seen[index >> 6] & bit ) != 0 ) { in.report->duplicate++; } + seen[index >> 6] |= bit; + } + // PRESENCE OF THE KEY IS THE PRESENCE (§16.2): reaching this line + // is the key being present, so an optional is set present + // whatever its value — with one exception the page names: a JSON + // null, which reads as ABSENT rather than as a value. + char got = TableJsonValueShape( in ); + if ( f->kind == 17 && !f->is_array ) + { + // a pointer: null is a null pointer, an object is the pointee + // in place or an `&node` reference to one (§16.7), a string is + // a BYTE BUFFER's bytes (§2.5), and anything else is the wrong + // shape for the kind + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) base + f->offset, f->elem_size, 0 ); + } + else if ( got != TableJsonShape( f ) ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, (uint8_t *) base + f->offset, f, depth ) ) + { + return false; + } + } + else if ( f->optional && got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + // absent, and back at its defaults: a repeated key whose last + // occurrence is null must not leave an earlier value standing + if ( f->table != NULL ) { f->table->reset( (uint8_t *) base + f->offset ); } + else { memset( (uint8_t *) base + f->offset, 0, (size_t) f->elem_size ); } + TableJsonSetRaw( (uint8_t *) base + f->present_offset, 1, 0 ); + } + else + { + if ( got != TableJsonShape( f ) ) + { + // the wrong JSON type for the kind: skipped, never coerced + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadField( in, base, f, depth ) ) + { + return false; + } + if ( f->optional ) + { + TableJsonSetRaw( (uint8_t *) base + f->present_offset, 1, 1 ); + } + } + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; return true; } + in.bad = true; + return false; + } +} + +// ---- the two entry points the per-table wrappers name ---- + +inline bool TableJsonRead( void * value, const TableTypeInfo * info, const char * text, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableJsonIn in; + in.text = text; + in.size = bytes; + in.pos = 0; + in.report = report != NULL ? report : &ignored; + in.bad = false; + in.graph = NULL; + info->reset( value ); + if ( text == NULL || bytes < 0 ) + { + in.report->malformed = true; + return false; + } + bool ok = TableJsonReadTable( in, value, info, 0 ); + if ( ok ) + { + TableJsonSpace( in ); + if ( in.pos != in.size ) { in.bad = true; } // trailing rubbish is not one text + } + if ( in.bad || !ok ) + { + in.report->malformed = true; + return false; + } + return true; +} + +inline int64_t TableJsonWrite( const void * value, const TableTypeInfo * info, char * buffer, int64_t capacity ) +{ + TableJsonOut out; + out.buffer = buffer; + out.capacity = capacity; + out.offset = 0; + out.overflow = false; + out.graph = NULL; + if ( !TableJsonWriteValue( out, value, info, 0 ) ) { return -1; } + // THE CANONICAL TEXT ENDS WITH EXACTLY ONE NEWLINE (docs/SPEC-TABLES.md + // §16.1). Every writer emits it — this walk, the C# walk and + // "schema unpack" — and every reader accepts a text with or without one, + // because the trailing whitespace a read already skips is what makes the + // two the same text. It is a byte of the FORM rather than a file + // convention: a text that is written to a file, pasted into a diff and + // handed back through a pipe has to be one text in all three places, and a + // buffer whose last byte is a closing brace is the one shape that is not. + out.put( '\n' ); + if ( out.overflow ) { return -1; } + return out.offset; +} + +// ---- json walk: end ---- + +// ---- json graph walk: begin ---- +// +// THE VARIABLE CLASS's half of the text form (docs/SPEC-TABLES.md §16.7). The +// walk above places every kind but one; this defines the three adapters it +// calls for that one, and the two entry points a pointered table's wrappers +// name. The text is the fixed class's — a pointee is an object in place — and a +// node named more than once carries `&node`: defined once, with its fields, +// and referenced after by `{ "&node": N }` alone. + +// ---- the identity map ---- +// +// ONE map shape serves both directions. Writing keys it by a node's ADDRESS and +// counts the slots that name the node, so the second pass knows at a node's +// first occurrence whether it will be named again; reading keys it by the +// text's own label and answers the node it defined. Open addressing, a +// multiply-shift hash and quadrupling growth — TablePackMap's shape (§6.2), on +// the same terms: proportional to nodes, never to bytes, on the authoring +// side, and released before the call returns. + +struct TableJsonGraphEntry +{ + uint64_t key; // a node's address (write) or a label (read); 0 is an empty slot + int64_t count; // write: how many slots name this node + int64_t label; // write: the `&node` label assigned at its first write, 0 until then + uint8_t open; // the descent is still open: a reference here is a cycle (write), a self-reference (read) + uint32_t node; // read: the node's arena offset; 0 for a definition the reader dropped + const TableTypeInfo * type; // read: the node's table; NULL for a dropped one +}; + +struct TableJsonGraphMap +{ + TableJsonGraphEntry * entries; + int64_t capacity; // a power of two, or zero while empty + int64_t count; + TableAllocator allocator; // the caller's pair (§6.5): the builder's on read, the one handed to ToJson on write +}; + +inline void TableJsonGraphMapInit( TableJsonGraphMap & map, TableAllocator allocator ) +{ + map.entries = NULL; + map.capacity = 0; + map.count = 0; + map.allocator = allocator; +} + +inline void TableJsonGraphMapShutdown( TableJsonGraphMap & map ) +{ + map.allocator.free( map.allocator.context, map.entries ); + TableJsonGraphMapInit( map, map.allocator ); +} + +inline int64_t TableJsonGraphMapSlot( const TableJsonGraphMap & map, uint64_t key ) +{ + uint64_t hash = key * 0x9E3779B97F4A7C15ull; + hash ^= hash >> 29; + int64_t mask = map.capacity - 1; + int64_t at = (int64_t) ( hash & (uint64_t) mask ); + while ( map.entries[at].key != 0 && map.entries[at].key != key ) + { + at = ( at + 1 ) & mask; + } + return at; +} + +inline TableJsonGraphEntry * TableJsonGraphMapFind( TableJsonGraphMap & map, uint64_t key ) +{ + if ( map.capacity == 0 ) { return NULL; } + TableJsonGraphEntry * entry = &map.entries[ TableJsonGraphMapSlot( map, key ) ]; + return entry->key == key ? entry : NULL; +} + +inline bool TableJsonGraphMapGrow( TableJsonGraphMap & map ) +{ + TableJsonGraphMap grown; + grown.allocator = map.allocator; + grown.capacity = map.capacity != 0 ? map.capacity * 4 : 64; + grown.count = 0; + grown.entries = (TableJsonGraphEntry *) map.allocator.alloc( map.allocator.context, grown.capacity * (int64_t) sizeof( TableJsonGraphEntry ) ); // zeroed, by the pair's contract + if ( grown.entries == NULL ) { return false; } + for ( int64_t i = 0; i < map.capacity; i++ ) + { + if ( map.entries[i].key == 0 ) { continue; } + grown.entries[ TableJsonGraphMapSlot( grown, map.entries[i].key ) ] = map.entries[i]; + grown.count++; + } + map.allocator.free( map.allocator.context, map.entries ); + map = grown; + return true; +} + +// the entry for a key, made if it was not there; `taken` says which. NULL is the +// allocator refusing, and the walk refuses with it. +inline TableJsonGraphEntry * TableJsonGraphMapReach( TableJsonGraphMap & map, uint64_t key, bool & taken ) +{ + if ( ( map.count + 1 ) * 4 >= map.capacity * 3 ) // keep the load factor under three quarters + { + if ( !TableJsonGraphMapGrow( map ) ) { return NULL; } + } + TableJsonGraphEntry * entry = &map.entries[ TableJsonGraphMapSlot( map, key ) ]; + taken = entry->key != key; + if ( taken ) + { + entry->key = key; + map.count++; + } + return entry; +} + +// ---- reading: into a builder ---- + +struct TableJsonGraphIn +{ + TableWorker * worker; // where every node comes from + TableJsonGraphMap labels; // a label -> the node it defined +}; + +// `&node`'s value, the LABEL: a positive integer spelled as one — digits, no sign, no +// fraction, no exponent, no leading zero (§16.7). Anything else is malformed. +inline bool TableJsonScanLabel( TableJsonIn & in, uint64_t & label ) +{ + TableJsonSpace( in ); + if ( in.pos >= in.size || in.text[in.pos] < '1' || in.text[in.pos] > '9' ) + { + in.report->malformed = true; + in.bad = true; + return false; + } + uint64_t value = 0; + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) + { + uint64_t digit = (uint64_t) ( in.text[in.pos] - '0' ); + if ( value > ( UINT64_MAX - digit ) / 10 ) + { + in.report->malformed = true; + in.bad = true; + return false; + } + value = value * 10 + digit; + in.pos++; + } + label = value; + return true; +} + +// A BYTE BUFFER's text (docs/SPEC-TABLES.md §2.5, §16.2): a string. For a +// *string the string's bytes become the blob; for a *bytes the string is base64 +// and its decoded bytes do. The blob is allocated at EXACTLY the decoded +// length — the string is scanned once without keeping it to learn the length, +// and once into the node — so a blob of any size reads with no window and no +// bound to clamp against. A *bytes body that is not base64 is the wrong shape +// for the kind: the reference stays null and the event is counted. +inline bool TableJsonReadBlob( TableJsonIn & in, void * slot, const TableFieldInfo * f ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + TableRef * ref = (TableRef *) slot; + ref->value = 0; + if ( strcmp( f->type_name, "string" ) == 0 ) + { + const int64_t mark = in.pos; + int32_t length = 0; + if ( !TableJsonScanString( in, NULL, 0, &length ) ) { return false; } + in.pos = mark; + char * data = TableStringEmplace( *graph->worker, *ref, NULL, (int64_t) length ); + if ( data == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + int32_t placed = 0; + return TableJsonScanString( in, data, length, &placed ); + } + // base64: the alphabet characters decide the length, six bits apiece + const char * alphabet = TableJsonBase64Alphabet(); + const int64_t mark = in.pos + 1; + int64_t symbols = 0; + bool malformed = false; + in.pos++; + for ( ;; ) + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos++]; + if ( c == '"' ) { break; } + if ( c == '=' || malformed ) { continue; } + if ( c == 0 || strchr( alphabet, c ) == NULL ) { malformed = true; continue; } + symbols++; + } + if ( malformed ) + { + in.report->kind_mismatch++; + return true; + } + const int64_t length = ( symbols * 6 ) / 8; + uint8_t * data = TableBytesEmplace( *graph->worker, *ref, length ); + if ( data == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + int64_t placed = 0; + uint32_t accumulator = 0; + int32_t held = 0; + for ( int64_t at = mark; ; at++ ) + { + char c = in.text[at]; + if ( c == '"' ) { break; } + const char * symbol = c != '=' ? strchr( alphabet, c ) : NULL; + if ( symbol == NULL ) { continue; } + accumulator = ( accumulator << 6 ) | (uint32_t) ( symbol - alphabet ); + held += 6; + if ( held >= 8 ) + { + held -= 8; + if ( placed < length ) { data[placed++] = (uint8_t) ( ( accumulator >> held ) & 0xff ); } + } + } + return true; +} + +// A pointer's object. Its FIRST key decides what it is: `&node` naming a label not +// yet defined, with fields after it, is a DEFINITION; `&node` naming one already +// defined, alone, is a REFERENCE; any other key is a node named once, its +// object in place. The node comes from the +// builder's arena, and the slot holds its arena offset (§6.3). A pointer whose +// target is a BYTE BUFFER — no table — takes a string instead (§2.5). +inline bool TableJsonReadPointer( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( f->table == NULL ) { return TableJsonReadBlob( in, slot, f ); } + // the pointee nests one level down, exactly as a by-value table does, and + // takes the same cap: a chain nests as deep as it is long (§16.7) + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + char c = TableJsonPeek( in ); + if ( c == '}' ) + { + // an empty object: a node at its defaults, named once + in.pos++; + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + return true; + } + if ( c == 0 ) { in.bad = true; return false; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + if ( strcmp( key, "&node" ) != 0 ) + { + // a node named once: the pointee's object in place, and this key is + // its first field — unless it is the reserved prefix under a spelling + // this form does not have, which ReadTableKeys refuses + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } + return TableJsonReadTableKeys( in, node, f->table, depth + 1, key ); + } + uint64_t label = 0; + if ( !TableJsonScanLabel( in, label ) ) { return false; } + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph->labels, label, taken ); + if ( entry == NULL ) { in.report->malformed = true; in.bad = true; return false; } + // ONE SPELLING, and what follows the label says which half it is: fields + // after a label the text has not defined DEFINE it, and a label alone that + // the text has defined REFERS to it. The other two are malformed — a label + // alone that the text never defined, which would otherwise read as a default + // node under a silent report, and a field after a label already defined, + // which would be a second definition. That is what keeps a typo loud. + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; c = TableJsonPeek( in ); } + bool bare = c == '}'; + if ( bare == taken ) { in.report->malformed = true; in.bad = true; return false; } + if ( bare ) + { + // A REFERENCE. A label is defined when its object CLOSES, so a + // reference met inside its own definition — at any depth of by-value + // nesting — names a node whose descent is still open: the cycle the + // wire refuses (§3.1), refused here where it is written. A definition + // the reader dropped names no node, so the slot stays null with + // nothing more counted — the drop was counted where it happened. A + // node of another table than the slot declares is a kind mismatch, as + // on the wire. + in.pos++; + if ( entry->open != 0 ) { in.report->malformed = true; in.bad = true; return false; } + TableRef ref; + if ( entry->type == NULL ) + { + memcpy( slot, &ref, sizeof( ref ) ); + return true; + } + if ( entry->type != f->table ) + { + memcpy( slot, &ref, sizeof( ref ) ); + in.report->kind_mismatch++; + return true; + } + ref.value = (int64_t) entry->node; + memcpy( slot, &ref, sizeof( ref ) ); + return true; + } + // A DEFINITION: the node is allocated, the label is its, and the keys after + // `&node` are its fields. The entry is OPEN until the object closes, so a + // reference to the label from inside the node's own fields is refused as + // the cycle it is; the node and its table are filled in at the close. + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } + entry->open = 1; + if ( !TableJsonReadTableKeys( in, node, f->table, depth + 1, NULL ) ) { return false; } + entry = TableJsonGraphMapFind( graph->labels, label ); // the map may have grown under the descent + if ( entry == NULL ) { in.report->malformed = true; in.bad = true; return false; } + TableRef ref; + memcpy( &ref, slot, sizeof( ref ) ); + entry->node = (uint32_t) ref.value; + entry->type = f->table; + entry->open = 0; + return true; +} + +// An `&`-prefixed key opening an object the walk is SKIPPING — a value past an +// array's bound, an unknown key's value, a value of the wrong shape. A +// definition in there still takes its label, so the numbering survives whatever +// the storage could not hold (§16.7): the label is registered with no node, and a +// reference to it reads null. Any other prefixed key is the reserved prefix +// out of place. +inline bool TableJsonSkippedAmpersand( TableJsonIn & in, const char * key, int32_t ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL || strcmp( key, "&node" ) != 0 ) { in.report->malformed = true; in.bad = true; return false; } + uint64_t label = 0; + if ( !TableJsonScanLabel( in, label ) ) { return false; } + bool taken = false; + if ( TableJsonGraphMapReach( graph->labels, label, taken ) == NULL ) { in.report->malformed = true; in.bad = true; return false; } + return true; // a fresh entry is node 0, type NULL: a definition with no node +} + +// ---- writing: from a region's const root ---- + +struct TableJsonGraphOut +{ + TableJsonGraphMap nodes; // a node's address -> how many slots name it, and its `&node` once assigned + bool counting; // PASS ONE: count the references, refuse a cycle, emit nothing + int64_t next_label; +}; + +// The node a slot names: null as `null`, a node named once as its object in +// place, and a node named more than once under the construct. Which of the +// last two it is was learned in pass one; pass two spells it. +inline bool TableJsonWritePointer( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphOut * graph = (TableJsonGraphOut *) out.graph; + if ( graph == NULL ) { return false; } + const void * node = f->resolve( slot ); + if ( node == NULL ) + { + out.raw( "null", 4 ); + return true; + } + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph->nodes, (uint64_t) (uintptr_t) node, taken ); + if ( entry == NULL ) { return false; } + if ( f->table == NULL ) + { + // A BYTE BUFFER (§2.5, §16.7): its text is a string, which has no + // first key to carry `&node`, so a blob named from more than one + // slot has no spelling this form can carry and the graph is refused — + // as a shared node with nothing to write is. A blob named once is its + // bytes in place: base64 for a *bytes, the string itself for a *string. + if ( graph->counting ) { entry->count++; return true; } + if ( entry->count > 1 ) { return false; } + const TableBlob * blob = (const TableBlob *) node; + if ( blob->length > (uint32_t) 0x7fffffff ) { return false; } + if ( strcmp( f->type_name, "string" ) == 0 ) { TableJsonWriteString( out, (const char *) ( blob + 1 ), (int32_t) blob->length ); } + else { TableJsonWriteBase64( out, (const uint8_t *) ( blob + 1 ), (int32_t) blob->length ); } + return true; + } + if ( graph->counting ) + { + // PASS ONE: one visit per node, every slot that names it counted, and + // a reference to a node whose descent is still open is a cycle — + // refused here as the wire refuses it (§3.1) + entry->count++; + if ( !taken ) { return entry->open == 0; } + entry->open = 1; + if ( !TableJsonWriteValue( out, node, f->table, depth ) ) { return false; } + entry = TableJsonGraphMapFind( graph->nodes, (uint64_t) (uintptr_t) node ); // the map may have grown under the descent + if ( entry == NULL ) { return false; } + entry->open = 0; + return true; + } + // PASS TWO: a node named once is its object in place; a node named more + // than once is DEFINED at its first occurrence — `&node` first, then its + // fields — and REFERENCED by `&node` alone after that, spelled the same way at + // every site. Labels run from 1 in first-write order and are the text's own, + // so a stray number in a hand-edited text is most often one never defined. + if ( entry->count <= 1 ) + { + return TableJsonWriteValue( out, node, f->table, depth ); + } + if ( depth > kTableJsonMaxDepth ) { return false; } + if ( entry->label != 0 ) + { + out.put( '{' ); + out.line( depth + 1 ); + out.raw( "\"&node\": ", 9 ); + TableJsonWriteUnsigned( out, (uint64_t) entry->label ); + out.line( depth ); + out.put( '}' ); + return true; + } + entry->label = ++graph->next_label; + out.put( '{' ); + out.line( depth + 1 ); + out.raw( "\"&node\": ", 9 ); + TableJsonWriteUnsigned( out, (uint64_t) entry->label ); + bool any = true; + int64_t before = out.offset; + if ( !TableJsonWriteFields( out, node, f->table, depth, any ) ) { return false; } + // a definition carries at least one field, because a label alone is a + // reference: a shared node with nothing to write has no definition this + // form can spell, and the writer refuses it as it refuses any value it + // cannot spell (§16.3) + if ( out.offset == before ) { return false; } + out.line( depth ); + out.put( '}' ); + return true; +} + +// ---- the two entry points a pointered table's wrappers name ---- + +// The text into the builder's root. Every node the text names is allocated in +// the builder's arena through the field's own Emplace; the label map is the +// walk's, released before this returns. The root itself takes no label — nothing +// may name it (§16.7) — so an `&node` at the root is refused like any other key +// of the prefix. +inline bool TableJsonReadGraph( TableWorker & worker, void * root, const TableTypeInfo * info, const char * text, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + if ( worker.arena == NULL ) { if ( report != NULL ) { report->malformed = true; } return false; } + TableJsonGraphIn graph; + graph.worker = &worker; + TableJsonGraphMapInit( graph.labels, worker.arena->allocator ); + TableJsonIn in; + in.text = text; + in.size = bytes; + in.pos = 0; + in.report = report != NULL ? report : &ignored; + in.bad = false; + in.graph = &graph; + info->reset( root ); + if ( text == NULL || bytes < 0 ) + { + in.report->malformed = true; + return false; + } + bool ok = TableJsonReadTable( in, root, info, 0 ); + if ( ok ) + { + TableJsonSpace( in ); + if ( in.pos != in.size ) { in.bad = true; } // trailing rubbish is not one text + } + TableJsonGraphMapShutdown( graph.labels ); + if ( in.bad || !ok ) + { + in.report->malformed = true; + return false; + } + return true; +} + +// The text of a region's const root: measured when the buffer is NULL, written +// when it is not, over one code path. Two passes over one walk — the first +// counts how many slots name each node and refuses a cycle, the second writes +// — so a node's first occurrence knows whether it will be named again. The +// ROOT's entry is open for the whole first pass, so a reference back at it is +// the cycle it is (§3.1), and it takes no label. +inline int64_t TableJsonWriteGraph( const void * root, const TableTypeInfo * info, char * buffer, int64_t capacity, TableAllocator allocator ) +{ + if ( root == NULL ) { return -1; } + TableJsonGraphOut graph; + TableJsonGraphMapInit( graph.nodes, allocator ); + graph.counting = true; + graph.next_label = 0; + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph.nodes, (uint64_t) (uintptr_t) root, taken ); + if ( entry == NULL ) { TableJsonGraphMapShutdown( graph.nodes ); return -1; } + entry->open = 1; + TableJsonOut count; + count.buffer = NULL; + count.capacity = 0; + count.offset = 0; + count.overflow = false; + count.graph = &graph; + bool ok = TableJsonWriteValue( count, root, info, 0 ); + graph.counting = false; + TableJsonOut out; + out.buffer = buffer; + out.capacity = capacity; + out.offset = 0; + out.overflow = false; + out.graph = &graph; + if ( ok ) { ok = TableJsonWriteValue( out, root, info, 0 ); } + TableJsonGraphMapShutdown( graph.nodes ); + if ( !ok ) { return -1; } + out.put( '\n' ); // the canonical text ends with exactly one newline (§16.1) + if ( out.overflow ) { return -1; } + return out.offset; +} + +// ---- json graph walk: end ---- + +// ---- the out-of-line array's slot (docs/SPEC-TABLES.md §8.1) ---- + +inline int32_t TableJsonExtentCount( const void * slot ) +{ + int32_t count = 0; + memcpy( &count, (const uint8_t *) slot + 8, sizeof( count ) ); + return count < 0 ? 0 : count; +} + +inline const uint8_t * TableJsonExtentElements( const void * slot ) +{ + int64_t delta = 0; + memcpy( &delta, slot, sizeof( delta ) ); + return delta != 0 ? (const uint8_t *) slot + delta : NULL; +} + +// ---- json map walk: begin ---- + +inline bool TableJsonIsMap( const TableFieldInfo * f ) +{ + return f->is_array && f->array_bound == 0 && strncmp( f->type_name, "map[", 4 ) == 0; +} + +// the entry's two rows: fields[0] IS the key and fields[1] IS the value, which +// is what makes a user's own table of pairs the same bytes (§2.8) +inline const TableFieldInfo * TableJsonMapKeyField( const TableFieldInfo * f ) { return &f->table->fields[0]; } +inline const TableFieldInfo * TableJsonMapValueField( const TableFieldInfo * f ) { return &f->table->fields[1]; } + +inline bool TableJsonMapKeyIsString( const TableFieldInfo * key ) { return key->kind == 12; } +inline bool TableJsonMapKeySigned( const TableFieldInfo * key ) { return key->kind >= 2 && key->kind <= 5; } + +// AN INTEGER KEY IS THE INTEGER'S DECIMAL SPELLING, QUOTED, because a JSON +// object's keys are strings. Written digit by digit so no locale can move it. +inline void TableJsonWriteMapIntegerKey( TableJsonOut & out, const void * storage, const TableFieldInfo * key ) +{ + uint64_t magnitude = 0; + bool negative = false; + if ( TableJsonMapKeySigned( key ) ) + { + int64_t value = 0; + switch ( key->kind ) + { + case 2: value = (int64_t) *(const int8_t *) storage; break; + case 3: value = (int64_t) *(const int16_t *) storage; break; + case 4: value = (int64_t) *(const int32_t *) storage; break; + default: value = *(const int64_t *) storage; break; + } + negative = value < 0; + magnitude = negative ? ( ~(uint64_t) value ) + 1 : (uint64_t) value; + } + else + { + switch ( key->kind ) + { + case 6: magnitude = (uint64_t) *(const uint8_t *) storage; break; + case 7: magnitude = (uint64_t) *(const uint16_t *) storage; break; + case 8: magnitude = (uint64_t) *(const uint32_t *) storage; break; + default: magnitude = *(const uint64_t *) storage; break; + } + } + char digits[24]; + int32_t at = (int32_t) sizeof( digits ); + do { digits[--at] = (char) ( '0' + ( magnitude % 10 ) ); magnitude /= 10; } while ( magnitude != 0 ); + if ( negative ) { digits[--at] = '-'; } + TableJsonWriteString( out, digits + at, (int32_t) sizeof( digits ) - at ); +} + +inline void TableJsonWriteMapKey( TableJsonOut & out, const void * entry, const TableFieldInfo * key ) +{ + const uint8_t * storage = (const uint8_t *) entry + key->offset; + if ( TableJsonMapKeyIsString( key ) ) + { + // A STRING KEY IS THE STRING (§2.8): every JSON key of a map object is + // a KEY OF THE MAP and none is a field key, so the `&` prefix §16.7 + // reserves for field keys is ordinary data here. + TableJsonWriteString( out, (const char *) storage, *(const int32_t *) ( (const uint8_t *) entry + key->count_offset ) ); + return; + } + TableJsonWriteMapIntegerKey( out, (const void *) storage, key ); +} + +// ToJson WRITES ENTRIES IN ASCENDING KEY ORDER, so unpack then pack is +// byte-stable and a diff of two texts is a diff of two maps (§2.8, §17.2). +// A region holds them in that order already, so this is the array in place. +inline bool TableJsonWriteMap( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + const int32_t count = TableJsonExtentCount( slot ); + if ( count == 0 ) { out.raw( "{}", 2 ); return true; } + const TableFieldInfo * key = TableJsonMapKeyField( f ); + const TableFieldInfo * value = TableJsonMapValueField( f ); + const uint8_t * entries = TableJsonExtentElements( slot ); + out.put( '{' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + const void * entry = (const void *) ( entries + (int64_t) i * f->elem_size ); + TableJsonWriteMapKey( out, entry, key ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteField( out, entry, value, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( '}' ); + return true; +} + +// AN INTEGER KEY IS READ BY §16.2's INTEGER RULE AND BY NOTHING ELSE, so +// "2.0" and "1e3" are the integers 2 and 1000 and "-0" is zero. The token is +// walked as a JSON number over its own bytes; a token that rule calls +// malformed makes the KEY malformed, and a genuinely fractional value, or one +// outside the key kind's range, is kind_mismatch for that entry. +// +// THE KEY IS THE SPELLING AND NOTHING AROUND IT. The number walk steps over +// leading whitespace and comments, which is right BETWEEN tokens and wrong +// INSIDE one: a key is an identity, and a padded spelling that resolved to the +// same integer would be a second name for one entry. So the walk must begin at +// the token's first byte, and a token with anything before the number is not a +// JSON number at all, which is malformed on the terms "1-2" is. +inline bool TableJsonMapKeyValue( const char * token, int32_t length, const TableFieldInfo * key, + int64_t & value, bool & fits ) +{ + fits = false; + TableReport scratch; + TableJsonIn probe = { token, (int64_t) length, 0, &scratch, false, NULL }; + bool integral = false; + TableJsonSpace( probe ); + if ( probe.pos != 0 ) { return false; } // whitespace is never part of a key + if ( !TableJsonWalkNumber( probe, &integral ) ) { return false; } + if ( probe.pos != (int64_t) length ) { return false; } // trailing bytes: not a number + // A MAP KEY'S POLICY over the one interpreted value: it REJECTS THE WHOLE + // ENTRY. A key is an identity, so a clamped one is two entries merged, and + // a value the key kind does not hold is kind_mismatch for that entry, + // dropped and counted, never clamped. + const TableJsonInteger number = TableJsonInterpretExact( token, length ); + if ( number.fractional || number.saturated ) { return true; } + bool moved = false; + value = TableJsonIntegerInDomain( number, TableJsonMapKeySigned( key ), (int32_t) key->elem_size, moved ); + fits = !moved; + return true; +} + +// FromJson READS KEYS IN WHATEVER ORDER THE TEXT GIVES THEM. A repeated key is +// last-wins and counted duplicate, the object rule (§16.2) applied inside the +// map. An empty object is an empty map, and null is kind_mismatch. +inline bool TableJsonReadMap( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; + const TableFieldInfo * key = TableJsonMapKeyField( f ); + const TableFieldInfo * value = TableJsonMapValueField( f ); + const char shape = TableJsonShape( value ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + char token[kTableJsonMaxKey]; + int32_t token_length = 0; + bool key_over = false; // longer than THIS buffer: never truncated into a key + if ( !TableJsonScanString( in, token, kTableJsonMaxKey - 1, &token_length, &key_over ) ) { return false; } + token[token_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t key_value = 0; + bool place = true; + if ( !TableJsonMapKeyIsString( key ) ) + { + // AN INTEGER KEY PAST THIS SCAN'S BUFFER DROPS AS kind_mismatch, + // here and in the tool's walker. The bytes kept are a PREFIX, and a + // prefix is a different token, so the entry drops rather than a + // truncation being read as a value. The length alone does not + // settle it: a token that long can still spell a number an integer + // kind holds, "1" padded by an exponent of zeroes for one, and this + // read declines to find out. + bool fits = false; + if ( key_over ) { in.report->kind_mismatch++; place = false; } + else if ( !TableJsonMapKeyValue( token, token_length, key, key_value, fits ) ) + { + // A MALFORMED KEY STOPS THE READ where §16.1's rule stops it, + // with the instance holding what was placed before the stop. + in.report->malformed = true; + in.bad = true; + return false; + } + else if ( !fits ) { in.report->kind_mismatch++; place = false; } + } + else if ( key_over || token_length > key->array_bound ) + { + // A KEY LONGER THAN N DROPS ITS ENTRY AND COUNTS clamped, the + // wire's rule, because a clamped key is a merged entry (§2.8). The + // BOUND IS THE WALKER'S, tested here against the key field's own + // descriptor, so placement is left with one failure to report. A + // key past this scan's own buffer is the SAME event, because a + // truncated key is the merged entry the rule exists to prevent. + in.report->clamped++; + place = false; + } + const int32_t before = TableJsonExtentCount( (const void *) slot ); + void * entry = place ? f->place( *graph->worker, slot, token, token_length, key_value ) : NULL; + if ( place && entry == NULL ) + { + // AN ALLOCATION FAILURE IS NOT AN OVERSIZED KEY (§2.8, §16.1). The + // key was checked above, so the arena is what refused, and the read + // stops where the list, blob and pointer paths stop on one rather + // than handing back an instance short of entries the text spelled + // and calling itself clean. + in.report->malformed = true; + in.bad = true; + return false; + } + else if ( entry != NULL && TableJsonExtentCount( (const void *) slot ) == before ) + { + in.report->duplicate++; // last-wins, the object rule inside the map + } + const char got = TableJsonValueShape( in ); + if ( entry == NULL ) + { + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( value->kind == 17 && !value->is_array ) + { + // A POINTER VALUE IS SHARED EXACTLY AS A POINTER FIELD IS (§2.8): + // null is a null slot, an object is the pointee in place or an + // &node reference to one (§16.7), anything else is the wrong shape — + // the same three the field-key loop gives a pointer field, because + // an entry's value IS a field line. + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) entry + value->offset, value->elem_size, 0 ); + } + else if ( got != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, (uint8_t *) entry + value->offset, value, depth + 1 ) ) + { + return false; + } + } + else if ( got != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadField( in, entry, value, depth + 1 ) ) + { + return false; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; +} + +// ---- json map walk: end ---- + +// ---- json list walk: begin ---- + +// an unbounded array is the out-of-line array that is not a map (§8.1) +inline bool TableJsonIsList( const TableFieldInfo * f ) +{ + return f->is_array && f->array_bound == 0 && !TableJsonIsMap( f ); +} + +// ToJson WRITES THE ELEMENTS IN INDEX ORDER, which is the only order there is, +// so unpack then pack is byte-stable without a rule of its own (§2.9, §17.2). +// A region holds the array in place, so this steps it at the descriptor's pitch. +inline bool TableJsonWriteList( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + const int32_t count = TableJsonExtentCount( slot ); + if ( count == 0 ) { out.raw( "[]", 2 ); return true; } + const uint8_t * elements = TableJsonExtentElements( slot ); + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + const uint8_t * element = elements + (int64_t) i * f->elem_size; + if ( f->kind == 17 ) + { + // a []*T's elements take the pointer row (§16.7): the pointee's + // object in place, null, or `&node` for a shared one + if ( !TableJsonWritePointer( out, element, f, depth + 1 ) ) { return false; } + } + else if ( !TableJsonWriteScalar( out, element, f, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( ']' ); + return true; +} + +// FromJson READS EVERY ELEMENT THE TEXT CARRIES, appending each through the +// descriptor's place resolver: `[]` is an empty list, and null is +// kind_mismatch, the array row's own rule (§16.2). LAST WINS holds for a +// repeated key: the list goes back to EMPTY before this occurrence's elements +// land, the builder's storage being reclaimed at reset (§2.9). +inline bool TableJsonReadList( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; + TableJsonSetRaw( (uint8_t *) slot, 8, 0 ); + TableJsonSetRaw( (uint8_t *) slot + 8, 4, 0 ); + const char shape = TableJsonElementShape( f ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + void * element = f->place( *graph->worker, slot, NULL, 0, 0 ); + if ( element == NULL ) + { + // NOT ADDED: the arena could not carve another segment, or the + // count met the int32 cap. The text cannot be placed whole, and + // the read stops where §16.1's rule stops it. + in.report->malformed = true; + in.bad = true; + return false; + } + if ( f->kind == 17 ) + { + // an element of a []*T (§2.9): null is a null slot, an object is the + // pointee in place or an `&node` reference (§16.7) + char got = TableJsonValueShape( in ); + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) element, f->elem_size, 0 ); + } + else if ( got != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, element, f, depth + 1 ) ) { return false; } + } + else if ( TableJsonValueShape( in ) != shape ) + { + // the wrong shape for the element kind: the slot keeps its + // defaults and the event counts, the array row's rule (§16.2) + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadScalar( in, element, f, depth + 1 ) ) { return false; } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; +} + +// ---- json list walk: end ---- + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_JSON + +namespace mapdemo { + +bool TrailsFromJson( TrailsBuilder & builder, const char * text, int64_t bytes, TableReport * report ) +{ + Trails * root = builder.GetRoot(); + if ( root == NULL ) { if ( report != NULL ) { report->malformed = true; } return false; } // locked, or the root allocation failed + return TableJsonReadGraph( builder.main, root, TrailsTableType(), text, bytes, report ); +} + +int64_t TrailsToJsonMeasure( const Trails * root, TableAllocator allocator ) +{ + return TableJsonWriteGraph( root, TrailsTableType(), NULL, 0, allocator ); +} + +int64_t TrailsToJson( const Trails * root, char * buffer, int64_t capacity, TableAllocator allocator ) +{ + return TableJsonWriteGraph( root, TrailsTableType(), buffer, capacity, allocator ); +} + +} // namespace mapdemo diff --git a/testdata/golden/tables/maps/TrailsTable.h b/testdata/golden/tables/maps/TrailsTable.h new file mode 100644 index 000000000..6ad9b9906 --- /dev/null +++ b/testdata/golden/tables/maps/TrailsTable.h @@ -0,0 +1,9623 @@ +// Code generated by the schema compiler from Trails.schema. DO NOT EDIT. +// SPDX-License-Identifier: NONE — this generated output is yours, under terms of +// your choice. See the LICENSE exception in the schema compiler; the compiler is +// AGPL-3.0, its output is not. +// package mapdemo — protocol id 0x1ac124decde5b2aa (packets only: tables version by field id, not by protocol id) +// The TABLE wire (evolution-tolerant, docs/SPEC-TABLES.md): no serialize +// dependency — includable from any TU. + +#pragma once + +#include +#include // the prefill's scalar-array fills +#include // offsetof, for the reflection descriptors + +// ---- the hooks (docs/USAGE.md, "the C++ table runtime's hooks") ---- +// +// schema_assert — the runtime's own assert, and the refusal a debugger reads. +// NDEBUG removes it, exactly as it removes assert. A caller who already routes +// serialize's asserts writes `#define schema_assert serialize_assert` before +// including this header and both halves land in one handler. +#ifndef schema_assert +#include +#define schema_assert assert +#endif // #ifndef schema_assert + +// schema_fatal — what stands after the assert on a path that cannot continue. +// NDEBUG does not remove it. Supply it and is never included. +#ifndef schema_fatal +#include // abort +#define schema_fatal abort +#endif // #ifndef schema_fatal + +// schema_allocate / schema_release — what "no allocator handed in" means for +// this program. schema_allocate hands back ZEROED bytes and NULL on failure: +// an arena segment is copied whole, padding included, so anything left +// uninitialized here would reach a packed region. Supply both and +// is never included; hand a TableAllocator to a builder to route one +// structure's allocations somewhere else again. +#ifndef schema_allocate +#include // calloc, free +#define schema_allocate( bytes ) calloc( (size_t) 1, (size_t) ( bytes ) ) +#define schema_release( pointer ) free( pointer ) +#endif // #ifndef schema_allocate +#include // a node's lifetime starts in arena storage (placement new) +#include // one atomic per slab: the arena is lock-free by ownership + +#include "Trails.h" +#include "FleetTable.h" + +#ifndef MAPDEMO_SCHEMA_TABLE_PRIMITIVES +#define MAPDEMO_SCHEMA_TABLE_PRIMITIVES + +// THE CODEC DOES NOT DEPEND ON THE COMPILER'S INLINING BUDGET. A table of a +// realistic field count emits one large body per type, and the cursor a body +// writes through lives in the caller's `TableWriter`: across a call boundary +// that cursor round-trips through memory, and a `uint8_t *` store may alias the +// writer itself, so every put reloads it. When a budget runs out mid-body the +// codec silently degrades to that shape. Forcing the primitives and the +// fixed-class bodies inline is what keeps the cursor in registers and lets +// adjacent constant framing bytes merge into one store. +#if defined( _MSC_VER ) +#define MAPDEMO_TABLE_INLINE __forceinline +#elif defined( __GNUC__ ) || defined( __clang__ ) +#define MAPDEMO_TABLE_INLINE inline __attribute__(( always_inline )) +#else +#define MAPDEMO_TABLE_INLINE inline +#endif + +namespace mapdemo { + +// WHY A READ WAS REFUSED, by name (docs/SPEC-TABLES.md §3.3, §11). A REFUSAL +// is not one of §4's events: nothing is decoded, no counter moves and no +// damage is reported, so five zero counters and a false flag are what a clean +// read prints too and only the verdict tells them apart. The reason says which +// refusal it was. +// +// This is the MESSAGE PATH's vocabulary and not the cooked form's (§7.4): a +// caller meeting one of these has been refused a MESSAGE on a connection, +// which is a different recovery with a different owner than a file a header +// match turned down. +enum TableMessageReason +{ + newer_form, // a FORM BYTE this reader does not carry (§3) + no_vocabulary, // no table for this connection: the message arrived before the announcement, or after a refused one + second_announcement, // a second announcement on a connection: it sets nothing, amends nothing, and the connection closes + vocabulary_too_large, // an announcement above the receiver's declared bound, refused before an entry is touched + message_form_as_file, // a form 2 wire where a FILE was expected: its table is somewhere else + batch_too_large // a batch of more than 256 bodies on the write side, or of more than the caller has room for on the read side: nothing is written or decoded, and the count says what the wire carries +}; + +// The table-wire read report — the permissive contract's ledger. Silence +// (all zero) means the data matched this reader's schema exactly. +struct TableReport +{ + int32_t unknown = 0; // unknown field ids skipped (newer data) + int32_t kind_mismatch = 0; // known id, changed type — skipped, never misdecoded + // a kind that GREW since the writer (docs/SPEC-TABLES.md §4): an integer + // kind read into a wider one of the same signedness, or f32 into f64, + // decoded EXACTLY. One count per field or per map. It is the one counter + // that names no loss: the bytes were not the shape this reader declares, + // and the number survived. + int32_t widened = 0; + int32_t clamped = 0; // out-of-range values clamped to declared bounds + // a key the TEXT form saw twice: last wins, and the repeat is counted + // (docs/SPEC-TABLES.md §16.2). The wire never raises it — a body carrying an + // id twice is legal input whose last occurrence wins, silently (§3). + int32_t duplicate = 0; + bool malformed = false; // framing damage; decode stopped, partial result kept + // THE REFUSAL VERDICT, which is not one of §4's events and moves no counter + // (docs/SPEC-TABLES.md §3): a FORM BYTE this reader does not carry. Five + // zero counters and a false flag are what a clean read prints too, so the + // verdict is what tells the two apart. + bool refused = false; + // WHICH refusal, and it is read only when refused is set: a read that + // was not refused has no reason, and this member is the one the caller + // must not look at then (docs/SPEC-TABLES.md §3.3). + TableMessageReason reason = newer_form; + // RETAIN-UNKNOWN's pair (docs/SPEC-TABLES.md §6.6), on the same struct for + // the reason duplicate is: a caller has one report type and not two. Both + // are ZERO in every read that did not opt in, and retention moves no + // counter above. A retained field still counts unknown, because unknown + // says what a READER could not name and that stays true. + int32_t retained = 0; // unknown fields whose bytes were kept + int32_t retain_lost = 0; // every unknown this load or save could not keep +}; + + +// WHY A FILE WAS REFUSED, by name (docs/SPEC-TABLES.md §6.5, §7, §19.2): the +// one vocabulary a cook's Open, a block's BlockOpen and a load measure's -1 +// share, because a caller asking "why can I not have this file" is +// asking one question whichever call refused it. The FIRST failing clause names the +// reason, in the order §7 enumerates, so one file answers one value in every +// language. A refusal moves no counter, and a match writes nothing: the +// out-parameter is touched on the refusal path only. +// +// It is not the MESSAGE FORM's vocabulary (TableMessageReason, §3.3): a caller +// meeting one of these has been refused a FILE, by a header match or by a +// measure. +enum TableRefuseReason +{ + ok, // no clause failed: the only value beside a non-null root (§7) + not_a_cook, // the magic is neither this build's constant nor its byte reversal, or the byte-order word contradicts the magic + foreign_order, // the magic byte-reversed: a cook of the other byte order (§7.1) + wrong_build_version, // the build_version word is not this build's (§20) + reserved_not_zero, // a reserved header word is not zero (§7.1) + bad_alignment, // the alignment word is not a power of two, is below eight, is above sixty-four, or is not a multiple of the root's own alignof + truncated, // the part lengths against the caller's length, or a data part too short to hold the root + unaligned_base, // the pointer the caller passed is not aligned for the region: the caller's defect, not the file's + bad_layout, // BlockOpen (§19.2): a pitch, a count, an offset or an extent that disagrees with this build's or leaves the block + unknown_form, // at a MEASURE (§3, §6.5): a form byte this build does not carry, refused before any read + count_over_length, // an array or map count whose elements cannot fit the field's own L (§2.8, §2.9) + count_over_extent_cap, // a count above the int32 extent cap (§2.2), which no region can hold whatever its size + blob_over_size_cap, // a blob whose length is past the derived-size cap (§3.1, §11) + data_cycle // a data cycle reached from a builder: the AUTHORING side's -1 (§3.1, §7.6) +}; +// ---- reflection (tables only, docs/SPEC-TABLES.md) ---- +// +// Static field descriptors for every type in the table closure: name, wire +// id/kind, storage offset, bounds, ranges, enum names and branch guards — +// enough to walk, print, diff, edit or bind any table value at runtime with +// no RTTI and no schema files. TableType() returns X's descriptor. + +struct TableTypeInfo; + +// One arm of a union field: where its payload sits inside the union's storage +// and what its payload looks like. The arm's NAME and its table-wire id come +// from the field's enum_name/variant_id functions at the same tag, so nothing +// is spelled twice (docs/SPEC-TABLES.md §8). +struct TableFieldInfo; + +struct TableUnionArmInfo +{ + uint32_t offset; // offsetof the arm's payload within the union storage + const TableTypeInfo * table; // the arm payload's descriptor, or NULL + // AN ARM IS A FIELD LINE (docs/SPEC-TABLES.md §2.6): an arm that names no + // declared type or table carries the FIELD descriptor a field of that + // type would carry instead — offsets taken within the union storage — so + // a generic walk meets an arm's kind, width, bounds and companions where + // it meets a field's. Exactly one of the two is non-NULL on a set arm. + const TableFieldInfo * field; + uint32_t size; // the arm's whole storage, which selection zero-establishes +}; + +// A union field's shape: the tag, and the arms indexed by it. Arms run +// [0, enum_max]; index 0 is the EMPTY arm and carries no payload. +struct TableUnionInfo +{ + uint32_t tag_offset; // offsetof the tag within the union storage + uint32_t tag_size; // sizeof the tag + const TableUnionArmInfo * arms; +}; + +// The exact raw range of a wide-kind field (docs/SPEC-TABLES.md §8.2): two 128-bit +// values as 64-bit lanes, low lane first, two's complement for the signed kinds. +struct TableWideRange +{ + uint64_t lo[2]; + uint64_t hi[2]; +}; + +// THE SHARED EMPTY DOC (docs/SPEC-TABLES.md §8.1): a declaration with no /// +// block carries a doc column pointing at this one object, so absence costs a +// unit no string data and a printer concatenates doc columns with no null +// test. One definition for the whole unit: every absent doc compares equal by +// address. +inline const char TableDocNone[1] = ""; + +// the arena's allocation front, defined with the variable-length runtime +// below; a descriptor names it only through a pointer parameter. +struct TableWorker; + +struct TableFieldInfo +{ + const char * name; // schema field name, e.g. "health" + const char * json; // the TEXT form's key: the json = "key" attribute, else name (§16.3) + const char * type_name; // schema type name, e.g. "float32", "Grade" + uint64_t id; // table-wire field id: fnv1a64 of the name, of the was alias after a rename (§5) + uint8_t kind; // table-wire kind; for arrays/strings/bytes, the ELEMENT kind + bool is_array; // fixed or counted array (bytes included) + bool is_pointer; // a *T pointer field: storage is an 8-byte TableRef; the target is a table + // THE TWO THE TEXT FORM NEEDS (docs/SPEC-TABLES.md §16.7), and they + // are here for the same reason is_pointer is: the walk is ONE walk + // over descriptors and cannot spell a target's own At or + // Emplace. `resolve` reads a slot in a REGION and answers the + // node it names, or NULL; `emplace` allocates one in a BUILDER's + // arena and points the slot at it. NULL on every field that is not + // a pointer, and emitted only in a unit that declares one. + const void * (*resolve)( const void * slot ); + void * (*emplace)( TableWorker & worker, void * slot ); + bool counted; // a _count/_length int32 companion exists (counted arrays, strings, bytes) + bool optional; // a ?T field: a _present bool companion decides whether it rides + int32_t array_bound; // array capacity / string max length; 0 for plain scalars + uint32_t offset; // offsetof the storage member + uint32_t elem_size; // sizeof the member (element size for arrays) + uint32_t count_offset; // offsetof the _count/_length companion, or 0xffffffff + uint32_t present_offset; // offsetof the _present companion, or 0xffffffff + const TableTypeInfo * table; // nested table's descriptor, or NULL + bool has_range; // a declared [min, max] (int or float) + double range_min; // NOTE: int64 ranges beyond 2^53 lose precision here + double range_max; + // the WIDE kinds (18-29, docs/SPEC-TABLES.md §3, §8.2): frac_bits is a fixed + // field's F — its storage holds units × 2^F — and wide is the declared + // range on that RAW scale, exact, as two 128-bit two's-complement values + // in 64-bit lanes (low lane first). NULL where the declaration bounds + // nothing (a bare uint128) and for every other kind; frac_bits is 0 for + // every kind that is not fixed-point. range_min/range_max still carry + // the declared bounds as doubles — whole units for a fixed field — for + // a walker that only shows them. + uint8_t frac_bits; + const TableWideRange * wide; + int64_t enum_max; // enums: highest valid value (None = 0 always valid); + // unions: the arm count (tag range [0, enum_max]); + // flags: the highest declared BIT INDEX; else -1 + // the vocabulary's names, indexed the same way enum_max bounds: an enum's + // value -> name, a union's tag -> arm name, a FLAGS field's bit index -> + // variant name. NULL for every other kind. + const char * (*enum_name)( uint64_t value ); + // the TABLE-WIRE id of one variant (docs/SPEC-TABLES.md §5): for an enum, the + // hash of the variant's name; for a union, the hash of the arm's name. + // 0 is the reserved id — an enum's None, a union's empty. NULL for every + // other kind — a FLAGS field's variants have no per-variant wire id (§4), + // so a NULL here beside a non-NULL enum_name is what says "flags". + // Walk [0, enum_max] to enumerate a vocabulary and its ids. + uint64_t (*variant_id)( uint64_t value ); + // an ENUM-KEYED array (docs/SPEC-TABLES.md §2.4): the array has one slot per + // variant of key_type_name, indexed by the variant's value, and its slots + // ride under variant ids rather than positions. key_name and key_id are + // the key's vocabulary — walk [0, array_bound) to print slots by name. + // NULL on every other field. + const char * key_type_name; + const char * (*key_name)( uint64_t value ); + uint64_t (*key_id)( uint64_t value ); + // union fields: the tag and its arms, behind a function so the whole + // descriptor stays CONSTANT-INITIALISED (a captureless lambda converts to + // a function pointer at compile time; the arms themselves are a static + // inside it). NULL for every other kind. + const TableUnionInfo * (*arms)(); + // an OUT-OF-LINE array (docs/SPEC-TABLES.md §8.1): place one element and + // hand it back at its defaults. A MAP places BY KEY, a string key comes + // in as the bytes and the length, an integer key as the value, and NULL + // is NOT INSERTED: a key past the bound, or an arena that could not carve + // another segment. A LIST ignores the key and APPENDS, NULL at the arena + // or the int32 cap. NULL on every field that is neither. + void * ( * place )( TableWorker & worker, void * slot, const char * key, int32_t key_length, int64_t key_value ); + const char * guard; // branch guard, e.g. "at_rest" or "!at_rest"; "" if unguarded + // what a PERSON wrote about the field (docs/SPEC-TABLES.md §8.1): the /// + // block above it, verbatim (SPEC §4.1). It is TableDocNone when there is + // none, never NULL. Its tags (SPEC §4.2) follow in declared order, and an + // untagged field is 0 beside NULL. Static, constant-initialized, + // allocating nothing. + const char * doc; + int32_t num_tags; + const char * const * tags; +}; + +struct TableTypeInfo +{ + const char * name; // schema type name + uint32_t size; // sizeof the storage struct + int32_t num_fields; + const TableFieldInfo * fields; + // put one instance back at its declared defaults, in place. A generic + // walker that fills a value has to be able to establish the defaults an + // absent field takes, and it holds no type to spell — this is the one + // thing the descriptors could not express without it. Placement-new + // value-init, exactly what the wire's read path does, and no temporary. + void (*reset)( void * storage ); + // the DERIVED mode (docs/SPEC-TABLES.md): false = fixed-size, a plain + // relocatable struct; true = variable-length, built through a Builder + // and read through a region root. Nobody declares it; the compiler + // works it out. + bool variable; + // the declaration's own doc and tags, on the same terms as a field's + // (docs/SPEC-TABLES.md §8.1) + const char * doc; + int32_t num_tags; + const char * const * tags; +}; + +struct TableWriter +{ + uint8_t * buffer; + int64_t capacity; + int64_t offset = 0; + bool overflow = false; + + // the parameters do not repeat the member names: a parameter that hides a + // member is a warning the estate's compilers disagree about (gcc's + // -Wshadow and cl's C4458 refuse it, clang's -Wshadow does not), and this + // is a header a consumer compiles under its OWN flags + TableWriter( uint8_t * to_buffer, int64_t to_capacity ) : buffer( to_buffer ), capacity( to_capacity ) {} + + MAPDEMO_TABLE_INLINE void raw( const void * data, int64_t bytes ) + { + if ( offset + bytes > capacity ) { overflow = true; return; } + memcpy( buffer + offset, data, (size_t) bytes ); + offset += bytes; + } + MAPDEMO_TABLE_INLINE void put8( uint8_t v ) { raw( &v, 1 ); } + MAPDEMO_TABLE_INLINE void put16( uint16_t v ) { uint8_t b[2] = { uint8_t( v ), uint8_t( v >> 8 ) }; raw( b, 2 ); } + MAPDEMO_TABLE_INLINE void put32( uint32_t v ) { uint8_t b[4] = { uint8_t( v ), uint8_t( v >> 8 ), uint8_t( v >> 16 ), uint8_t( v >> 24 ) }; raw( b, 4 ); } + MAPDEMO_TABLE_INLINE void put64( uint64_t v ) { put32( uint32_t( v ) ); put32( uint32_t( v >> 32 ) ); } + // a 128-bit value as two lanes, the low half first (docs/SPEC-TABLES.md §3) + MAPDEMO_TABLE_INLINE void put128( uint64_t lo, uint64_t hi ) { put64( lo ); put64( hi ); } + // EVERY LENGTH, COUNT, INDEX AND ID REFERENCE IS ONE CANONICAL UNSIGNED + // LEB128 (docs/SPEC-TABLES.md §3): seven value bits a byte, the lowest + // group first, the high bit set on every byte but the last. One value has + // one spelling, so two conforming writers agree byte for byte. + MAPDEMO_TABLE_INLINE void putleb( uint64_t v ) + { + while ( v >= 0x80 ) { put8( uint8_t( v ) | 0x80 ); v >>= 7; } + put8( uint8_t( v ) ); + } +}; + +// TableLebBytes is one value's spelling length, which a MEASURE needs before +// the bytes exist — the length of a body has to be known before it is written, +// because a length whose own width moves cannot be patched in place. +inline int64_t TableLebBytes( uint64_t v ) +{ + int64_t n = 1; + while ( v >= 0x80 ) { v >>= 7; n++; } + return n; +} + +// THE ID TABLE, WRITER SIDE (docs/SPEC-TABLES.md §3). It holds every id the +// body used, once each, in FIRST-USE order over the whole wire, and the body +// names them by position: reference k is the kth entry, counted from 1, and +// reference 0 names NO ID. +// +// Its capacity is a COMPILE-TIME fact of the unit — the distinct names its +// table closure can spell — so a save allocates nothing: the table is a local +// of Measure and of Save. The bucket chain makes ref constant time and makes +// truncate constant time too, which is what an ELIDED field needs: a field +// that turns out not to ride costs nothing in the id table either, so the walk +// interns its id, builds the payload that decides, and undoes the entry when +// nothing rides. +struct TableIds +{ + static const int32_t kCapacity = 76; + static const int32_t kBuckets = 256; + + uint64_t ids[ kCapacity ]; + int32_t chain[ kCapacity ]; + int32_t head[ kBuckets ]; + int32_t count; + bool overflow; + + TableIds() : count( 0 ), overflow( false ) + { + for ( int32_t i = 0; i < kBuckets; i++ ) { head[i] = -1; } + } + + static MAPDEMO_TABLE_INLINE uint32_t bucket_of( uint64_t id ) + { + return uint32_t( ( id * 0x9E3779B97F4A7C15ull ) >> 56 ) & uint32_t( kBuckets - 1 ); + } + + // the reference an id takes: the file's own first-use entry, appended on + // first use. The MESSAGE form names no id at all: its references are + // compile-time slots of the announced vocabulary (docs/SPEC-TABLES.md §3.3). + MAPDEMO_TABLE_INLINE uint64_t ref( uint64_t id ) + { + const uint32_t b = bucket_of( id ); + for ( int32_t i = head[b]; i >= 0; i = chain[i] ) + { + if ( ids[i] == id ) { return uint64_t( i ) + 1; } + } + if ( count >= kCapacity ) { overflow = true; return 1; } + ids[count] = id; chain[count] = head[b]; head[b] = count; count++; + return uint64_t( count ); + } + + // undo every entry appended since mark. An entry removed is the most + // recent one in its bucket, so it sits at that bucket's head. + void truncate( int32_t mark ) + { + while ( count > mark ) + { + count--; + head[ bucket_of( ids[count] ) ] = chain[count]; + } + } +}; + +// TableIdsBytes is the trailer's own size: the entries, each a fixed +// little-endian u64, and the ENTRY COUNT, the one fixed-width number on the +// wire (docs/SPEC-TABLES.md §3). +inline int64_t TableIdsBytes( const TableIds & ids ) { return int64_t( ids.count ) * 8 + 8; } + +// TableIdsWrite puts the trailer where the walk ended: a writer never patches, +// because first-use order is known only when the walk ends. +inline void TableIdsWrite( TableWriter & w, const TableIds & ids ) +{ + for ( int32_t i = 0; i < ids.count; i++ ) { w.put64( ids.ids[i] ); } + w.put64( uint64_t( ids.count ) ); +} + +// THE ID TABLE, READER SIDE (docs/SPEC-TABLES.md §3). A reader locates it from +// the END of the wire and resolves it ONCE, at open: the entries are eight +// bytes each and a body names them by position, so every field dispatches +// through an index rather than through a search over hashes. +struct TableIdTable +{ + const uint8_t * entries = NULL; + int64_t count = 0; + + // the id a reference names. ref is 1-based and bounds-checked by the + // caller: a reference ABOVE the entry count is framing damage on the body + // that carries it, and 0 names no id at all. + uint64_t at( uint64_t ref ) const + { + const uint8_t * e = entries + ( ref - 1 ) * 8; + uint64_t lo = uint64_t( e[0] ) | uint64_t( e[1] ) << 8 | uint64_t( e[2] ) << 16 | uint64_t( e[3] ) << 24; + uint64_t hi = uint64_t( e[4] ) | uint64_t( e[5] ) << 8 | uint64_t( e[6] ) << 16 | uint64_t( e[7] ) << 24; + return lo | ( hi << 32 ); + } +}; + +struct TableReader +{ + const uint8_t * buffer; + int64_t size; + int64_t offset = 0; + TableReport * report; + const TableIdTable * ids = NULL; + // ONLY THE ROOT BODY CARRIES THE NODE TABLE (docs/SPEC-TABLES.md §3.1), so + // a body has to know which it is: the reserved id inside a NESTED body is + // malformed, because a second numbering cannot exist. Every reader made + // for a payload is nested; the two the wire surfaces make for a root say so. + bool nested = true; + + TableReader( const uint8_t * from_buffer, int64_t from_size, TableReport * to_report ) + : buffer( from_buffer ), size( from_size ), report( to_report ) {} + + TableReader( const uint8_t * from_buffer, int64_t from_size, TableReport * to_report, const TableIdTable * to_ids ) + : buffer( from_buffer ), size( from_size ), report( to_report ), ids( to_ids ) {} + + MAPDEMO_TABLE_INLINE bool has( int64_t bytes ) const { return offset + bytes <= size; } + // A LENGTH IS A 64-BIT NUMBER AND A BUFFER IS NOT (docs/SPEC-TABLES.md + // §3): every length, count and index on this wire has sixty-four bits of + // capability, so one past what remains must be compared UNSIGNED. Casting + // it to int64 first turns 0xFFFFFFFFFFFFFFFF into -1, and a negative + // length looks like room. + MAPDEMO_TABLE_INLINE bool room( uint64_t bytes ) const { return bytes <= (uint64_t) ( size - offset ); } + MAPDEMO_TABLE_INLINE uint8_t get8() { return buffer[offset++]; } + MAPDEMO_TABLE_INLINE uint16_t get16() { uint16_t v = uint16_t( buffer[offset] ) | uint16_t( buffer[offset+1] ) << 8; offset += 2; return v; } + MAPDEMO_TABLE_INLINE uint32_t get32() { uint32_t v = uint32_t( buffer[offset] ) | uint32_t( buffer[offset+1] ) << 8 | uint32_t( buffer[offset+2] ) << 16 | uint32_t( buffer[offset+3] ) << 24; offset += 4; return v; } + MAPDEMO_TABLE_INLINE uint64_t get64() { uint64_t lo = get32(); uint64_t hi = get32(); return lo | ( hi << 32 ); } + MAPDEMO_TABLE_INLINE void get128( uint64_t & lo, uint64_t & hi ) { lo = get64(); hi = get64(); } + + // ONE CANONICAL UNSIGNED LEB128 (docs/SPEC-TABLES.md §3), and a + // non-minimal spelling is MALFORMED: 0x80 0x00 and 0x00 both spell zero, + // and only the second is legal input. An encoding past ten bytes, or a + // tenth byte with a bit above the 64th value bit, is malformed on the same + // rule. false = framing damage on the body carrying it. + bool getleb( uint64_t & value ) + { + // A NUMBER THIS READER REFUSES LEAVES THE CURSOR WHERE IT WAS. The + // caller's next question is often "did this body end exactly at its + // L", and a rejected number that had moved the cursor would answer + // that question with the damage already stepped over. + const int64_t at = offset; + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( !has( 1 ) ) { offset = at; return false; } + const uint8_t b = get8(); + if ( i == 9 && b > 1 ) { offset = at; return false; } + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) + { + if ( i > 0 && b == 0 ) { offset = at; return false; } // a redundant continuation + return true; + } + shift += 7; + } + offset = at; + return false; + } + + // resolve one id reference against the file's table. false = a reference + // ABOVE the entry count, or a 0 where an id is required, both of which + // are framing damage on the body that carries it. + bool getid( uint64_t & id ) + { + uint64_t ref = 0; + if ( !getleb( ref ) ) { return false; } + if ( ref == 0 || ids == NULL || ref > (uint64_t) ids->count ) { return false; } + id = ids->at( ref ); + return true; + } + + // skip one payload by kind; false = framing damage. FOUR RULES COVER THE + // SET (docs/SPEC-TABLES.md §3), and a kind outside it is not skippable — + // which is why the set is closed and why kind 31 exists. + bool skip( uint8_t kind ) + { + switch ( kind ) + { + // the fixed-width kinds, each by its width: 18-29 are the 128-bit integers and + // the fixed-point family at every storage width (docs/SPEC-TABLES.md §3) + case 1: case 2: case 6: case 20: case 25: return has( 1 ) ? ( offset += 1, true ) : false; + case 3: case 7: case 21: case 26: return has( 2 ) ? ( offset += 2, true ) : false; + case 4: case 8: case 10: case 22: case 27: return has( 4 ) ? ( offset += 4, true ) : false; + case 5: case 9: case 11: case 23: case 28: return has( 8 ) ? ( offset += 8, true ) : false; + case 18: case 19: case 24: case 29: return has( 16 ) ? ( offset += 16, true ) : false; + case 17: case 30: // a NODE INDEX (§3.1) and an ENUM's variant reference: one LEB128 and stop + { + uint64_t ignored = 0; + return getleb( ignored ); + } + case 12: case 13: case 14: case 16: case 31: case 32: case 33: // 31 is the ESCAPE, 32 the payload-free kind, 33 wide text + { + uint64_t n = 0; + if ( !getleb( n ) ) return false; + return room( n ) ? ( offset += (int64_t) n, true ) : false; + } + case 15: // union: the arm id reference, then its kind, its L and its payload (reference 0 = empty) + { + uint64_t arm = 0; + if ( !getleb( arm ) ) return false; + if ( arm == 0 ) return true; + if ( !has( 1 ) ) return false; + offset += 1; // the arm's kind byte + uint64_t n = 0; + if ( !getleb( n ) ) return false; + return room( n ) ? ( offset += (int64_t) n, true ) : false; + } + // KIND 34 IS RESERVED FOR float16 AND IS NOT PART OF THIS MAJOR (§3): + // no writer emits it and no reader has a rule for it, so a reader + // meets it only as DAMAGE, exactly as it meets 35 or 200. A bare 34 + // is a writer that ignored the escape kind 31. + case 34: return false; + } + return false; + } +}; + + +// WIDENING (docs/SPEC-TABLES.md §4): a payload under a kind BELOW the reader's +// on the same ladder decodes exactly. The signed ladder is kinds 2, 3, 4, 5, +// 18, the unsigned one 6, 7, 8, 9, 19, and 10 into 11 is the float rung. Every +// other pair is a kind mismatch. The declared kind is a constant at every call +// site, so this folds to one or two comparisons on the mismatch path and to +// nothing on the matching one. +inline bool TableKindWidens( uint8_t kind, uint8_t declared ) +{ + switch ( declared ) + { + case 3: case 4: case 5: return kind >= 2 && kind < declared; + case 18: return kind >= 2 && kind <= 5; + case 7: case 8: case 9: return kind >= 6 && kind < declared; + case 19: return kind >= 6 && kind <= 9; + case 11: return kind == 10; + } + return false; +} + +// a fixed-width kind's payload width, for the one place the width is a +// runtime fact: an arm whose kind byte the reader widens, whose L must be the +// wire kind's own width (§3) +inline int64_t TableKindWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + } + return 0; +} + +// the payload of a kind on the SIGNED ladder (2 to 5), sign-extended to +// sixty-four bits; false = the body cannot cover it, which is framing damage +inline bool TableReadSignedAt( TableReader & r, uint8_t kind, int64_t & out ) +{ + switch ( kind ) + { + case 2: if ( !r.has( 1 ) ) { return false; } out = (int8_t) r.get8(); return true; + case 3: if ( !r.has( 2 ) ) { return false; } out = (int16_t) r.get16(); return true; + case 4: if ( !r.has( 4 ) ) { return false; } out = (int32_t) r.get32(); return true; + default: if ( !r.has( 8 ) ) { return false; } out = (int64_t) r.get64(); return true; + } +} + +// the payload of a kind on the UNSIGNED ladder (6 to 9), zero-extended +inline bool TableReadUnsignedAt( TableReader & r, uint8_t kind, uint64_t & out ) +{ + switch ( kind ) + { + case 6: if ( !r.has( 1 ) ) { return false; } out = r.get8(); return true; + case 7: if ( !r.has( 2 ) ) { return false; } out = r.get16(); return true; + case 8: if ( !r.has( 4 ) ) { return false; } out = r.get32(); return true; + default: if ( !r.has( 8 ) ) { return false; } out = r.get64(); return true; + } +} + +// f32 into f64, exact: a NaN's payload is data and rides on the bits, since +// the hardware conversion would set the quiet bit (§4) +inline double TableWidenF32( uint32_t bits ) +{ + if ( ( bits & 0x7F800000u ) == 0x7F800000u && ( bits & 0x007FFFFFu ) != 0 ) + { + const uint64_t sign = (uint64_t) ( bits >> 31 ) << 63; + const uint64_t payload = (uint64_t) ( bits & 0x007FFFFFu ) << 29; + const uint64_t nan_bits = sign | 0x7FF0000000000000ull | payload; + double d; memcpy( &d, &nan_bits, 8 ); return d; + } + float f; memcpy( &f, &bits, 4 ); return (double) f; +} + +// ILL-FORMED TEXT IS DAMAGE (docs/SPEC-TABLES.md §3, §4): a kind 12 payload is +// well-formed UTF-8 with no zero byte among its bytes, checked AS IT ARRIVES +// and before the reader's own bound, because a payload that is not text is not +// text at whatever length the reader would have kept. Rejects a zero byte, a +// truncated sequence, a bare continuation, an overlong encoding, a surrogate +// and a code point past U+10FFFF, which is SPEC.md §4.7's rule in this wire's +// idiom: the field reads its declared default, one malformed counts, and the +// parent reads on past L. +// +// A LENGTH IS A 64-BIT NUMBER (§3), so it arrives as one: a payload length is +// whatever the wire spelled, and narrowing it to a signed count would read +// 0xFFFFFFFFFFFFFFFF as an empty payload. +inline bool TableUtf8Valid( const uint8_t * bytes, uint64_t length ) +{ + uint64_t i = 0; + while ( i < length ) + { + const uint8_t lead = bytes[i]; + uint64_t continuations; + uint32_t code_point; + if ( lead == 0 ) { return false; } + if ( lead < 0x80 ) { i++; continue; } + else if ( ( lead & 0xE0 ) == 0xC0 ) { continuations = 1; code_point = lead & 0x1F; } + else if ( ( lead & 0xF0 ) == 0xE0 ) { continuations = 2; code_point = lead & 0x0F; } + else if ( ( lead & 0xF8 ) == 0xF0 ) { continuations = 3; code_point = lead & 0x07; } + else { return false; } + if ( i + continuations >= length ) { return false; } + for ( uint64_t k = 1; k <= continuations; k++ ) + { + if ( ( bytes[i + k] & 0xC0 ) != 0x80 ) { return false; } + code_point = ( code_point << 6 ) | uint32_t( bytes[i + k] & 0x3F ); + } + if ( continuations == 1 && code_point < 0x80 ) { return false; } + if ( continuations == 2 && ( code_point < 0x800 || ( code_point >= 0xD800 && code_point <= 0xDFFF ) ) ) { return false; } + if ( continuations == 3 && ( code_point < 0x10000 || code_point > 0x10FFFF ) ) { return false; } + i += 1 + continuations; + } + return true; +} + +// A CLAMP CUTS AT A CODE POINT BOUNDARY (§3, §16.2): the last whole code point +// that fits within the bound, over a payload the check above already accepted, +// so a clamp can never invent ill-formed storage. +// +// THE ANSWER IS NEVER ABOVE THE BOUND. The length arrives as the wire's own +// 64-bit number and the caller turns the answer back into the size of a copy, +// so a length no reader could have bounded has to leave here bounded: taken as +// a signed count, 0xFFFFFFFFFFFFFFFF is -1, -1 is under every bound, and the +// copy would run at SIZE_MAX. +inline int64_t TableUtf8Clamp( const uint8_t * bytes, uint64_t length, int64_t bound ) +{ + if ( length <= (uint64_t) bound ) { return (int64_t) length; } + int64_t cut = bound; + while ( cut > 0 && ( bytes[cut] & 0xC0 ) == 0x80 ) { cut--; } + return cut; +} + +// ONE CODE UNIT off the wire: two bytes LITTLE-ENDIAN, this wire's order for +// every fixed-width number (docs/SPEC-TABLES.md §3). No unit can exceed +// 0xFFFF, because two bytes cannot spell one. +inline uint16_t TableUtf16Unit( const uint8_t * bytes, int64_t index ) +{ + return uint16_t( uint16_t( bytes[index * 2] ) | ( uint16_t( bytes[index * 2 + 1] ) << 8 ) ); +} + +// ILL-FORMED WIDE TEXT IS DAMAGE (docs/SPEC-TABLES.md §3, §4): a kind 33 +// payload carrying an UNPAIRED SURROGATE or a ZERO CODE UNIT among its units, +// checked AS IT ARRIVES and before the reader's own bound, on the rule kind 12 +// takes for UTF-8. An ODD L is framing damage and the caller rejects it ahead +// of this, because units is L / 2. SPEC.md §4.12 refuses the same content +// TERMINALLY on the packet wire; here the field reads its declared default, +// one malformed counts, and the parent reads on past L. +inline bool TableUtf16Valid( const uint8_t * bytes, int64_t units ) +{ + int64_t i = 0; + while ( i < units ) + { + const uint16_t unit = TableUtf16Unit( bytes, i ); + if ( unit == 0 ) { return false; } + if ( unit >= 0xD800 && unit <= 0xDBFF ) + { + if ( i + 1 >= units ) { return false; } // a high surrogate with no low half + const uint16_t low = TableUtf16Unit( bytes, i + 1 ); + if ( low < 0xDC00 || low > 0xDFFF ) { return false; } + i += 2; + continue; + } + if ( unit >= 0xDC00 && unit <= 0xDFFF ) { return false; } // a low surrogate first + i++; + } + return true; +} + +// A CLAMP CUTS AT A CODE UNIT BOUNDARY AND NEVER SPLITS A PAIR (§3, §16.2): +// the first bound units of a payload the check above already accepted, and +// where the last kept unit is a HIGH SURROGATE whose low half did not fit, +// that unit is dropped with it. So a clamp can never invent an unpaired +// surrogate, exactly as kind 12's clamp can never invent a broken sequence. +inline int64_t TableUtf16Clamp( const uint8_t * bytes, int64_t units, int64_t bound ) +{ + if ( units <= bound ) { return units; } + int64_t cut = bound; + if ( cut > 0 ) + { + const uint16_t last = TableUtf16Unit( bytes, cut - 1 ); + if ( last >= 0xD800 && last <= 0xDBFF ) { cut--; } + } + return cut; +} + +// The RESERVED node-table id, the one id the language holds back +// (docs/SPEC-TABLES.md §3.1, §5). It rides in every unit, pointered or not, +// because every body has to know that a NESTED body claiming one is damaged. +static const uint64_t kTableNodeTableFieldId = 0xFFFFFFFFFFFFFFFFull; + +// TableWireForm is the FORM BYTE, and it is the whole header +// (docs/SPEC-TABLES.md §3). A reader that meets a byte it does not know +// refuses the wire by name and never reports damage. +const uint8_t kTableWireForm = 1; + +// TableOpen reads the form byte and the trailer, in that order, and hands back +// the ROOT BODY. It answers one of three verdicts, because five zero counters +// and a false flag are what a clean read prints too: +// +// TableOpenOk the form is known and the table read whole +// TableOpenRefused a FORM BYTE this reader does not carry: nothing is +// decoded, nothing is counted, and no damage is reported +// TableOpenDamaged a table that cannot be read whole — fewer than eight +// bytes, a count whose entries run past the front of the +// file, a count that leaves no room for the form byte, or +// ONE ID IN TWO ENTRIES. The whole wire is malformed, +// nothing is decoded, and one event is counted. +// TableOpenBodyStopped the form and the table were good and the ROOT BODY +// could not be walked to its own terminator. What it +// decoded before that is kept, as everywhere on this wire. +enum TableOpenVerdict { TableOpenOk, TableOpenRefused, TableOpenDamaged, TableOpenBodyStopped }; + +inline TableOpenVerdict TableOpen( const uint8_t * buffer, int64_t bytes, TableIdTable & table, int64_t & body_bytes ) +{ + if ( bytes < 1 ) { return TableOpenDamaged; } + if ( buffer[0] != kTableWireForm ) { return TableOpenRefused; } + if ( bytes < 9 ) { return TableOpenDamaged; } + const uint8_t * tail = buffer + bytes - 8; + uint64_t lo = uint64_t( tail[0] ) | uint64_t( tail[1] ) << 8 | uint64_t( tail[2] ) << 16 | uint64_t( tail[3] ) << 24; + uint64_t hi = uint64_t( tail[4] ) | uint64_t( tail[5] ) << 8 | uint64_t( tail[6] ) << 16 | uint64_t( tail[7] ) << 24; + uint64_t count = lo | ( hi << 32 ); + if ( count > (uint64_t) ( bytes / 8 ) ) { return TableOpenDamaged; } + const int64_t span = (int64_t) count * 8 + 8; + if ( span + 1 > bytes ) { return TableOpenDamaged; } + table.entries = buffer + bytes - span; + table.count = (int64_t) count; + // THE ENTRIES ARE DISTINCT: a table that carries one id twice is malformed + // for the whole wire, because no wire this schema writes carries a repeat + // and it would leave one more shape of table for a hostile writer to aim + // at (docs/SPEC-TABLES.md §3). + for ( int64_t i = 1; i < table.count; i++ ) + { + const uint64_t id = table.at( uint64_t( i ) + 1 ); + for ( int64_t j = 0; j < i; j++ ) + { + if ( table.at( uint64_t( j ) + 1 ) == id ) { return TableOpenDamaged; } + } + } + body_bytes = bytes - span - 1; + return TableOpenOk; +} + +// TableBodyExtent walks a body's framing to the zero reference that ends it, +// so a reader can tell a body that ENDED EARLY — leaving bytes no field claims +// — from one that is merely damaged. ANY BYTE BETWEEN THE ROOT'S TERMINATOR +// AND THE TABLE'S FIRST ENTRY IS MALFORMED, because no field claims it and the +// two ends of the file have met (docs/SPEC-TABLES.md §3). +inline bool TableBodyEndsEarly( const uint8_t * body, int64_t bytes, const TableIdTable & table ) +{ + TableReport ignored; + TableReader r( body, bytes, &ignored, &table ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.getleb( ref ) ) { return false; } + if ( ref == 0 ) { return r.offset != bytes; } + if ( ref > (uint64_t) table.count ) { return false; } + if ( !r.has( 1 ) ) { return false; } + if ( !r.skip( r.get8() ) ) { return false; } + } +} + +// THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3): a batch of BITPACKED bodies +// under one announced vocabulary. +// +// A form 2 wire is THREE PARTS: the form byte, the body count, and the bodies +// as one continuous bit stream, zero-padded to the next byte at the end and +// nowhere else. A body is a sequence of fields, each a REFERENCE followed by a +// PAYLOAD and nothing else: no kind byte and no length, because the +// announcement carries the kind and the shape of every entry. +const uint8_t kTableWireMessageForm = 2; + +// THE COUNT IS A RANGED INTEGER OVER [1, 256], eight bits carrying M - 1. 256 +// is a WIRE CONSTANT of this form rather than a receiver's policy, because the +// count's WIDTH depends on it and two peers that disagreed on the width would +// not be reading the same wire. A batch of zero is not spellable. +static const int64_t kTableMessageBatchMax = 256; + +// The RESERVED ids of the announcement's own two fields (§5, §11), beside the +// node table's. They are the announcement's transport, they never appear in a +// body, and they take no slot in the vocabulary. +static const uint64_t kTableBuildVersionFieldId = 0xFFFFFFFFFFFFFFFEull; +static const uint64_t kTableMessageVocabularyFieldId = 0xFFFFFFFFFFFFFFFDull; + +// THE WIDEST COUNT THIS FORM SPELLS, which is the count an UNBOUNDED array +// announces (§2.9): an unbounded array states no bound, so the announcement +// states the widest one a batch could carry. It is the ceiling an array's or a +// keyed entry's announced min and max are checked against. +static const uint64_t kTableMessageListMax = 0xFFFFFFFFull; + +// THIS UNIT'S OWN REFERENCE WIDTH: the bits a writer spends on every reference +// of every body it writes, which is a compile-time constant because the +// vocabulary is. A READER spends the width the SENDER's vocabulary settles. +static const int64_t kTableMessageRefBitsHere = 7; + +// THIS UNIT'S OWN ENTRY COUNT, which is the CAPACITY a receiver declares for +// its resolved vocabulary when it talks only to peers of this schema (§3.3). +// The vocabulary is a pure function of the build version, so a peer at this +// build announces exactly this many entries; a receiver that means to meet +// OTHER builds declares more, and an announcement above whatever it declared +// is refused as vocabulary_too_large. +static const int64_t kTableMessageEntriesHere = 75; + +// The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A +// pointered body names the node table through it, and the node table is the +// ROOT body's FIRST field because a pointer index's width is settled by the +// node count it carries. +static const uint64_t kTableNodeTableFieldSlot = 54; + +// THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own +// layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, +// so a value written here and a value written by a generated packet writer are +// the same bits in the same places. + +// EIGHT BYTES OF THE STREAM AS ONE WORD, and the word is LITTLE-END-FIRST +// whatever order this host is in, because the stream's own definition puts +// bit i in byte i/8: byte 0 of the run holds the word's low eight bits. That +// is what lets one value of any width move in one unaligned load or store +// instead of one touch a byte, and the BITS ON THE WIRE do not move. +inline uint64_t table_message_byteswap64( uint64_t v ) +{ + return ( v >> 56 ) | ( ( v >> 40 ) & 0xff00ull ) | ( ( v >> 24 ) & 0xff0000ull ) | ( ( v >> 8 ) & 0xff000000ull ) + | ( ( v << 8 ) & 0xff00000000ull ) | ( ( v << 24 ) & 0xff0000000000ull ) | ( ( v << 40 ) & 0xff000000000000ull ) + | ( v << 56 ); +} + +inline uint64_t table_message_load64( const uint8_t * p ) +{ + uint64_t v = 0; + memcpy( &v, p, 8 ); +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + v = table_message_byteswap64( v ); +#endif + return v; +} + +inline void table_message_store64( uint8_t * p, uint64_t v ) +{ +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + v = table_message_byteswap64( v ); +#endif + memcpy( p, &v, 8 ); +} + +struct TableBitWriter +{ + uint8_t * buffer; + int64_t capacity; // bytes + int64_t bits; + bool overflow; + + TableBitWriter() : buffer( NULL ), capacity( 0 ), bits( 0 ), overflow( false ) {} + TableBitWriter( uint8_t * to_buffer, int64_t to_capacity ) : buffer( to_buffer ), capacity( to_capacity ), bits( 0 ), overflow( false ) {} + + // ONE WORD AT A TIME, never one bit and never one byte of arithmetic: the + // value is shifted into place in a REGISTER once, and the bytes it + // occupies are stored from that register with no read back. A sixty-four + // bit field costs one shift rather than nine masked read-modify-writes. + // The word is assembled little-end-first, so the BITS ON THE WIRE are the + // same bits in the same places, bit i in byte i/8 at position i%8 with the + // low bit first, which is what the pinned goldens hold. IT WRITES EXACTLY + // THE BYTES THE VALUE OCCUPIES and never one past them, so a caller's + // buffer beyond the batch is its own. + void put( uint64_t value, int64_t n ) + { + if ( n <= 0 ) { return; } + if ( ( bits + n + 7 ) / 8 > capacity ) { overflow = true; bits += n; return; } + if ( n < 64 ) { value &= ( uint64_t( 1 ) << n ) - 1; } // a caller's high bits never leak + const int64_t index = bits >> 3; + const int64_t bit = bits & 7; + // the byte the write STARTS in keeps the bits already written to it, + // and every byte after it is this value's own + const uint64_t head = bit != 0 ? ( uint64_t( buffer[index] ) & ( ( uint64_t( 1 ) << bit ) - 1 ) ) : 0; + const uint64_t word = head | ( value << bit ); + const int64_t need = ( bit + n + 7 ) >> 3; // 1 to 9 bytes + if ( need >= 8 ) + { + table_message_store64( buffer + index, word ); + if ( need > 8 ) { buffer[index + 8] = uint8_t( value >> ( 64 - bit ) ); } + } + else + { + for ( int64_t i = 0; i < need; i++ ) { buffer[index + i] = uint8_t( word >> ( 8 * i ) ); } + } + bits += n; + } + + // THE ALIGN IS WHAT BUYS THIS (docs/SPEC-TABLES.md §3.3): a string(N), a + // bytes(N) and a blob record align before their bytes precisely so the + // largest payload on the wire moves as ONE memcpy. Off a boundary there is + // nothing to memcpy and the bytes go through put. + void putbytes( const uint8_t * data, int64_t n ) + { + if ( n <= 0 ) { return; } + if ( ( bits & 7 ) == 0 ) + { + if ( ( bits >> 3 ) + n > capacity ) { overflow = true; bits += n * 8; return; } + memcpy( buffer + ( bits >> 3 ), data, (size_t) n ); + bits += n * 8; + return; + } + for ( int64_t i = 0; i < n; i++ ) { put( (uint64_t) data[i], 8 ); } + } + + // a string's or a bytes' payload ALIGNS before its bytes, and a batch + // aligns once at its end. Both are zero fill, spent in one call. + void align() { put( 0, ( 8 - ( bits & 7 ) ) & 7 ); } +}; + +// TableAlignBits is what an align costs from a bit position, which a measure +// spends exactly where a save does. +inline int64_t TableAlignBits( int64_t bits ) { return ( 8 - ( bits % 8 ) ) % 8; } + +struct TableBitReader +{ + const uint8_t * buffer; + int64_t bits; // the stream's extent, in bits + int64_t offset; // bits consumed + + TableBitReader() : buffer( NULL ), bits( 0 ), offset( 0 ) {} + TableBitReader( const uint8_t * from_buffer, int64_t from_bytes ) : buffer( from_buffer ), bits( from_bytes * 8 ), offset( 0 ) {} + + bool has( int64_t n ) const { return n >= 0 && offset + n <= bits; } + + // the primitive is sixty-four bits, and a width above it is refused + // here as well as at the announcement: no field on any body can ask this + // reader to move more bits than it holds + // ONE WORD OF THE BUFFER AT A TIME, the mirror of the writer's put: the + // eight bytes the value starts in load as one little-end-first word and a + // ninth byte carries the spill a value that straddles the word needs. + // Within nine bytes of the stream's end there is no room for a word load + // and the bytes come one at a time, by the same arithmetic. + bool get( uint64_t & value, int64_t n ) + { + if ( n > 64 || !has( n ) ) { return false; } + value = 0; + if ( n == 0 ) { return true; } + const int64_t index = offset >> 3; + const int64_t bit = offset & 7; + const int64_t bytes = ( bits + 7 ) >> 3; + if ( index + 9 <= bytes ) + { + uint64_t v = table_message_load64( buffer + index ) >> bit; + if ( bit != 0 && bit + n > 64 ) { v |= uint64_t( buffer[index + 8] ) << ( 64 - bit ); } + value = n == 64 ? v : ( v & ( ( uint64_t( 1 ) << n ) - 1 ) ); + offset += n; + return true; + } + int64_t got = 0; + while ( got < n ) + { + const int64_t byte = offset >> 3; + const int64_t off = offset & 7; + const int64_t room = 8 - off; + const int64_t take = ( n - got ) < room ? ( n - got ) : room; + const uint64_t chunk = ( uint64_t( buffer[byte] ) >> off ) & ( ( uint64_t( 1 ) << take ) - 1 ); + value |= chunk << got; + offset += take; + got += take; + } + return true; + } + + // the bytes of an ALIGNED payload, which is the read side of the memcpy + // the align buys (docs/SPEC-TABLES.md §3.3) + bool getbytes( uint8_t * out, int64_t n ) + { + if ( n < 0 || !has( n * 8 ) ) { return false; } + if ( ( offset & 7 ) == 0 ) + { + memcpy( out, buffer + ( offset >> 3 ), (size_t) n ); + offset += n * 8; + return true; + } + for ( int64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !get( by, 8 ) ) { return false; } + out[i] = (uint8_t) by; + } + return true; + } + + bool skip( int64_t n ) { if ( !has( n ) ) { return false; } offset += n; return true; } + + // the pad to the next byte boundary is VERIFIED ZERO, which is the packet + // wire's rule for the same reason (SPEC.md §4.3) + bool align() + { + const int64_t pad = ( 8 - ( offset & 7 ) ) & 7; + if ( pad == 0 ) { return true; } + uint64_t bits_read = 0; + return get( bits_read, pad ) && bits_read == 0; + } +}; + +// TableBitsRequired is bits_required( min, max ): the bit length of max - min, +// and zero where the two are equal, which is a value that spends no bit at all. +inline int64_t TableBitsRequired( int64_t min, int64_t max ) +{ + if ( max <= min ) { return 0; } + uint64_t span = (uint64_t) ( max - min ); + int64_t n = 0; + while ( span > 0 ) { n++; span >>= 1; } + return n; +} + +// THE ANNOUNCED ENTRY (§3.3): an id, a kind, and a SHAPE, which is the width +// and range facts a reader needs to SKIP a field exactly and to DECODE one +// whose own declaration has moved. One name may take TWO entries, at two kinds or two +// shapes, and a body names the one it means. +// +// The ELEMENT's own facts ride beside the field's because an array's element +// is the one nesting this wire has: an array of arrays is not a table-wire +// construct, so one level is every level. +// +// IT IS THE RESOLVED ENTRY AND THE CALLER SIZES AN ARRAY OF THEM, so it +// carries what a DECODE takes and nothing a decode does not: qmin, qdelta and +// qcount are what SPEC.md §4.3's rule leaves behind, and the qmax and qres +// that rule CONSUMES are locals of the parse. The widths are int16 because a +// width is bounded by the kind it came under and no kind holds more than 128 +// bits. +struct TableMessageEntry +{ + uint64_t id = 0; + int64_t min = 0; // an array's minimum count + int64_t max = 0; // an array's maximum count, a string's capacity, a keyed array's slots + int64_t base_lo = 0; // the ranged base, low half: a signed kind's sign-extends, an unsigned kind's is whole + int64_t base_hi = 0; // its high half, for a 128-bit kind + int64_t elem_max = 0; + int64_t elem_base_lo = 0; + int64_t elem_base_hi = 0; + // what SPEC.md §4.3's derivation leaves: the base, the step and the count + float qmin = 0.0f; + float qdelta = 0.0f; + uint32_t qcount = 0; + float elem_qmin = 0.0f; + float elem_qdelta = 0.0f; + uint32_t elem_qcount = 0; + // THE PAYLOAD'S WIDTH, RESOLVED: what the kind, the packing and the + // announced bits together say, computed once at AnnounceRead, and -1 + // where the payload is not a fixed-width value at all + int16_t value_bits = -1; + int16_t elem_value_bits = -1; + uint8_t kind = 0; + uint8_t packing = 0; + uint8_t elem_kind = 0; + uint8_t elem_packing = 0; +}; + +// TableMessageEntrySame reports whether two RESOLVED entries carry the same +// shape, which is every fact of the entry but its id and its kind. It is what +// the announcement's duplicate rule is asked in: two entries that agree on all +// three parts are malformed (§3.3). +inline bool TableMessageEntrySame( const TableMessageEntry & a, const TableMessageEntry & b ) +{ + return a.min == b.min && a.max == b.max && a.base_lo == b.base_lo && a.base_hi == b.base_hi + && a.elem_max == b.elem_max && a.elem_base_lo == b.elem_base_lo && a.elem_base_hi == b.elem_base_hi + && a.qmin == b.qmin && a.qdelta == b.qdelta && a.qcount == b.qcount + && a.elem_qmin == b.elem_qmin && a.elem_qdelta == b.elem_qdelta && a.elem_qcount == b.elem_qcount + && a.value_bits == b.value_bits && a.elem_value_bits == b.elem_value_bits + && a.packing == b.packing && a.elem_kind == b.elem_kind && a.elem_packing == b.elem_packing; +} + +// TableMessageKindBits is the widest RANGED value a kind can carry, its own +// storage width: a width above it is a hostile width on the announcement. +inline int64_t TableMessageKindBits( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 6: case 20: case 25: return 8; + case 3: case 7: case 21: case 26: return 16; + case 4: case 8: case 22: case 27: return 32; + case 5: case 9: case 23: case 28: return 64; + default: return 128; + } +} + +// TableMessageQuantization is SPEC.md §4.3's derivation over an announced +// triple, in float32 and by nothing else: delta, the step count and the +// width. False is a triple SPEC.md calls non-conforming, which on the +// announcement is a hostile width like any other (§3.3). +inline bool TableMessageQuantization( float qmin, float qmax, float qres, float & delta, uint32_t & count, int64_t & bits ) +{ + if ( !( qmin < qmax ) || !( qres > 0.0f ) ) { return false; } + delta = qmax - qmin; + float values = delta / qres; + if ( !( delta - delta == 0.0f ) || !( values - values == 0.0f ) ) { return false; } // Inf - Inf is NaN + if ( !( values >= 1.0f ) ) { values = 1.0f; } + else if ( values > 4294967040.0f ) { values = 4294967040.0f; } // the largest float below 2^32 + count = (uint32_t) values; + if ( (float) count < values ) { count++; } // ceil, on a value the cast holds exactly + bits = TableBitsRequired( 0, (int64_t) count ); + return true; +} + +// The two roundings on each side of the rule (SPEC.md §7.2): the product +// rounds to float32 BEFORE the add, which a compiler permitted to contract +// would otherwise fuse into one rounding and move the wire. +#if ( defined( __GNUC__ ) || defined( __clang__ ) ) && ( defined( __aarch64__ ) || defined( _M_ARM64 ) ) +#define TABLE_FLOAT_FORCE_ROUND( x ) __asm__ ( "" : "+w" ( x ) ) +#elif ( defined( __GNUC__ ) || defined( __clang__ ) ) && ( defined( __x86_64__ ) || defined( __i386__ ) ) +#define TABLE_FLOAT_FORCE_ROUND( x ) __asm__ ( "" : "+x" ( x ) ) +#else +#define TABLE_FLOAT_FORCE_ROUND( x ) do { volatile float table_float_force_round_slot = ( x ); ( x ) = table_float_force_round_slot; } while ( 0 ) +#endif + +// TableMessageQuantize is the writer's half: the index a value takes. +inline uint32_t TableMessageQuantize( float value, float qmin, float delta, uint32_t count ) +{ + float normalized = ( value - qmin ) / delta; + if ( !( normalized >= 0.0f ) ) { normalized = 0.0f; } + else if ( !( normalized <= 1.0f ) ) { normalized = 1.0f; } + float scaled = normalized * (float) count; + TABLE_FLOAT_FORCE_ROUND( scaled ); + uint32_t index = (uint32_t) ( scaled + 0.5f ); // floor of a non-negative value + if ( index > count ) { index = count; } + return index; +} + +// TableMessageDequantize is the reader's half: the float an index names. +inline float TableMessageDequantize( uint32_t index, float qmin, float delta, uint32_t count ) +{ + if ( index > count ) { index = count; } + const float normalized = index / (float) count; + float scaled = normalized * delta; + TABLE_FLOAT_FORCE_ROUND( scaled ); + return scaled + qmin; +} + +inline bool TableMessageIntegerKind( uint8_t kind ) +{ + return ( kind >= 2 && kind <= 9 ) || kind == 18 || kind == 19; +} + +inline bool TableMessageFixedKind( uint8_t kind ) { return kind >= 20 && kind <= 29; } + +inline bool TableMessageKnownKind( uint8_t kind ) +{ + return kind == 0 || ( kind >= 1 && kind <= 17 ) || ( kind >= 18 && kind <= 29 ) || ( kind >= 30 && kind <= 33 ); +} + +// A CANONICAL LEB128, which is the announcement's own integer: the +// announcement is a form 1 FILE and takes §3's rule. +inline bool TableMessageLeb( const uint8_t * in, int64_t size, int64_t & at, uint64_t & value ) +{ + value = 0; + for ( int64_t shift = 0; at < size; shift += 7 ) + { + if ( shift >= 64 ) { return false; } + const uint8_t by = in[ at++ ]; + value |= uint64_t( by & 0x7F ) << shift; + if ( ( by & 0x80 ) == 0 ) { return !( shift > 0 && by == 0 ); } + } + return false; +} + +// TableMessageShapeFacts is where one shape's facts land: the field's own, +// or its element's, which is the one nesting this wire has. +struct TableMessageShapeFacts +{ + uint8_t & packing; int64_t & value_bits; int64_t & base_lo; int64_t & base_hi; + float & qmin; float & qmax; float & qres; float & qdelta; uint32_t & qcount; + int64_t & min; int64_t & max; uint8_t & elem_kind; +}; + +inline bool TableMessageShapeRead( const uint8_t * in, int64_t size, int64_t & at, uint8_t kind, TableMessageShapeFacts f ); +inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t value_bits ); + +// TableMessageEntryRead parses ONE entry, and answers false for a HOSTILE +// SHAPE: bits above the kind's own domain, an array whose min exceeds its +// max, an element kind outside the closed set, a quantized triple SPEC.md +// calls non-conforming, or a shape running past the vocabulary's own bytes. +inline bool TableMessageEntryRead( const uint8_t * in, int64_t size, int64_t & at, TableMessageEntry & entry ) +{ + if ( at + 9 > size ) { return false; } + entry = TableMessageEntry(); + for ( int i = 0; i < 8; i++ ) { entry.id |= uint64_t( in[ at + i ] ) << ( 8 * i ); } + entry.kind = in[ at + 8 ]; + at += 9; + if ( !TableMessageKnownKind( entry.kind ) ) { return false; } + // The parse lands in LOCALS and the entry keeps what a decode reads: the + // quantized max and res are the derivation's inputs and never a field's. + uint8_t packing = 0, elem_kind = 0; + int64_t bits = 0, base_lo = 0, base_hi = 0, min = 0, max = 0; + float qmin = 0.0f, qmax = 0.0f, qres = 0.0f, qdelta = 0.0f; + uint32_t qcount = 0; + TableMessageShapeFacts own = { packing, bits, base_lo, base_hi, + qmin, qmax, qres, qdelta, qcount, + min, max, elem_kind }; + if ( !TableMessageShapeRead( in, size, at, entry.kind, own ) ) { return false; } + entry.packing = packing; + entry.value_bits = (int16_t) TableMessageValueBits( entry.kind, packing, bits ); + entry.base_lo = base_lo; + entry.base_hi = base_hi; + entry.qmin = qmin; + entry.qdelta = qdelta; + entry.qcount = qcount; + entry.min = min; + entry.max = max; + entry.elem_kind = elem_kind; + if ( entry.kind == 14 || entry.kind == 16 ) + { + uint8_t elem_packing = 0, inner_kind = 0; + int64_t elem_bits = 0, elem_base_lo = 0, elem_base_hi = 0, elem_min = 0, elem_max = 0; + float elem_qmin = 0.0f, elem_qmax = 0.0f, elem_qres = 0.0f, elem_qdelta = 0.0f; + uint32_t elem_qcount = 0; + TableMessageShapeFacts elem = { elem_packing, elem_bits, elem_base_lo, elem_base_hi, + elem_qmin, elem_qmax, elem_qres, elem_qdelta, elem_qcount, + elem_min, elem_max, inner_kind }; + if ( !TableMessageShapeRead( in, size, at, entry.elem_kind, elem ) ) { return false; } + entry.elem_packing = elem_packing; + entry.elem_value_bits = (int16_t) TableMessageValueBits( entry.elem_kind, elem_packing, elem_bits ); + entry.elem_base_lo = elem_base_lo; + entry.elem_base_hi = elem_base_hi; + entry.elem_qmin = elem_qmin; + entry.elem_qdelta = elem_qdelta; + entry.elem_qcount = elem_qcount; + entry.elem_max = elem_max; + } + return true; +} + +// TableMessageShapeRead is one shape, by the kind that names it (§3.3's shape +// table). Every number in it is a canonical LEB128 except where the row says +// otherwise: a RANGED BASE IS ENCODED BY ITS KIND'S SIGNEDNESS, zigzag for the +// signed kinds, unsigned for the unsigned kinds and sixteen bytes for the +// 128-bit and fixed-point kinds, and a QUANTIZED f32 carries min, max and res +// as float32, from which the step count and the width derive by SPEC.md +// §4.3's rule and by nothing else. +inline bool TableMessageShapeRead( const uint8_t * in, int64_t size, int64_t & at, uint8_t kind, TableMessageShapeFacts f ) +{ + uint64_t v = 0; + if ( TableMessageIntegerKind( kind ) || TableMessageFixedKind( kind ) || kind == 10 ) + { + if ( at >= size ) { return false; } + f.packing = in[ at++ ]; + if ( f.packing == 0 ) { return true; } + if ( f.packing == 1 && kind != 10 ) + { + if ( !TableMessageLeb( in, size, at, v ) || (int64_t) v > TableMessageKindBits( kind ) ) { return false; } + f.value_bits = (int64_t) v; + if ( kind == 18 || kind == 19 || TableMessageFixedKind( kind ) ) + { + if ( at + 16 > size ) { return false; } + uint64_t lo = 0, hi = 0; + for ( int i = 0; i < 8; i++ ) { lo |= uint64_t( in[ at + i ] ) << ( 8 * i ); } + for ( int i = 0; i < 8; i++ ) { hi |= uint64_t( in[ at + 8 + i ] ) << ( 8 * i ); } + f.base_lo = (int64_t) lo; f.base_hi = (int64_t) hi; + at += 16; + return true; + } + if ( !TableMessageLeb( in, size, at, v ) ) { return false; } + if ( kind >= 2 && kind <= 5 ) { f.base_lo = (int64_t) ( v >> 1 ) ^ -(int64_t) ( v & 1 ); } // zigzag + else { f.base_lo = (int64_t) v; } // the unsigned domain, whole + return true; + } + if ( f.packing == 2 && kind == 10 ) + { + if ( at + 12 > size ) { return false; } + uint32_t raw[3] = { 0, 0, 0 }; + for ( int k = 0; k < 3; k++ ) { for ( int i = 0; i < 4; i++ ) { raw[k] |= uint32_t( in[ at + 4 * k + i ] ) << ( 8 * i ); } } + at += 12; + memcpy( &f.qmin, &raw[0], 4 ); + memcpy( &f.qmax, &raw[1], 4 ); + memcpy( &f.qres, &raw[2], 4 ); + return TableMessageQuantization( f.qmin, f.qmax, f.qres, f.qdelta, f.qcount, f.value_bits ); + } + return false; // a packing outside the closed set + } + // A MAX ABOVE WHAT THE KIND CAN HOLD IS A HOSTILE WIDTH (§3.3). A string + // and a wide string are bounded by the int32 storage cap the checker + // applies to every N (SPEC §4.3, §6.1), and an array and a keyed entry by + // the 32-bit count an unbounded array announces (§2.9), which is the + // widest count this form spells. A larger bound is a shape no conforming + // declaration can produce, and a reader that carried it would do its + // length arithmetic in a range that overflows. + if ( kind == 12 || kind == 33 ) + { + if ( !TableMessageLeb( in, size, at, v ) || v > (uint64_t) INT32_MAX ) { return false; } + f.max = (int64_t) v; + return true; + } + if ( kind == 14 || kind == 16 ) + { + if ( kind == 14 ) + { + if ( !TableMessageLeb( in, size, at, v ) || v > kTableMessageListMax ) { return false; } + f.min = (int64_t) v; + } + if ( !TableMessageLeb( in, size, at, v ) || v > kTableMessageListMax ) { return false; } + if ( (int64_t) v < f.min ) { return false; } + f.max = (int64_t) v; + if ( at >= size ) { return false; } + f.elem_kind = in[ at++ ]; + if ( !TableMessageKnownKind( f.elem_kind ) ) { return false; } + // AND AN ELEMENT KIND OF 12 OR 33 IS REFUSED HERE, at the + // announcement, rather than at the skip that would meet it (§3.3): no + // declaration this language accepts is an array of string(N) or of + // wstring(N), so a shape announcing one is one rule's business and not + // two. + if ( f.elem_kind == 12 || f.elem_kind == 33 ) { return false; } + return true; + } + return true; +} + +// TableMessageValueBits is one value's width under a shape, and -1 where the +// kind's payload is not a fixed-width value at all. +inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t value_bits ) +{ + if ( kind == 1 ) { return 1; } + if ( kind == 11 ) { return 64; } + if ( kind == 10 ) { return packing == 2 ? value_bits : 32; } + if ( TableMessageIntegerKind( kind ) || TableMessageFixedKind( kind ) ) + { + if ( packing == 1 ) { return value_bits; } + switch ( kind ) + { + case 2: case 6: case 20: case 25: return 8; + case 3: case 7: case 21: case 26: return 16; + case 4: case 8: case 22: case 27: return 32; + case 5: case 9: case 23: case 28: return 64; + default: return 128; + } + } + return -1; +} + +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an +// ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under +// the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 +// over element kind 6, and a trailer of those two reserved ids. +// +// THE VOCABULARY IS A FIELD AND NOT THE TRAILER, and that buys three things: +// §3's writer rule that an id no body references is never written is restored +// unbroken, an entry can carry a KIND and a SHAPE which a trailer of bare ids +// cannot, and one NAME can appear at two shapes. +// +// The order is the COOK PROJECTION's (§20.2): each record in the order the +// projection renders it and each record's fields in the order the projection +// renders them, then each enum's variants and each union's arms. Then comes +// the tail the projection does not name: the reserved node-table id, the three +// blob type ids as bytes, string and wstring, and every table's own name id in +// the projection's sorted record order. The tail is UNCONDITIONAL, so an +// ordinary edit only ever grows it at its end and never moves a slot a +// generated field header carries as a literal. +static const int64_t kTableAnnounceBytes = 901; +static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, + 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, + 0xe4, 0x7c, 0x11, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, + 0x20, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0d, 0xec, 0x10, + 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, + 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, + 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, + 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, + 0x07, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x21, 0x06, + 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0c, 0x10, 0x38, 0x81, + 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, + 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, +}; + +// TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries +// an announcement carried, RESOLVED ONCE, under one numbering. +// +// THE RECEIVER RESOLVES ONCE (§3.3), so this holds the entries themselves and +// not the announcement's bytes: every entry is parsed at AnnounceRead, and +// every body after it dispatches through ONE ARRAY INDEX with nothing to +// re-read and nothing to decide. The announcement is free the moment +// AnnounceRead returns. +// +// THE STORAGE IS THE CALLER'S and this library never allocates. The caller +// declares an array of entries wherever it wants it, static, on a heap, in an +// arena or beside its connection, and hands it here with its CAPACITY. The +// announcement holds for the life of the connection (§3.3), so the array does +// too, and a peer holds TWO for a connection, the one it writes with and the +// one it reads with. A restart opens a fresh connection with an empty +// vocabulary and nothing is cached across connections. +// +// kTableMessageEntriesHere is the capacity a receiver that talks only to peers +// of THIS schema declares, and a receiver meeting other builds declares more. +struct TableVocabulary +{ + // THE CONFORMING DEFAULT BYTE BOUND (§3.3). The ENTRY bound has no default + // because it IS the caller's capacity: an announcement naming more entries + // than the caller made room for is refused as vocabulary_too_large before + // an entry is touched, and the byte bound is read off the vocabulary + // field's own length before that. + static const int64_t kDefaultMaxBytes = 64 * 1024; + + TableVocabulary( TableMessageEntry * storage, int64_t capacity ) + : entries( storage ), max_entries( capacity ) {} + + TableMessageEntry * entries; // THE CALLER'S, capacity max_entries + int64_t max_entries; + int64_t count = 0; + int64_t ref_bits = 0; + uint64_t build_version = 0; + bool announced = false; + // REFUSAL IS TERMINAL (§3.3): a connection whose first announcement was + // refused, for any reason, carries no vocabulary for its life, and every + // announcement after it is refused as second_announcement + bool refused = false; + int64_t max_bytes = kDefaultMaxBytes; +}; + +// TableVocabularyEntryAt is the entry a reference names, counted from 1: ONE +// ARRAY INDEX into the caller's resolved storage, no parse and no branch. +inline const TableMessageEntry & TableVocabularyEntryAt( const TableVocabulary & vocabulary, uint64_t slot ) +{ + return vocabulary.entries[ slot - 1 ]; +} + +// AnnounceRead reads an announcement into one direction's vocabulary (§3.3). +// +// The announcement IS a file, so every malformed rule of §3 already covers it. +// Over its body there are EXACTLY TWO STRICT CHECKS: the BUILD VERSION +// present, exactly once, under kind 9, eight bytes wide, and the VOCABULARY +// present, exactly once, under kind 14 over element kind 6. Everything else is +// ordinary and tolerant, so an unknown field is skipped and counted and the +// announcement can GAIN a field in a later minor without a lockstep redeploy. +// +// The FIRST announcement sets the vocabulary and it is the only one that can. +// A SECOND is refused by name: it does not replace it, does not amend it and +// changes nothing. A refused announcement sets NO VOCABULARY, and the refusal +// is TERMINAL: every announcement after it, whether or not the first set +// anything, is second_announcement, so a peer holds no retry on the +// connection and cannot buy a second resolve by having its first refused. +inline bool AnnounceReadOnce( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * to ); + +inline bool AnnounceRead( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * to = report != NULL ? report : &ignored; + if ( vocabulary.announced || vocabulary.refused ) + { + to->refused = true; + to->reason = second_announcement; + return false; + } + const bool set = AnnounceReadOnce( vocabulary, buffer, bytes, to ); + if ( !set ) { vocabulary.refused = true; } + return set; +} + +inline bool AnnounceReadOnce( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * to ) +{ + if ( bytes < 1 ) { to->malformed = true; return false; } + if ( buffer[0] != kTableWireForm ) + { + to->refused = true; + to->reason = buffer[0] == kTableWireMessageForm ? message_form_as_file : newer_form; + return false; + } + if ( bytes < 9 ) { to->malformed = true; return false; } + TableIdTable table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( buffer, bytes, table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { to->malformed = true; } + else { to->refused = true; to->reason = newer_form; } + return false; + } + if ( TableBodyEndsEarly( buffer + 1, body_bytes, table ) ) { to->malformed = true; return false; } + TableReader r( buffer + 1, body_bytes, to, &table ); + uint64_t version = 0; + const uint8_t * words = NULL; + int64_t words_bytes = 0; + int32_t seen_version = 0, seen_vocabulary = 0; + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.getleb( ref ) ) { to->malformed = true; return false; } + if ( ref == 0 ) { break; } + if ( ref > (uint64_t) table.count || !r.has( 1 ) ) { to->malformed = true; return false; } + const uint64_t id = table.at( ref ); + const uint8_t kind = r.get8(); + if ( id == kTableBuildVersionFieldId ) + { + if ( kind != 9 || !r.has( 8 ) ) { to->malformed = true; return false; } + version = r.get64(); + // THE BUILD VERSION IS KEPT THE MOMENT IT IS READ, refusal or not, so + // that a refusal on this connection NAMES IT (§3.3). It is not the + // vocabulary, and a refused announcement still sets none. + vocabulary.build_version = version; + seen_version++; + continue; + } + if ( id == kTableMessageVocabularyFieldId ) + { + // kind 14 over element kind 6, which is §3's spelling for an + // opaque run of bytes + uint64_t framed = 0; + if ( kind != 14 || !r.getleb( framed ) || !r.has( (int64_t) framed ) ) { to->malformed = true; return false; } + const int64_t begin = r.offset, end = r.offset + (int64_t) framed; + r.offset = end; + if ( begin >= end || r.buffer[ begin ] != 6 ) { to->malformed = true; return false; } + int64_t at = begin + 1; + uint64_t length = 0; + if ( !TableMessageLeb( r.buffer, end, at, length ) || at + (int64_t) length != end ) { to->malformed = true; return false; } + if ( (int64_t) length > vocabulary.max_bytes ) { to->refused = true; to->reason = vocabulary_too_large; return false; } + words = r.buffer + at; + words_bytes = (int64_t) length; + seen_vocabulary++; + continue; + } + to->unknown++; + if ( !r.skip( kind ) ) { to->malformed = true; return false; } + } + if ( seen_version != 1 || seen_vocabulary != 1 ) { to->malformed = true; return false; } + + // THE ENTRIES, RESOLVED ONCE into the caller's storage (§3.3): every width + // is checked here and never again, and no body after this parses a byte of + // an announcement. An entry count above the caller's CAPACITY is refused + // by name before the entry is touched. + int64_t at = 0, count = 0, node_table_slots = 0; + while ( at < words_bytes ) + { + if ( count >= vocabulary.max_entries ) { to->refused = true; to->reason = vocabulary_too_large; return false; } + TableMessageEntry & parsed = vocabulary.entries[ count ]; + if ( !TableMessageEntryRead( words, words_bytes, at, parsed ) ) { to->malformed = true; return false; } + // THE RESERVED IDS WHERE THEY DO NOT BELONG (§3.3): the announcement's + // own two never take a slot, and the node-table id takes exactly one, + // so a vocabulary carrying either of the first or a SECOND node-table + // id is malformed whole and sets nothing + if ( parsed.id == kTableBuildVersionFieldId || parsed.id == kTableMessageVocabularyFieldId ) { to->malformed = true; return false; } + if ( parsed.id == kTableNodeTableFieldId ) { if ( node_table_slots++ > 0 ) { to->malformed = true; return false; } } + // A TRIPLE ALREADY PLACED IS NEVER PLACED TWICE, so two entries that + // agree on the id, the kind and every fact of the shape are malformed + // (§3.3): no writer this wire has produces one, and a reader that took + // it would carry two slots naming one thing. The scan is quadratic in + // the entry count, and the entry count is bounded above at 4096, so it + // is at most eight million compares on a path that runs ONCE a + // connection and never again. + for ( int64_t seen = 0; seen < count; seen++ ) + { + const TableMessageEntry & other = vocabulary.entries[ seen ]; + if ( other.id == parsed.id && other.kind == parsed.kind && TableMessageEntrySame( other, parsed ) ) { to->malformed = true; return false; } + } + count++; + } + vocabulary.count = count; + vocabulary.ref_bits = TableBitsRequired( 0, count ); + vocabulary.build_version = version; + vocabulary.announced = true; + return true; +} + +// TableMessageReserved is one of the three ids the language holds back (§3.1, +// §3.3, §5): each is malformed anywhere but its own transport, and the rule +// OUTRANKS the wrong-sort rule below. +inline bool TableMessageReserved( uint64_t id ) +{ + // THE THREE ARE THE TOP THREE VALUES a uint64 holds, so the test is ONE + // comparison: 0xFFFFFFFFFFFFFFFD, FE and FF and nothing else is at or + // above the vocabulary's own id, and a declaration hashing to any of them + // is refused by name (§11) + return id >= kTableMessageVocabularyFieldId; +} + +// TableMessageNameEntry resolves a reference used as a VALUE, which is an +// enum's variant, a keyed array's slot key or a node record's type id, and +// which must name a kind-0 entry (§3.3). A reference of 0 where an entry is required, one +// above E, one naming a reserved id and one naming an entry that carries a +// payload are each damage: the reader RESOLVED the entry and it contradicts +// the position it was used in, so the next bit's meaning is what is in doubt. +inline bool TableMessageNameEntry( const TableVocabulary & vocabulary, uint64_t ref, TableMessageEntry & entry ) +{ + if ( ref == 0 || ref > (uint64_t) vocabulary.count ) { return false; } + entry = TableVocabularyEntryAt( vocabulary, ref ); + return !TableMessageReserved( entry.id ) && entry.kind == 0; +} + +// TableMessageArmEntry resolves a UNION's arm reference, which must name an +// entry carrying the arm's own kind and shape: a kind-0 entry frames nothing, +// and a reserved id belongs to no arm (§3.3). +inline bool TableMessageArmEntry( const TableVocabulary & vocabulary, uint64_t ref, TableMessageEntry & entry ) +{ + if ( ref == 0 || ref > (uint64_t) vocabulary.count ) { return false; } + entry = TableVocabularyEntryAt( vocabulary, ref ); + return !TableMessageReserved( entry.id ) && entry.kind != 0; +} + +// TableMessageSkipVariant steps over an ENUM's variant reference on a SKIP +// path and RESOLVES it while it is there: 0 is None and the whole payload, and +// every other reference must name a kind-0 entry, because every reference +// above E is damage and one naming an entry that carries a payload +// contradicts the position it was used in, whether or not this reader was +// going to keep the value (§3.3). +inline bool TableMessageSkipVariant( TableBitReader & r, const TableVocabulary & vocabulary ) +{ + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + TableMessageEntry named; + return TableMessageNameEntry( vocabulary, ref, named ); +} +// TableMessageSkip steps over one field's payload without decoding it, using +// the announced ENTRY alone (§3.3). It is what makes an unknown entry +// skippable on a body with no kind byte, and it is ONE function over every +// table, because a shape says everything a skipper needs. +inline bool TableMessageSkipBody( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits ); +inline bool TableMessageSkip( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ); + +// TableMessageSkipElement steps over ONE element of an array or keyed entry +// by the element's own announced shape: a nested body to its zero reference, +// a variant or a node index at its reference width, a union arm by its own +// entry, and a fixed-width value at its bits. +inline bool TableMessageSkipElement( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ) +{ + switch ( entry.elem_kind ) + { + case 13: return TableMessageSkipBody( r, vocabulary, index_bits ); + case 30: return TableMessageSkipVariant( r, vocabulary ); + case 17: return index_bits > 0 && r.skip( index_bits ); + case 15: + { + TableMessageEntry inner; + inner.kind = 15; + return TableMessageSkip( r, vocabulary, index_bits, inner ); + } + default: + { + const int64_t elem = entry.elem_value_bits; + return elem >= 0 && r.skip( elem ); + } + } +} + +// TableMessageElementRunBits is the bits ONE element of an array or a keyed +// entry occupies on the SKIP path, where nothing is resolved and a run of them +// is one multiplication, and -1 where the element's width is its own +// content's. A ZERO is a real answer, and it is why this exists: a ranged +// element whose min equals its max rides no bits at all (§3.3). +inline int64_t TableMessageElementRunBits( const TableVocabulary & vocabulary, const TableMessageEntry & entry ) +{ + int64_t elem = 0; + switch ( entry.elem_kind ) + { + // a nested body, a union arm, an enum's variant and a node index each + // RESOLVE something, and a resolve that contradicts its position is + // damage this reader must still find, so they are walked + case 13: case 15: case 30: case 17: return -1; + default: elem = entry.elem_value_bits; break; + } + if ( elem < 0 ) { return -1; } + if ( entry.kind == 16 ) { elem += vocabulary.ref_bits; } // a keyed slot's own key reference + return elem; +} + +// TableMessageSkipRun steps over n elements of one fixed width in a single +// arithmetic step. A FIXED-WIDTH ELEMENT IS ARITHMETIC (§3.3), and a loop here +// would be the one superlinear thing in this form: a zero-width element under +// a count of 2^31 is six bytes of wire. +inline bool TableMessageSkipRun( TableBitReader & r, uint64_t n, int64_t width ) +{ + if ( width < 0 ) { return false; } + if ( width == 0 ) { return true; } + if ( n > (uint64_t) ( INT64_MAX / width ) ) { return false; } + return r.skip( (int64_t) ( n * (uint64_t) width ) ); +} + +inline bool TableMessageSkip( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ) +{ + switch ( entry.kind ) + { + case 0: case 32: return true; // a name, and a payload-free arm + case 30: return TableMessageSkipVariant( r, vocabulary ); + case 13: return TableMessageSkipBody( r, vocabulary, index_bits ); + case 17: return index_bits > 0 && r.skip( index_bits ); // a node index, at the width the body's node count settled + case 15: + { + uint64_t arm = 0; + if ( !r.get( arm, vocabulary.ref_bits ) ) { return false; } + if ( arm == 0 ) { return true; } + TableMessageEntry arm_entry; + if ( !TableMessageArmEntry( vocabulary, arm, arm_entry ) ) { return false; } + return TableMessageSkip( r, vocabulary, index_bits, arm_entry ); + } + case 12: + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() ) { return false; } + return r.skip( (int64_t) n * 8 ); + } + case 33: + { + // the length, NO align, then SIXTEEN bits a code unit (§3.3) + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + return r.skip( (int64_t) n * 16 ); + } + case 31: + { + // THE ESCAPE: align, a thirty-two bit L, then L bytes, opaque. It is + // the one path a later-major writer has on this form (§3.3) + uint64_t n = 0; + if ( !r.align() || !r.get( n, 32 ) ) { return false; } + return r.skip( (int64_t) n * 8 ); + } + case 14: case 16: + { + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, width ) ) { return false; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !r.align() ) { return false; } + // A RUN OF FIXED-WIDTH ELEMENTS IS ONE MULTIPLICATION (§3.3), and + // only an element whose width is its own content's is walked + const int64_t run = TableMessageElementRunBits( vocabulary, entry ); + if ( run >= 0 ) { return TableMessageSkipRun( r, n, run ); } + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 && !r.skip( vocabulary.ref_bits ) ) { return false; } + if ( !TableMessageSkipElement( r, vocabulary, index_bits, entry ) ) { return false; } + } + return true; + } + } + const int64_t width = entry.value_bits; + return width >= 0 && r.skip( width ); +} + +inline bool TableMessageSkipBody( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + if ( !TableMessageSkip( r, vocabulary, index_bits, TableVocabularyEntryAt( vocabulary, ref ) ) ) { return false; } + } +} + +// TableMessageNodeTableOpen reads the node table's opening when a body has +// one: the reserved id's reference and the count at thirty-two raw bits. A +// body whose first reference is anything else has no node table, and the +// reader is left where it was. False is damage: a reference past E, or bits +// that run out. +inline bool TableMessageNodeTableOpen( TableBitReader & r, const TableVocabulary & vocabulary, int64_t & count ) +{ + count = 0; + const int64_t at = r.offset; + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { r.offset = at; return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + if ( TableVocabularyEntryAt( vocabulary, ref ).id != kTableNodeTableFieldId ) { r.offset = at; return true; } + uint64_t n = 0; + if ( !r.get( n, 32 ) ) { return false; } + count = (int64_t) n; + return true; +} +// AnnounceMeasure is the announcement's byte count, which is a constant of the +// unit and not a walk. +inline int64_t AnnounceMeasure() { return kTableAnnounceBytes; } + +// Announce writes the announcement into the caller's buffer and answers the +// bytes written, which is exactly AnnounceMeasure's answer, or -1 when the +// buffer is too small. It allocates nothing and walks nothing. +inline int64_t Announce( uint8_t * buffer, int64_t capacity ) +{ + if ( buffer == NULL || capacity < kTableAnnounceBytes ) { return -1; } + memcpy( buffer, kTableAnnounce, (size_t) kTableAnnounceBytes ); + return kTableAnnounceBytes; +} + +// THE PRIMITIVE IS A BATCH (§3.3): a number of bodies of ONE ROOT in one +// buffer, one count and one continuous bit stream with no alignment between +// them. A single message is the batch of one. +// +// The count rides ahead of the bodies, so a writer declares it at Begin and +// End refuses a batch that wrote a different number: a count the bodies do not +// match is not a wire this writer will hand anyone. +struct TableMessageBatch +{ + TableBitWriter w; + int64_t declared = 0; + int64_t written = 0; +}; + +inline bool TableMessageBatchBegin( TableMessageBatch & batch, uint8_t * buffer, int64_t capacity, int64_t bodies ) +{ + if ( buffer == NULL || capacity < 1 || bodies < 1 || bodies > kTableMessageBatchMax ) { return false; } + buffer[0] = kTableWireMessageForm; // the FORM BYTE is read first, always + batch.w = TableBitWriter( buffer + 1, capacity - 1 ); + batch.declared = bodies; + batch.written = 0; + batch.w.put( (uint64_t) ( bodies - 1 ), 8 ); // a ranged integer over [1, 256] + return true; +} + +// TableMessageBatchEnd zero-fills to the next byte, the one alignment a batch +// spends at its end, and answers the whole batch's byte count, or -1. +inline int64_t TableMessageBatchEnd( TableMessageBatch & batch ) +{ + if ( batch.written != batch.declared || batch.w.overflow ) { return -1; } + batch.w.align(); + if ( batch.w.overflow ) { return -1; } + return 1 + batch.w.bits / 8; +} + +// TableMessageBatchBytes is a batch's byte count from its bodies' BIT count, +// which is what every MeasureMessages answers. +inline int64_t TableMessageBatchBytes( int64_t body_bits ) +{ + if ( body_bits < 0 ) { return -1; } + return 1 + ( 8 + body_bits + 7 ) / 8; +} + +// The reading half. A batch is opened once and its bodies are then read in +// order into the storage the caller sized for them: which root a batch carries +// is the APPLICATION's and never this wire's. +struct TableMessageBatchReader +{ + TableBitReader r; + const TableVocabulary * vocabulary = NULL; + TableReport * report = NULL; + int64_t remaining = 0; + // THE SINK A CALLER THAT PASSED NO REPORT WRITES INTO IS THE READER'S OWN, + // not a static: a static is shared mutable state, and two threads reading + // two batches without reports would be writing one object. LoadMessages + // already keeps its sink locally, for the same reason. + TableReport ignored; +}; + +// TableMessageBatchOpen answers the batch's body count, or -1 with the refusal +// on the report: a form byte this reader does not carry, or a body from a peer +// that never announced. +inline int64_t TableMessageBatchOpen( TableMessageBatchReader & br, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + br.report = report != NULL ? report : &br.ignored; + br.vocabulary = &vocabulary; + if ( bytes < 1 ) { br.report->malformed = true; return -1; } + if ( buffer[0] != kTableWireMessageForm ) { br.report->refused = true; br.report->reason = newer_form; return -1; } + if ( !vocabulary.announced ) { br.report->refused = true; br.report->reason = no_vocabulary; return -1; } + br.r = TableBitReader( buffer + 1, bytes - 1 ); + uint64_t count = 0; + if ( !br.r.get( count, 8 ) ) { br.report->malformed = true; return -1; } + br.remaining = (int64_t) count + 1; + return br.remaining; +} + +// TableMessageRefuseBatch is the batch's own refusal (§3.3): M above 256 on the +// write side, or above the caller's capacity on the read side. Nothing is +// written or decoded, no counter moves, and the reason names it. +inline void TableMessageRefuseBatch( TableReport * report ) +{ + if ( report == NULL ) { return; } + report->refused = true; + report->reason = batch_too_large; +} + +// TableMessageBatchClose verifies the trailing pad, and that NOTHING FOLLOWS +// IT: the batch ends at the pad to the byte boundary, and a buffer with bytes +// left over describes no batch this reader can name (§3.3). +inline bool TableMessageBatchClose( TableMessageBatchReader & br ) +{ + if ( br.remaining != 0 || !br.r.align() || br.r.offset != br.r.bits ) { br.report->malformed = true; return false; } + return true; +} + + +// An ENUM-KEYED array's storage: E.Max slots, ONE PER NAMED VARIANT, with the +// key k at index k-1 — the storage SHIFTS LEFT and nothing is stored for None. +// +// NOTHING OUTSIDE THE ARRAY NAMES ITS SIZE: the extent is derived from E::Max +// here and nowhere else, so there is no size parameter to spell and no count a +// consumer could put one out of step with. +// +// NONE IS THE NULL KEY: it names no slot, it never rides on the wire, a stored +// key of 0 is malformed, and INDEXING BY IT IS A PROGRAM ERROR IN EVERY +// CONFIGURATION — caught by operator[], which cannot see a runtime key any +// earlier, and REFUSED UNCONDITIONALLY. A KEY PAST Max IS THE SAME ERROR for +// the same reason — it names a variant this enum does not have — so the +// accessor refuses BOTH ENDS. NDEBUG does not remove the compare: +// there is NO UB PATH here in any build. ITERATION is still the surface a +// consumer of the whole array wants: begin()/end() walk every stored slot and +// yield the KEY, 1..E.Max, so a call site writes no bound, no cast, no shift +// and no None question. +template +struct TableKeyed +{ + // the extent is the enum's, derived here and named nowhere else + static constexpr int32_t kSlots = (int32_t) E::Max; + + T slots[kSlots] = {}; + + T & operator[]( E key ) + { + RefuseKey( key ); + return slots[ (int32_t) key - 1 ]; + } + const T & operator[]( E key ) const + { + RefuseKey( key ); + return slots[ (int32_t) key - 1 ]; + } + + // THE REFUSAL, and it stands in EVERY BUILD, AT BOTH ENDS. The storage + // holds one slot per NAMED variant: nothing for None below it and nothing + // above Max, so a build that skipped this compare would index one element + // BEFORE the array or past its end — undefined behavior in the + // configuration a game ships. Either key is a program error, so the + // accessor ends the program rather than reading something. The assert + // carries the message where a debugger can read it and NDEBUG removes + // that; the fatal is what stands after it. BOTH GO THROUGH THE HOOKS — + // define schema_assert and schema_fatal and this refusal lands in your + // own handler. + // + // ONE UNSIGNED COMPARE COVERS BOTH ENDS: the storage index is key - 1, and + // None's is -1, which wraps above kSlots unsigned. The cost is one + // perfectly-predicted compare, on a path that reads config. + static void RefuseKey( E key ) + { + if ( (uint32_t) ( (int32_t) key - 1 ) >= (uint32_t) kSlots ) + { + schema_assert( false && "an enum-keyed array holds one slot per named variant: None keys none, and neither does a key past Max" ); + schema_fatal(); + } + } + + // ---- iteration: keys 1..E.Max over storage 0..E.Max-1, key beside element ---- + // + // The entry is a key and a REFERENCE, handed out BY VALUE the way any + // proxy is: for ( auto [ key, element ] : keyed ) binds element to the + // reference member, so iterating fills the array as well as reads it. + // auto & [ key, element ] does NOT compile, and that is by design — a + // non-const lvalue reference cannot bind to the proxy. Write + // auto [ ... ], or auto && [ ... ] if you prefer the reference form. + // + // THE ITERATORS CARRY NO iterator_traits TYPEDEFS. They bought std::distance + // and the forward-pass algorithms for an audience that does not call them, + // and the they need is the single most expensive include the + // generated corpus had: 536 headers and 986 KB, in a header whose whole + // remaining set is 123. begin(), end() and size() need none of it. + + struct Entry { E key; T & element; }; + struct ConstEntry { E key; const T & element; }; + + struct Iterator + { + T * slots; + int32_t index; // the STORAGE index; the key it holds is index + 1 + Entry operator*() const { return Entry{ (E) ( index + 1 ), slots[index] }; } + Iterator & operator++() { index++; return *this; } + bool operator==( const Iterator & other ) const { return index == other.index; } + bool operator!=( const Iterator & other ) const { return index != other.index; } + }; + + struct ConstIterator + { + const T * slots; + int32_t index; // the STORAGE index; the key it holds is index + 1 + ConstEntry operator*() const { return ConstEntry{ (E) ( index + 1 ), slots[index] }; } + ConstIterator & operator++() { index++; return *this; } + bool operator==( const ConstIterator & other ) const { return index == other.index; } + bool operator!=( const ConstIterator & other ) const { return index != other.index; } + }; + + Iterator begin() { return Iterator{ slots, 0 }; } + Iterator end() { return Iterator{ slots, kSlots }; } + ConstIterator begin() const { return ConstIterator{ slots, 0 }; } + ConstIterator end() const { return ConstIterator{ slots, kSlots }; } +}; + +inline float table_bits_to_float( uint32_t bits ) { float f; memcpy( &f, &bits, 4 ); return f; } +inline uint32_t table_float_to_bits( float f ) { uint32_t b; memcpy( &b, &f, 4 ); return b; } +inline double table_bits_to_double( uint64_t bits ) { double d; memcpy( &d, &bits, 8 ); return d; } +inline uint64_t table_double_to_bits( double d ) { uint64_t b; memcpy( &b, &d, 8 ); return b; } + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_PRIMITIVES + +#ifndef MAPDEMO_SCHEMA_TABLE_ARENA +#define MAPDEMO_SCHEMA_TABLE_ARENA + +namespace mapdemo { + +// ---- variable-length tables: tuning constants (docs/SPEC-TABLES.md) ---- +// +// The segment size and the count multiply to exactly 2^32: the u32 reference +// is the arena's hard ceiling, and these constants saturate it rather than +// leaving address space unreachable. Slab handout costs one atomic per slab, +// so per-node allocation costs no synchronization at all. + +static const uint32_t kTableSegmentBits = 22; // 4 MiB segments +static const uint32_t kTableSegmentSize = 1u << kTableSegmentBits; +static const uint32_t kTableSegmentMask = kTableSegmentSize - 1u; +static const uint32_t kTableMaxSegments = 1u << ( 32 - kTableSegmentBits ); // 1024 -> 4 GiB +static const uint32_t kTableSlabBytes = 64u * 1024u; // one atomic per slab +static const uint32_t kTableAlign = 8; // every node starts 8-aligned +static const uint32_t kTableAllocFailed = 0xFFFFFFFFu; + +// ---- THE CALLER'S ALLOCATOR (docs/SPEC-TABLES.md §6.5) ---- +// +// Every allocation the variable-length runtime makes goes through one of +// these — the arena's segments, the pack walk's identity map, the numbering's +// entry array, the packed region, and the tool path's node directory. There is +// no other call to the C library on this path, so a counting allocator sees +// every byte and a game's own heap can own all of it. +// +// It is the shape TableBlockAllocator already has (§19.1): two function +// pointers and a context the caller carries. What it adds is a CONTRACT ON +// alloc — the bytes come back ZEROED. Lock copies whole nodes, PADDING +// INCLUDED, so anything left uninitialized reaches a packed region; the default +// pair reaches that through calloc, which costs nothing measurable because a +// fresh segment is untouched pages either way. +struct TableAllocator +{ + void * ( *alloc )( void * context, int64_t bytes ); // ZEROED bytes, NULL on failure + void ( *free )( void * context, void * pointer ); + void * context; +}; + +// The default pair, and it is the one every entry point takes when the caller +// names none. It calls schema_allocate / schema_release, so a program with its +// own C-library replacement can move the floor without writing a struct at all. +inline void * table_default_alloc( void * context, int64_t bytes ) { (void) context; return schema_allocate( bytes ); } +inline void table_default_free( void * context, void * pointer ) { (void) context; schema_release( pointer ); } + +inline TableAllocator TableDefaultAllocator() +{ + TableAllocator allocator; + allocator.alloc = table_default_alloc; + allocator.free = table_default_free; + allocator.context = NULL; + return allocator; +} + +// ---- TableRef: a relocatable reference (never a machine pointer) ---- +// +// Two encodings, one slot, and the FORM says which is in force: +// +// in the arena — the node's arena offset (segment index in the high bits) +// in a region — the SELF-RELATIVE byte delta from this slot's own address, +// so a deref is one add, needs no base pointer, and a whole +// region relocates by memcpy with zero fix-up +// +// 0 is null in both, and a slot can never name the node that contains it, so +// zero names nothing real in either form. +// +// A REGION DELTA HAS NO REQUIRED SIGN (§6.3). A region is packed depth-first, +// so a node's FIRST reference points forward; every LATER reference to that +// same node points BACK at the one body it already has, which is exactly what +// makes one node one node in a region. Sharing and a back-reference are the +// same fact, and nothing validates a reference by its sign. +// +// IT IS EIGHT BYTES, SIGNED, so ONE REGION REACHES EVERYTHING (§6.3, §7): a +// four-byte slot bounded a region at 2 GiB, and the scale a cook exists for is +// *"100mbs or many gigabytes of data in Assets.bin"*. +struct TableRef +{ + int64_t value = 0; + bool null() const { return value == 0; } +}; + +// TableSlot is what Alloc hands back: usable as the node pointer (write +// fields through it) AND as the reference to store in a pointer field. +template struct TableSlot +{ + T * ptr = NULL; + TableRef ref; + T * operator->() const { return ptr; } + T & operator*() const { return *ptr; } + operator T *() const { return ptr; } + operator TableRef() const { return ref; } + bool null() const { return ptr == NULL; } +}; + +inline uint32_t TableAlignUp( uint32_t bytes ) { return ( bytes + kTableAlign - 1 ) & ~( kTableAlign - 1 ); } +inline int64_t TableAlignUp64( int64_t bytes ) { return ( bytes + kTableAlign - 1 ) & ~( int64_t( kTableAlign ) - 1 ); } + +// ---- a BYTE BUFFER's node (docs/SPEC-TABLES.md §2.5, §6.3) ---- +// +// A *bytes or *string slot is a TableRef like every pointer slot, and it names +// a BLOB NODE: this eight-byte header and then the bytes, at offset eight so +// the data is eight-aligned. A *string blob carries one more zero byte after +// its data, so a region hands back a C string with no copy. The node's extent +// is the header plus its bytes, rounded to the arena's alignment like every +// node's; on the wire it is a record whose body is the bytes (§3.1). +struct TableBlob +{ + uint32_t length; + uint32_t zero; +}; + +static const int64_t kTableBlobHeader = 8; // length (u32), then four zero bytes +static const int64_t kTableBlobMaxLength = 0xFFFFFFFF; // a record's length is a u32 (§3.1) + +// the node's storage: the header, the bytes, a string's terminator, rounded +// to the arena's alignment like every node +inline int64_t TableBlobStorage( int64_t length, bool terminated ) +{ + return TableAlignUp64( kTableBlobHeader + length + ( terminated ? 1 : 0 ) ); +} + +// What a read answers: a pointer INTO the region and the length, NULL and +// zero for a null slot. Off a locked region, a loaded one or an opened cook +// the pointer is one add from the slot, and nothing is copied. +struct TableBytesView +{ + const uint8_t * data; + int64_t length; +}; + +struct TableStringView +{ + const char * data; // zero-terminated + int64_t length; +}; + +// What AllocBytes and AllocString hand back: the bytes to write through, the +// length asked for, and the reference to store in the slot — the three +// answers TableSlot gives for a table node. +struct TableBytesSlot +{ + uint8_t * data = NULL; + int64_t length = 0; + TableRef ref; + bool null() const { return data == NULL; } + operator TableRef() const { return ref; } +}; + +struct TableStringSlot +{ + char * data = NULL; // room for length bytes and the terminator, already zero + int64_t length = 0; + TableRef ref; + bool null() const { return data == NULL; } + operator TableRef() const { return ref; } +}; + +// ---- the arena: segmented, slab-handed, lock-free by ownership ---- +// +// Allocation is thread-local inside a worker's slab — no atomics on the node +// path. A worker takes its next slab with ONE compare-exchange, and a new +// segment is published with one more. Nothing ever moves: a segment, once +// allocated, lives untouched until the arena is torn down, so a T* obtained +// from Alloc stays valid while other workers allocate, and an offset stays +// correct while the arena grows. +// +// The model this DELIBERATELY refuses: one buffer under a lock, grown by +// realloc. A realloc moves the buffer under workers mid-write; offsets fix +// identity but not the raw references already resolved from them, and the +// resulting corruption is invisible until much later. Segments never move, so +// that bug class cannot be written here. +// +// Slack: at most one slab tail per worker plus one slab per segment (a slab +// that will not fit is skipped rather than split), i.e. under 2% of a segment +// plus threads x 64 KiB. That is the price of never synchronizing per node. +struct TableArena +{ + std::atomic segments[ kTableMaxSegments ]; + std::atomic cursor; // (segment << kTableSegmentBits) | bytes handed out + bool locked = false; // MONOTONIC: Lock() is one-way, there is no unlock + // THE ARENA CARRIES ITS OWN, so everything downstream of a builder — + // segments, pack map, numbering, region, node directory — allocates through + // the one pair the caller named, with nothing to thread by hand. + TableAllocator allocator; +}; + +inline void TableArenaInit( TableArena & arena, TableAllocator allocator ) +{ + for ( uint32_t i = 0; i < kTableMaxSegments; i++ ) + { + arena.segments[i].store( NULL, std::memory_order_relaxed ); + } + arena.cursor.store( 0, std::memory_order_relaxed ); + arena.locked = false; + arena.allocator = allocator; +} + +inline void TableArenaShutdown( TableArena & arena ) +{ + for ( uint32_t i = 0; i < kTableMaxSegments; i++ ) + { + uint8_t * segment = arena.segments[i].exchange( NULL, std::memory_order_acq_rel ); + if ( segment != NULL ) { arena.allocator.free( arena.allocator.context, segment ); } + } + arena.cursor.store( 0, std::memory_order_relaxed ); +} + +// one L1 load plus an add: the segment table is 8 KiB and stays hot +inline uint8_t * TableArenaAt( const TableArena & arena, uint32_t offset ) +{ + return arena.segments[ offset >> kTableSegmentBits ].load( std::memory_order_relaxed ) + ( offset & kTableSegmentMask ); +} + +// TableArenaGrabSlab hands one worker its next private slab. Returns +// kTableAllocFailed when the arena's address space or the allocator is +// exhausted — a loud refusal, never a silent smaller slab. +inline uint32_t TableArenaGrabSlab( TableArena & arena ) +{ + for ( ;; ) + { + uint32_t cursor = arena.cursor.load( std::memory_order_acquire ); + uint32_t segment = cursor >> kTableSegmentBits; + uint32_t used = cursor & kTableSegmentMask; + // strictly less: a slab is never split across segments, and the tail + // is the documented slack + if ( used + kTableSlabBytes < kTableSegmentSize ) + { + if ( arena.segments[segment].load( std::memory_order_acquire ) == NULL ) + { + // THE SEGMENT COMES BACK ZEROED, which is the allocator's + // contract and not an extra pass here: Lock copies whole nodes, + // PADDING INCLUDED, so anything uninitialized reaches a packed + // region. Value-initializing a node with placement new zeroes + // its MEMBERS and not its padding, so the zeroing has to happen + // at the segment or not at all. It costs nothing measurable: a + // fresh segment is untouched pages either way, and the default + // pair's calloc has the kernel hand them over zeroed. + uint8_t * memory = (uint8_t *) arena.allocator.alloc( arena.allocator.context, (int64_t) kTableSegmentSize ); + if ( memory == NULL ) { return kTableAllocFailed; } + uint8_t * expected = NULL; + if ( !arena.segments[segment].compare_exchange_strong( expected, memory, std::memory_order_acq_rel ) ) + { + // another worker published this segment first + arena.allocator.free( arena.allocator.context, memory ); + } + } + if ( arena.cursor.compare_exchange_weak( cursor, cursor + kTableSlabBytes, std::memory_order_acq_rel ) ) + { + return ( segment << kTableSegmentBits ) | used; + } + continue; + } + uint32_t next_segment = segment + 1; + if ( next_segment >= kTableMaxSegments ) { return kTableAllocFailed; } // 4 GiB: the u32 reference's ceiling + arena.cursor.compare_exchange_weak( cursor, next_segment << kTableSegmentBits, std::memory_order_acq_rel ); + } +} + +// TableArenaGrabSpan reserves a SPAN of the arena's address space for one node +// larger than a slab — a BYTE BUFFER of any size (docs/SPEC-TABLES.md §2.5) — +// and allocates it as one contiguous block. It takes whole segment indices +// from the cursor, starting at the index after the cursor's so nothing else +// is ever handed out inside the span, and publishes the block under the first +// of them; the indices the span covers past that one stay NULL, which is +// enough, because only a node's START is ever resolved through the segment +// table and a blob's bytes follow its header inside the one allocation. The +// unused tail of the segment the cursor was in is slack, like a slab tail. +// Returns kTableAllocFailed when the address space or the allocator is +// exhausted — a loud refusal, never a smaller blob. +inline uint32_t TableArenaGrabSpan( TableArena & arena, int64_t bytes ) +{ + if ( bytes <= 0 || bytes > ( (int64_t) kTableMaxSegments - 2 ) * (int64_t) kTableSegmentSize ) { return kTableAllocFailed; } + const uint32_t spanned = (uint32_t) ( ( bytes + kTableSegmentSize - 1 ) >> kTableSegmentBits ); + for ( ;; ) + { + uint32_t cursor = arena.cursor.load( std::memory_order_acquire ); + uint32_t start = ( cursor >> kTableSegmentBits ) + 1; + if ( start + spanned >= kTableMaxSegments ) { return kTableAllocFailed; } // 4 GiB: the u32 reference's ceiling + uint32_t next = ( start + spanned ) << kTableSegmentBits; + if ( !arena.cursor.compare_exchange_weak( cursor, next, std::memory_order_acq_rel ) ) { continue; } + // the span is this worker's now: nothing else can publish under its + // first index, so a plain store suffices, and the block comes back + // ZEROED like every segment — the blob's bytes and its tail are zeros + // until written + uint8_t * memory = (uint8_t *) arena.allocator.alloc( arena.allocator.context, bytes ); + if ( memory == NULL ) { return kTableAllocFailed; } + arena.segments[start].store( memory, std::memory_order_release ); + return start << kTableSegmentBits; + } +} + +// ---- TableWorker: one thread's allocation front ---- +// +// The threading contract, stated plainly: +// * Alloc on YOUR OWN worker is safe concurrently with any other worker's. +// No locks, no atomics per node. +// * Writing fields of a node ANOTHER worker allocated is your own +// synchronization problem — this runtime does not arbitrate it. +// * Lock and Save are single-threaded: call them after the workers have +// joined. +struct TableWorker +{ + TableArena * arena = NULL; + uint32_t next = 0; + uint32_t end = 0; + + template TableSlot Alloc() + { + static_assert( alignof( T ) <= kTableAlign, "a table node's alignment must fit the arena's" ); + TableSlot slot; + if ( arena == NULL || arena->locked ) { return slot; } + uint32_t bytes = TableAlignUp( (uint32_t) sizeof( T ) ); + if ( bytes > kTableSlabBytes ) { return slot; } // a node larger than a slab: refused, never split + if ( end == 0 || next + bytes > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return slot; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + uint32_t at = next; + next += bytes; + // A NODE IS BORN IN TWO HALVES: start its lifetime in the raw + // storage, then write the declared defaults ONE MEMBER AT A TIME. + // + // It is "T", not "T{}". Value-initialising the whole aggregate says + // the same thing and costs cl O(BYTES) TO COMPILE — it expands element + // by element in its front end — while both halves here cost + // O(declarations). The slab cap below refuses a large node at RUN + // TIME and bounds nothing at compile time: the cost is paid by + // whatever T a caller instantiates this with. + // Padding is not the difference: value-initialisation zeroes MEMBERS + // and not padding either way, which is why the segment is calloc'd. + // + // TableReset is an OVERLOAD SET, one per closure member, reached from + // this template by argument-dependent lookup on T's own namespace — + // Alloc is a template and cannot spell Reset. + // + // The reset is here because ONE DEFINITION SAYS WHAT THE DECLARED + // DEFAULTS ARE, and it is Reset. Default-initialisation lands on + // the same values today, because a member with a non-zero default + // carries a member initializer that says so — but that is the class + // definition agreeing with Reset, not the arena reading it, and #320's + // fix was itself a pass that MOVED initialisation between the two. + // The arena reads the definition. + slot.ptr = new ( TableArenaAt( *arena, at ) ) T; + TableReset( *slot.ptr ); + slot.ref.value = at; + return slot; + } + + // Alloc a BYTE BUFFER's node of exactly length bytes (docs/SPEC-TABLES.md + // §2.5): the blob header and its bytes, zeroed, in this thread's slab when + // it fits and in a span of the arena's own when it does not. NULL is the + // arena locked, a length below zero or past a record's u32, or the + // allocator refusing. The offset comes back for the reference. + TableBlob * AllocBlob( int64_t length, bool terminated, uint32_t & at ) + { + at = 0; + if ( arena == NULL || arena->locked ) { return NULL; } + if ( length < 0 || length > kTableBlobMaxLength ) { return NULL; } + const int64_t bytes = TableBlobStorage( length, terminated ); + if ( bytes > (int64_t) kTableSlabBytes ) + { + at = TableArenaGrabSpan( *arena, bytes ); + if ( at == kTableAllocFailed ) { at = 0; return NULL; } + } + else + { + if ( end == 0 || next + (uint32_t) bytes > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return NULL; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + at = next; + next += (uint32_t) bytes; + } + TableBlob * blob = (TableBlob *) TableArenaAt( *arena, at ); + blob->length = (uint32_t) length; // the bytes after it are the segment's zeros + blob->zero = 0; + return blob; + } + + // RAW, ZEROED storage of the bytes asked for, at the alignment asked for: a MAP's or a LIST's builder + // head and its segments (docs/SPEC-TABLES.md §2.8, §2.9). It is not a node: it carries + // no type id, takes no index and has no Reset, so it goes through the same + // slab and span the blob path uses rather than through Alloc. + uint8_t * AllocRaw( int64_t bytes, int64_t align, uint32_t & at ) + { + at = 0; + if ( arena == NULL || arena->locked ) { return NULL; } + if ( bytes <= 0 || align > (int64_t) kTableAlign ) { return NULL; } + const int64_t rounded = TableAlignUp64( bytes ); + if ( rounded > (int64_t) kTableSlabBytes ) + { + at = TableArenaGrabSpan( *arena, rounded ); + if ( at == kTableAllocFailed ) { at = 0; return NULL; } + return TableArenaAt( *arena, at ); + } + if ( end == 0 || next + (uint32_t) rounded > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return NULL; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + at = next; + next += (uint32_t) rounded; + return TableArenaAt( *arena, at ); // the segment came back zeroed + } + // a *bytes node: the bytes to write through, and the reference to store + TableBytesSlot AllocBytes( int64_t length ) + { + TableBytesSlot slot; + uint32_t at = 0; + TableBlob * blob = AllocBlob( length, false, at ); + if ( blob == NULL ) { return slot; } + slot.data = (uint8_t *) ( blob + 1 ); + slot.length = length; + slot.ref.value = at; + return slot; + } + + // a *string node: room for length bytes and the zero byte after them + TableStringSlot AllocString( int64_t length ) + { + TableStringSlot slot; + uint32_t at = 0; + TableBlob * blob = AllocBlob( length, true, at ); + if ( blob == NULL ) { return slot; } + slot.data = (char *) ( blob + 1 ); + slot.length = length; + slot.ref.value = at; + return slot; + } +}; + +// ---- TablePackMap: the pack walk's identity map (docs/SPEC-TABLES.md §3.1, §6.2) ---- +// +// ONE ENTRY PER REACHABLE NODE, and that map IS identity: a node must know +// where it landed to be named a second time, so Lock packs a shared node ONCE +// and every later reference resolves to the one body it already has. That is +// the same first-visit numbering the wire uses, so the pack order and the node +// order are one order. +// +// COLOURING AN ENTRY WHILE ITS DESCENT IS OPEN COSTS ONE BIT, and it is what +// makes a data cycle free to refuse: a reference to an entry still open is a +// cycle, and Lock returns failure rather than recursing away. The ROOT's entry +// is open for the whole walk. +// +// The map is proportional to NODES, never to bytes, and it lives on the +// AUTHORING side, where §6.5 licenses allocation. Nothing on the reading path +// ever builds one. +struct TablePackEntry +{ + const void * key; // the node's address in the graph being packed + int64_t offset; // where that node landed in the region + uint8_t open; // its descent is still open: a reference here is a cycle +}; + +struct TablePackMap +{ + TablePackEntry * entries = NULL; + int64_t capacity = 0; // a power of two, or zero while empty + int64_t count = 0; + TableAllocator allocator; // the caller's, carried from the walk that built it +}; + +inline void TablePackMapInit( TablePackMap & map, TableAllocator allocator ) +{ + map.entries = NULL; + map.capacity = 0; + map.count = 0; + map.allocator = allocator; +} + +inline void TablePackMapShutdown( TablePackMap & map ) +{ + map.allocator.free( map.allocator.context, map.entries ); + TablePackMapInit( map, map.allocator ); +} + +// The two walks behind Lock re-derive the SAME map from the same graph — the +// numbering is never carried between them (§3.1) — so the second starts from +// an empty map and keeps the capacity the first paid for. +inline void TablePackMapReset( TablePackMap & map ) +{ + if ( map.entries != NULL ) { memset( map.entries, 0, (size_t) map.capacity * sizeof( TablePackEntry ) ); } + map.count = 0; +} + +// open addressing, linear probing, a multiply-shift hash over the address: a +// node key is a pointer and its low bits are alignment, so the low bits alone +// would collide on every node of one type +inline int64_t TablePackMapSlot( const TablePackMap & map, const void * key ) +{ + uint64_t hash = (uint64_t) (uintptr_t) key; + hash *= 0x9E3779B97F4A7C15ull; + hash ^= hash >> 29; + int64_t mask = map.capacity - 1; + int64_t at = (int64_t) ( hash & (uint64_t) mask ); + while ( map.entries[at].key != NULL && map.entries[at].key != key ) + { + at = ( at + 1 ) & mask; + } + return at; +} + +inline TablePackEntry * TablePackMapFind( TablePackMap & map, const void * key ) +{ + if ( map.capacity == 0 ) { return NULL; } + TablePackEntry * entry = &map.entries[ TablePackMapSlot( map, key ) ]; + return entry->key == key ? entry : NULL; +} + +// QUADRUPLING, not doubling, and the reason is measured: growth rehashes every +// entry, and on a graph of 131,071 nodes the doubling schedule spent 45% of +// Lock in rehashing alone. Quadrupling from 1024 buys 1.35x on that graph and +// keeps the map NODE-proportional (§6.2) — under 128 bytes a node at its +// worst, right after a grow, and about 64 on average. +inline bool TablePackMapGrow( TablePackMap & map ) +{ + TablePackMap grown; + grown.allocator = map.allocator; + grown.capacity = map.capacity != 0 ? map.capacity * 4 : 1024; + grown.entries = (TablePackEntry *) map.allocator.alloc( map.allocator.context, grown.capacity * (int64_t) sizeof( TablePackEntry ) ); + if ( grown.entries == NULL ) { return false; } + for ( int64_t i = 0; i < map.capacity; i++ ) + { + if ( map.entries[i].key == NULL ) { continue; } + grown.entries[ TablePackMapSlot( grown, map.entries[i].key ) ] = map.entries[i]; + grown.count++; + } + map.allocator.free( map.allocator.context, map.entries ); + map = grown; + return true; +} + +// REACH a node: one probe answers both questions the walk has. A true "taken" +// says this is a FIRST visit, and the entry is now the node's, coloured open +// at "offset"; otherwise the entry is the one the node already has, and its +// open bit says cycle or sharing. NULL is an allocation failure, and it is a +// refusal like any other: Lock fails rather than packing a graph it cannot +// track. +// +// It is one call and not a find followed by an insert because the walk asks +// this question twice per node — once to measure, once to pack — and every +// probe is a miss into a table larger than L2. +inline TablePackEntry * TablePackMapReach( TablePackMap & map, const void * key, int64_t offset, bool & taken, int64_t & slot ) +{ + if ( ( map.count + 1 ) * 4 >= map.capacity * 3 ) // keep the load factor under three quarters + { + if ( !TablePackMapGrow( map ) ) { return NULL; } + } + slot = TablePackMapSlot( map, key ); + TablePackEntry * entry = &map.entries[slot]; + taken = entry->key != key; // an empty slot is a first visit; the key is never NULL + if ( taken ) + { + entry->key = key; + entry->offset = offset; + entry->open = 1; + map.count++; + } + return entry; +} + +// The descent finished: the node keeps its entry — identity outlives the +// descent — and stops being a cycle. The "hint" is the slot Reach returned, and it +// is checked against the key rather than trusted, so a rehash between the two +// costs a second probe instead of correctness. +inline void TablePackMapClose( TablePackMap & map, const void * key, int64_t hint ) +{ + if ( hint >= 0 && hint < map.capacity && map.entries[hint].key == key ) + { + map.entries[hint].open = 0; + return; + } + TablePackEntry * entry = TablePackMapFind( map, key ); + if ( entry != NULL ) { entry->open = 0; } +} + +// ---- resolution contexts: which encoding a walk is reading ---- + +struct TableArenaCtx { const TableArena * arena; }; +struct TableRegionCtx {}; + +// ---- a BYTE BUFFER's resolution (docs/SPEC-TABLES.md §2.5, §6.3) ---- +// +// The same two encodings a table pointer has, resolved the same way: a +// self-relative delta in a region — one add, no base — and an arena offset +// while the builder is mutable. The blob is reached through its header, and a +// view is the header plus eight and the header's first word. Nothing here +// allocates and nothing copies: off a locked region, a loaded one or an +// opened cook the view points INTO the region. +inline const TableBlob * TableBlobAt( const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) ( (const uint8_t *) &ref + ref.value ) : NULL; +} +inline const TableBlob * TableBlobAt( const TableRegionCtx &, const TableRef & ref ) { return TableBlobAt( ref ); } +inline const TableBlob * TableBlobAt( const TableArenaCtx & ctx, const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) TableArenaAt( *ctx.arena, (uint32_t) ref.value ) : NULL; +} +inline const TableBlob * TableBlobAt( const TableArena & arena, const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) TableArenaAt( arena, (uint32_t) ref.value ) : NULL; +} + +inline TableBytesView TableBytesViewOf( const TableBlob * blob ) +{ + TableBytesView view = { NULL, 0 }; + if ( blob != NULL ) { view.data = (const uint8_t *) ( blob + 1 ); view.length = (int64_t) blob->length; } + return view; +} +inline TableStringView TableStringViewOf( const TableBlob * blob ) +{ + TableStringView view = { NULL, 0 }; + if ( blob != NULL ) { view.data = (const char *) ( blob + 1 ); view.length = (int64_t) blob->length; } + return view; +} + +// the const form's hot path: one add, no base +inline TableBytesView TableBytesAt( const TableRef & ref ) { return TableBytesViewOf( TableBlobAt( ref ) ); } +inline TableStringView TableStringAt( const TableRef & ref ) { return TableStringViewOf( TableBlobAt( ref ) ); } +// and the context forms a walk uses: a region context, an arena context, or +// the arena itself while the builder is mutable +template inline TableBytesView TableBytesAt( const Ctx & ctx, const TableRef & ref ) { return TableBytesViewOf( TableBlobAt( ctx, ref ) ); } +template inline TableStringView TableStringAt( const Ctx & ctx, const TableRef & ref ) { return TableStringViewOf( TableBlobAt( ctx, ref ) ); } + +// allocate a blob in the arena and point the slot at it; the slot holds the +// arena offset, as every slot does while the builder is mutable +inline uint8_t * TableBytesEmplace( TableWorker & worker, TableRef & slot, int64_t length ) +{ + TableBytesSlot allocated = worker.AllocBytes( length ); + slot = allocated.ref; + return allocated.data; +} +// the text is copied in when one is given; a NULL text leaves the zeros for +// the caller to fill +inline char * TableStringEmplace( TableWorker & worker, TableRef & slot, const char * text, int64_t length ) +{ + TableStringSlot allocated = worker.AllocString( length ); + slot = allocated.ref; + if ( allocated.data != NULL && text != NULL && length > 0 ) { memcpy( allocated.data, text, (size_t) length ); } + return allocated.data; +} + +// ---- the FLAT NODE TABLE (docs/SPEC-TABLES.md §3.1) ---- +// +// A pointered save writes every reachable node ONCE, into a node table, and a +// pointer field rides as an INDEX into it under kind 17. The encoding is +// flat: no pointer edge is a nesting level, so a chain's length is not a depth, +// and two references to one node are one node. +// +// THE FIELD RIDES ONCE: an L with sixty-four bits of capability frames a +// numbering of any size, so the whole numbering is one contiguous payload and a +// save's node bodies have no aggregate ceiling. + +static const uint64_t kTableNodeIndexNull = 0; // absence and null are one value +static const uint64_t kTableNodeIndexRoot = 1; // the body that hosts the table + +// The not-materialized sentinel (§6.3): a record whose type id this build could +// not name. Distinct from every real offset including the root's 0, so an index +// resolving through it yields NULL and can never fabricate the root. +static const uint64_t kTableNodeAbsent = 0xFFFFFFFFFFFFFFFFull; + +// What a node's storage answers when the FRAMING ITSELF is refused rather than +// merely unnameable: a count its L cannot carry, one above the int32 cap, or a +// blob past the size cap (docs/SPEC-TABLES.md §3.1, §6.5). An unnameable type +// id commands no storage and keeps its index. This one makes the whole measure +// answer -1 with its reason. +static const int64_t kTableNodeRefused = -2; + +// ---- the numbering, on the SAVE side ---- +// +// One entry per reachable node in FIRST-VISIT order, so entry k is node index +// k + 2. The two thunks are what let one loop write a table of mixed types: the +// numbering walk knows each target's type STATICALLY at the site it numbers it, +// so it stores the instantiation there and the loop never asks what a node is. +struct TableNumbering; + +struct TableNodeEntry +{ + const void * node; + uint64_t type_id; + // the type id's MESSAGE-FORM SLOT (docs/SPEC-TABLES.md §3.3), stored where + // the numbering walk stores the id itself and for the same reason: the + // target's type is known STATICALLY at the site that numbers it, so a + // form 2 save reads the slot out of the entry instead of looking an id up. + // Every pointer target's type id is an entry of the announcement, which is + // what makes the slot a compile-time fact of a POINTERED message too. + uint64_t type_slot; + int64_t ( * measure )( const void * ctx, const TableNumbering & numbering, TableIds & ids, const void * node ); + bool ( * save )( const void * ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const void * node ); + // the same two over the MESSAGE FORM (docs/SPEC-TABLES.md §3.3): a bitpacked + // body at a bit position, its pointer indices at the width the node count + // settled + int64_t ( * message_measure )( const void * ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const void * node ); + bool ( * message_save )( const void * ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const void * node ); +}; + +struct TableNumbering +{ + TablePackMap seen; // node -> index; the ROOT is index 1, open for the whole walk + TableNodeEntry * entries = NULL; + int64_t count = 0; + int64_t capacity = 0; +}; + +// The numbering allocates through the map's pair rather than carrying a second +// copy of it: one numbering is one walk, and a walk has one allocator. +inline void TableNumberingInit( TableNumbering & n, TableAllocator allocator ) +{ + TablePackMapInit( n.seen, allocator ); + n.entries = NULL; + n.count = 0; + n.capacity = 0; +} + +inline void TableNumberingShutdown( TableNumbering & n ) +{ + TableAllocator allocator = n.seen.allocator; + TablePackMapShutdown( n.seen ); + allocator.free( allocator.context, n.entries ); + n.entries = NULL; + n.count = 0; + n.capacity = 0; +} + +// The index a numbered node was given, for the save that writes it into a +// pointer slot. False means the two walks disagree about the graph, which is a +// refusal and never a guess. +inline bool TableNumberingIndex( const TableNumbering & n, const void * node, uint64_t & index ) +{ + if ( n.seen.capacity == 0 ) { return false; } + const TablePackEntry & entry = n.seen.entries[ TablePackMapSlot( n.seen, node ) ]; + if ( entry.key != node ) { return false; } + index = (uint64_t) entry.offset; + return true; +} + +inline bool TableNumberingAppend( TableNumbering & n, const TableNodeEntry & entry ) +{ + if ( n.count == n.capacity ) + { + // GROW BY COPY, never by realloc: the allocator hook is a PAIR, and a + // game's heap is not required to have a resize primitive at all. The + // schedule quadruples, so the copying is amortized to a constant per + // entry and the growth is the same growth it always was. + int64_t capacity = n.capacity != 0 ? n.capacity * 4 : 256; + TableAllocator allocator = n.seen.allocator; + TableNodeEntry * grown = (TableNodeEntry *) allocator.alloc( allocator.context, capacity * (int64_t) sizeof( TableNodeEntry ) ); + if ( grown == NULL ) { return false; } + if ( n.entries != NULL ) + { + memcpy( grown, n.entries, (size_t) n.count * sizeof( TableNodeEntry ) ); + allocator.free( allocator.context, n.entries ); + } + n.entries = grown; + n.capacity = capacity; + } + n.entries[n.count++] = entry; + return true; +} + +// The thunks the numbering stores. Each resolves to the closure member's own +// MeasureBody / SaveBodyFields through an overload set in the member's DECLARING +// file, reached by argument-dependent lookup at instantiation — the same bridge +// the arena's TableReset uses, and the reason a numbering may span the files of +// one unit without any file naming another's members. +template +inline int64_t TableNodeMeasureThunk( const void * ctx, const TableNumbering & numbering, TableIds & ids, const void * node ) +{ + return TableNodeMeasure( *(const Ctx *) ctx, numbering, ids, *(const T *) node ); +} + +template +inline bool TableNodeSaveThunk( const void * ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const void * node ) +{ + return TableNodeSave( *(const Ctx *) ctx, numbering, w, ids, *(const T *) node ); +} + +template +inline int64_t TableNodeMessageMeasureThunk( const void * ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const void * node ) +{ + return TableNodeMessageMeasure( *(const Ctx *) ctx, numbering, index_bits, at, *(const T *) node ); +} + +template +inline bool TableNodeMessageSaveThunk( const void * ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const void * node ) +{ + return TableNodeMessageSave( *(const Ctx *) ctx, numbering, index_bits, w, *(const T *) node ); +} +// ---- a BYTE BUFFER's record (docs/SPEC-TABLES.md §2.5, §3.1) ---- +// +// A blob rides as a node record under one of two RESERVED type ids — the fold +// a table's name takes, over the keywords "bytes" and "string", which no table +// can be named — with the bytes as its body and nothing framed inside. These +// two thunks are what the numbering stores for a blob, as it stores a +// member's codec for a table: the length, and the bytes verbatim. +static const uint64_t kTableBytesTypeId = 0x2f2ec0474f1c4fe4ull; // fnv1a64( "bytes" ) +static const uint64_t kTableStringTypeId = 0x704be0d8faaffc58ull; // fnv1a64( "string" ) + +template +inline int64_t TableBlobMeasureThunk( const void *, const TableNumbering &, TableIds &, const void * node ) +{ + return (int64_t) ( (const TableBlob *) node )->length; +} + +template +inline bool TableBlobSaveThunk( const void *, const TableNumbering &, TableWriter & w, TableIds &, const void * node ) +{ + const TableBlob * blob = (const TableBlob *) node; + w.raw( (const void *) ( blob + 1 ), (int64_t) blob->length ); + return true; +} + +// and the same two on the MESSAGE FORM (§3.3): a blob record is its length at +// thirty-two raw bits, an ALIGN, then the bytes verbatim +template +inline int64_t TableBlobMessageMeasureThunk( const void *, const TableNumbering &, int64_t, int64_t at, const void * node ) +{ + const int64_t length = (int64_t) ( (const TableBlob *) node )->length; + return 32 + TableAlignBits( at + 32 ) + length * 8; +} + +template +inline bool TableBlobMessageSaveThunk( const void *, const TableNumbering &, int64_t, TableBitWriter & w, const void * node ) +{ + const TableBlob * blob = (const TableBlob *) node; + w.put( (uint64_t) blob->length, 32 ); + w.align(); + w.putbytes( (const uint8_t *) ( blob + 1 ), (int64_t) blob->length ); + return !w.overflow; +} +// TableNodeTableMeasure and TableNodeTableSave are the framing, and they are +// ONE fill rule written twice — measure derives it from the graph and save +// derives the same one, which is what makes measure == save hold across a +// pointer graph (§3.1). +// +// The field rides ONCE, under the reserved id, kind 12: the payload opens with +// the count and then carries the records back to back, each a type id +// REFERENCE, a length and a body. The reserved id is interned BEFORE the +// records, and a record's type id before its body, which is the first-use order +// the trailer is written in (§3). +template +inline int64_t TableNodeTablePayload( const Ctx & ctx, TableIds & ids, const TableNumbering & n ) +{ + int64_t payload = TableLebBytes( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + payload += TableLebBytes( ids.ref( n.entries[k].type_id ) ); + const int64_t body = n.entries[k].measure( (const void *) &ctx, n, ids, n.entries[k].node ); + if ( body < 0 ) { return -1; } + payload += TableLebBytes( (uint64_t) body ) + body; + } + return payload; +} + +template +inline int64_t TableNodeTableMeasure( const Ctx & ctx, TableIds & ids, const TableNumbering & n ) +{ + if ( n.count == 0 ) { return 0; } // a root that reaches no nodes writes none of them + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayload( ctx, ids, n ); + if ( payload < 0 ) { return -1; } + return TableLebBytes( ref ) + 1 + TableLebBytes( (uint64_t) payload ) + payload; +} + +template +inline bool TableNodeTableSave( const Ctx & ctx, TableWriter & w, TableIds & ids, const TableNumbering & n ) +{ + if ( n.count == 0 ) { return true; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayload( ctx, ids, n ); + if ( payload < 0 ) { return false; } + w.putleb( ref ); + w.put8( 12 ); // kind 12 is the opaque byte payload: a reader that cannot name the id skips by L + w.putleb( (uint64_t) payload ); + w.putleb( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.putleb( ids.ref( n.entries[k].type_id ) ); + const int64_t body = n.entries[k].measure( (const void *) &ctx, n, ids, n.entries[k].node ); + if ( body < 0 ) { return false; } + w.putleb( (uint64_t) body ); + if ( !n.entries[k].save( (const void *) &ctx, n, w, ids, n.entries[k].node ) ) { return false; } + } + return true; +} + +// ---- the numbering, on the LOAD side: a region's NODE DIRECTORY (§6.3) ---- +// +// The wire's numbering made resident: one entry per numbered node, in index +// order, position i describing node index i + 1 — so position 0 is the ROOT at +// offset 0. It is ATTRIBUTION, and attribution is separable: nothing that reads +// a structure touches it, a deref is one add on a self-relative offset, and a +// caller may release it once Load returns. +struct TableNodeDirEntry +{ + uint64_t offset; + uint64_t type_id; +}; + +// the node's extent cursor, defined with the extent runtime (docs/SPEC-TABLES.md +// §2.8, §2.9); the node map names it only through a pointer. +struct TableExtentCarve; + +// TableNodeMap is what a pointer slot resolves through while a body decodes. +struct TableNodeMap +{ + uint8_t * base = NULL; + const TableNodeDirEntry * entries = NULL; + int64_t count = 0; // the ROOT's entry included, so it is records + 1 + bool good = false; // the node table read whole; a numbering that failed resolves nothing + // WHERE THE NODES LIVE, and therefore what a resolved slot holds: a region + // takes the SELF-RELATIVE delta so a deref is one add, and the tool's + // builder path takes the node's ARENA OFFSET (§6.3). + bool arena = false; + // WHERE A MAP'S ENTRIES AND A LIST'S ELEMENTS LAND while this node's body + // decodes (docs/SPEC-TABLES.md §2.8, §2.9): the node's own extent on the + // region path and the builder's arena on the tool's. It is MUTABLE + // because the cursor belongs to ONE node's decode and the dispatch that + // owns that node holds the map by const reference, exactly as it did + // before either construct existed. The decoder's signature does not + // move for a construct it may not carry. + mutable TableExtentCarve * carve = NULL; + // and the TOOL's path's allocation front, set once: there the arrays + // are the builder's arena's rather than a node's extent. + TableWorker * worker = NULL; + // THE TOOL PATH'S REFUSAL (docs/SPEC-TABLES.md §2.9): a count above the + // int32 cap met while a body decoded. LoadBuilder answers NULL for it + // and moves no counter; mutable for the reason the cursor is. + mutable bool refused = false; +}; + +// TableNodeResolve places one node index in a pointer slot, and every failure +// is one of §4's events with the pointer left null. The declared TARGET type id +// is checked at every index, the root's included: the root carries no record +// and therefore no wire type id, so the READER'S OWN root type is what the +// claim is checked against. +inline void TableNodeResolve( const TableNodeMap & map, TableRef & slot, uint64_t index, uint64_t target, TableReport * report ) +{ + slot.value = 0; + if ( index == kTableNodeIndexNull || !map.good ) { return; } + if ( index - 1 >= (uint64_t) map.count ) + { + report->malformed = true; // an index above node_count + 1 + return; + } + const TableNodeDirEntry & entry = map.entries[index - 1]; + if ( entry.offset == kTableNodeAbsent ) + { + // a node whose type id this build could not name KEEPS ITS INDEX, and + // every pointer naming it reads null. The unknown was counted once, at + // the node, not once per pointer. + return; + } + if ( entry.type_id != target ) + { + report->kind_mismatch++; + return; + } + slot.value = map.arena ? (int64_t) entry.offset + : (int64_t) ( ( map.base + entry.offset ) - (const uint8_t *) &slot ); +} + +// ---- the record SCAN, and it is the whole of load's bound (§3.1) ---- +// +// Reading follows no reference. The scan walks the root body's top-level fields, +// finds the ONE under the reserved id, and reads records out of its payload in +// order — the field rides once, so nothing is copied to make a body contiguous +// and the generated body decoder never learns the transport exists. +struct TableNodeScan +{ + TableReader fields; // over the ROOT body, skipping past everything else + const uint8_t * payload; // the node-table field's payload + int64_t payload_size; + int64_t payload_offset; + bool opened; // the root body has been walked for the field + uint64_t declared; + int64_t records; + bool present; // the root body carries a node table at all + bool malformed; + const TableIdTable * ids; +}; + +inline TableNodeScan TableNodeScanBegin( const uint8_t * body, int64_t size, TableReport * report, const TableIdTable * ids ) +{ + TableNodeScan s = { TableReader( body, size, report, ids ), NULL, 0, 0, false, 0, 0, false, false, ids }; + return s; +} + +// find the node-table field, or answer false when the root body has none. A +// body carrying an id more than once is legal input and THE LAST OCCURRENCE +// WINS (docs/SPEC-TABLES.md §3), so the walk runs to the terminator and keeps +// the last rather than stopping at the first. +inline bool TableNodeScanOpen( TableNodeScan & s ) +{ + if ( s.opened ) { return false; } + s.opened = true; + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.fields.getleb( ref ) ) { break; } + if ( ref == 0 ) { break; } // the terminator + if ( s.ids == NULL || ref > (uint64_t) s.ids->count ) { break; } + const uint64_t id = s.ids->at( ref ); + if ( !s.fields.has( 1 ) ) { break; } + const uint8_t kind = s.fields.get8(); + if ( id == kTableNodeTableFieldId ) + { + s.present = true; + if ( kind != 12 ) { s.malformed = true; return false; } + uint64_t length = 0; + if ( !s.fields.getleb( length ) || !s.fields.room( length ) ) { s.malformed = true; return false; } + s.payload = s.fields.buffer + s.fields.offset; + s.payload_size = (int64_t) length; + s.fields.offset += (int64_t) length; + continue; + } + if ( !s.fields.skip( kind ) ) { break; } + } + if ( s.payload == NULL ) { return false; } + TableReader head( s.payload, s.payload_size, s.fields.report, s.ids ); + if ( !head.getleb( s.declared ) ) { s.malformed = true; return false; } + s.payload_offset = head.offset; + return true; +} + +// the next record, or false at the end of the table — s.malformed says whether +// the end was the end or the framing giving out +inline bool TableNodeScanNext( TableNodeScan & s, uint64_t & type_id, const uint8_t * & body, int64_t & length ) +{ + if ( !s.opened && !TableNodeScanOpen( s ) ) { return false; } + if ( s.payload == NULL || s.payload_offset >= s.payload_size ) { return false; } + TableReader rec( s.payload, s.payload_size, s.fields.report, s.ids ); + rec.offset = s.payload_offset; + uint64_t ref = 0; + if ( !rec.getleb( ref ) || ref == 0 || s.ids == NULL || ref > (uint64_t) s.ids->count ) + { + s.malformed = true; // a type id reference of 0, or one past the table + return false; + } + type_id = s.ids->at( ref ); + uint64_t declared_length = 0; + if ( !rec.getleb( declared_length ) ) + { + s.malformed = true; // a record whose length is damaged + return false; + } + if ( declared_length > (uint64_t) ( s.payload_size - rec.offset ) ) + { + s.malformed = true; // a record whose length runs past its field + return false; + } + body = s.payload + rec.offset; + length = (int64_t) declared_length; + s.payload_offset = rec.offset + length; + s.records++; + return true; +} + +// The record scan is AUTHORITATIVE: node_count is data from the wire, and a +// count that disagrees with the scan is malformed. Nothing is sized from it +// before the scan has confirmed it. +inline bool TableNodeScanWhole( TableNodeScan & s ) +{ + if ( s.malformed ) { return false; } + if ( !s.present ) { return true; } // no node table at all is not a broken one + return s.declared == (uint64_t) s.records; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_ARENA + +#ifndef MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES +#define MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES + +namespace mapdemo { + +// ---- the NODE TABLE on the message wire (docs/SPEC-TABLES.md §3.1, §3.3) ---- +// +// THE NODE TABLE, WHEN A BODY HAS ONE, IS THE FIRST FIELD OF THE ROOT BODY: the +// reserved id as a reference, the node count at THIRTY-TWO RAW BITS, then the +// records back to back, each a type id reference and a body: a table's fields +// end at their own zero reference, and a blob's body is a length, an align and +// its bytes. A root +// that reaches no node elides the field, like every other empty thing. +// +// Measure derives the numbering from the graph and save derives the same one, +// and the two thunks stored at numbering time are what let one loop write a +// table of mixed types. +template +inline int64_t TableMessageNodeTableMeasure( const Ctx & ctx, const TableNumbering & n, int64_t index_bits, int64_t at ) +{ + if ( n.count == 0 ) { return 0; } // a root that reaches no nodes writes none of them + int64_t bits = kTableMessageRefBitsHere + 32; + for ( int64_t k = 0; k < n.count; k++ ) + { + bits += kTableMessageRefBitsHere; + const int64_t body = n.entries[k].message_measure( (const void *) &ctx, n, index_bits, at + bits, n.entries[k].node ); + if ( body < 0 ) { return -1; } + bits += body; + } + return bits; +} + +template +inline bool TableMessageNodeTableSave( const Ctx & ctx, const TableNumbering & n, int64_t index_bits, TableBitWriter & w ) +{ + if ( n.count == 0 ) { return true; } + w.put( kTableNodeTableFieldSlot, kTableMessageRefBitsHere ); + w.put( (uint64_t) n.count, 32 ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.put( n.entries[k].type_slot, kTableMessageRefBitsHere ); + if ( !n.entries[k].message_save( (const void *) &ctx, n, index_bits, w, n.entries[k].node ) ) { return false; } + } + return !w.overflow; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES + +#ifndef MAPDEMO_SCHEMA_TABLE_RETAIN +#define MAPDEMO_SCHEMA_TABLE_RETAIN + +namespace mapdemo { + +// ---- RETAIN-UNKNOWN (docs/SPEC-TABLES.md §6.6) ---- +// +// A REGION ROUND TRIP AND ONLY THAT: LoadRetain is Load's path into a region +// and SaveRetain saves from that same region. The builder path carries no +// retention, because a builder has no node directory to anchor a record on and +// re-derives its numbering from the reader's declaration order. +// +// Nothing here allocates. The record bytes and the retained-id list are the +// caller's storage, declared with their capacities, and a record that does not +// fit whole is dropped with one retain_lost. + +// THE PATH NAMES THE BODY, and it is the REGION's own address (§6.6). Step one +// is the node's index in the region's node directory, 1 for the root body and +// k for the node at directory position k - 1. Every further step is the PAIR: +// the field ordinal in the body the step descends from, in the READER's own +// declaration order, and the element index inside that field: zero for a +// scalar body, the element's index for an array of any of the four kinds, the +// ARM's OWN ORDINAL for a union, and the key's slot for a map. +static const int32_t kTableRetainDepthMax = 5; + +struct TableRetainStep +{ + uint32_t ordinal; + uint32_t index; +}; + +// at is the node's own address, which is what the SAVE side matches on: a +// record carries the directory INDEX and the directory answers the address in +// one add, so neither side ever searches a numbering. +struct TableRetainPath +{ + const void * at; + uint32_t node; + int32_t depth; + TableRetainStep steps[ kTableRetainDepthMax ]; +}; + +inline TableRetainPath TableRetainPathRoot( const void * at, uint32_t node ) +{ + TableRetainPath path; + path.at = at; + path.node = node; + path.depth = 0; + return path; +} + +// A STEP IS COMPUTED LOCALLY, at the moment the walk descends (§6.6), and it +// is taken by VALUE so that a descent is an expression: both sides walk the +// same declaration order, so neither numbers a tree and neither pops. +inline TableRetainPath TableRetainStepInto( const TableRetainPath & path, uint32_t ordinal, uint32_t index ) +{ + TableRetainPath out = path; + if ( out.depth < kTableRetainDepthMax ) + { + out.steps[ out.depth ].ordinal = ordinal; + out.steps[ out.depth ].index = index; + } + out.depth++; + return out; +} + +// THE CALLER'S TWO STORES (§6.6): the record bytes and the retained ids, each +// a pointer, a capacity and what has been used of it. A retention buffer +// belongs to ONE loaded region, and the next LoadRetain into it resets both. +struct TableRetain +{ + // AN ENTRY IS THE ID AND ITS SLOT IN THE TRAILER BEING WRITTEN. The two + // stores are numbered into ONE trailer in merged first-use order, so an + // index into this list is not the number a second reference wants and the + // slot rides beside the id. The layout is this port's own. + struct Id + { + uint64_t id; + int32_t slot; + }; + + uint8_t * bytes = NULL; + int64_t capacity = 0; + int64_t used = 0; + Id * ids = NULL; + int32_t id_capacity = 0; + int32_t id_used = 0; + int32_t count = 0; // records held + + // the REGION this buffer belongs to: a record carries a directory index + // and the save resolves it here, so nothing searches and nothing allocates + const uint8_t * base = NULL; + const TableNodeDirEntry * directory = NULL; + int64_t directory_count = 0; +}; + +// A RETAINED RECORD IS READER-PRIVATE (§6.6). It is not a wire form: no form +// byte, no version, no declared byte order, and nothing ever writes one to +// disk or hands one to another process. What it must CARRY is the body it +// belongs to, and the field's identity and bytes with every reference +// resolved. This layout is one sound way to carry them and nothing compares +// two ports' buffers. +// +// u32 record bytes, this header included +// u32 node the path's first step +// u32 depth the step pairs that follow +// u32 payload bytes +// u64 field id +// u8 kind +// u8 placed the save's own mark, cleared before every save +// depth x { u32 ordinal, u32 index } +// payload the field's payload with every reference resolved +static const int64_t kTableRetainRecordHeader = 26; + +inline uint32_t TableRetainRead32( const uint8_t * p ) +{ + return uint32_t( p[0] ) | uint32_t( p[1] ) << 8 | uint32_t( p[2] ) << 16 | uint32_t( p[3] ) << 24; +} + +inline uint64_t TableRetainRead64( const uint8_t * p ) +{ + return uint64_t( TableRetainRead32( p ) ) | ( uint64_t( TableRetainRead32( p + 4 ) ) << 32 ); +} + +inline void TableRetainWrite32( uint8_t * p, uint32_t v ) +{ + p[0] = uint8_t( v ); p[1] = uint8_t( v >> 8 ); p[2] = uint8_t( v >> 16 ); p[3] = uint8_t( v >> 24 ); +} + +inline void TableRetainWrite64( uint8_t * p, uint64_t v ) +{ + TableRetainWrite32( p, uint32_t( v ) ); + TableRetainWrite32( p + 4, uint32_t( v >> 32 ) ); +} + +inline int64_t TableRetainRecordBytes( const uint8_t * record ) { return (int64_t) TableRetainRead32( record ); } +inline uint32_t TableRetainRecordNode( const uint8_t * record ) { return TableRetainRead32( record + 4 ); } +inline int32_t TableRetainRecordDepth( const uint8_t * record ) { return (int32_t) TableRetainRead32( record + 8 ); } +inline int64_t TableRetainRecordPayloadBytes( const uint8_t * record ) { return (int64_t) TableRetainRead32( record + 12 ); } +inline uint64_t TableRetainRecordId( const uint8_t * record ) { return TableRetainRead64( record + 16 ); } +inline uint8_t TableRetainRecordKind( const uint8_t * record ) { return record[24]; } +inline bool TableRetainRecordPlaced( const uint8_t * record ) { return record[25] != 0; } +inline const uint8_t * TableRetainRecordSteps( const uint8_t * record ) { return record + kTableRetainRecordHeader; } +inline const uint8_t * TableRetainRecordPayload( const uint8_t * record ) +{ + return record + kTableRetainRecordHeader + 8 * (int64_t) TableRetainRecordDepth( record ); +} +inline uint8_t * TableRetainRecordPayload( uint8_t * record ) +{ + return record + kTableRetainRecordHeader + 8 * (int64_t) TableRetainRecordDepth( record ); +} + +// THE RECORD'S OWN BODY, resolved through the directory the buffer holds. A +// node index names one node for the life of the region, so this is one add. +inline const void * TableRetainRecordAt( const TableRetain & retain, const uint8_t * record ) +{ + const uint32_t node = TableRetainRecordNode( record ); + if ( retain.directory == NULL || node == 0 || (int64_t) node > retain.directory_count ) { return NULL; } + return (const void *) ( retain.base + retain.directory[ node - 1 ].offset ); +} + +// Does this record belong to the body the walk is standing in? The node first, +// which rejects almost everything in one compare, then the step pairs. +inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * record, const TableRetainPath & path ) +{ + if ( TableRetainRecordDepth( record ) != path.depth ) { return false; } + if ( TableRetainRecordAt( retain, record ) != path.at ) { return false; } + const uint8_t * steps = TableRetainRecordSteps( record ); + for ( int32_t i = 0; i < path.depth; i++ ) + { + if ( TableRetainRead32( steps + 8 * i ) != path.steps[i].ordinal ) { return false; } + if ( TableRetainRead32( steps + 8 * i + 4 ) != path.steps[i].index ) { return false; } + } + return true; +} + +// EVERY ID THIS BUILD CAN NAME, ascending: the set TableIds's capacity is +// derived from. An id inside a retained record takes its trailer entry from +// the GENERATED table when it is here and from the CALLER's list otherwise, so +// no retained id ever enters the generated table and no id is written twice. +static const int32_t kTableRetainKnownIds = 76; +static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, + 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, + 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, + 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, + 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, +}; + +inline bool TableRetainNameable( uint64_t id ) +{ + int32_t low = 0, high = kTableRetainKnownIds - 1; + while ( low <= high ) + { + const int32_t mid = low + ( high - low ) / 2; + if ( kTableRetainKnown[mid] == id ) { return true; } + if ( kTableRetainKnown[mid] < id ) { low = mid + 1; } else { high = mid - 1; } + } + return false; +} + +// THE TWO STORES, NUMBERED INTO ONE TRAILER in merged first-use order (§6.6). +// It answers the surface TableIds answers, ref, count, truncate and +// overflow, so the retain family's codec is the plain one with its names +// changed, and +// the GENERATED TABLE IS UNTOUCHED: its capacity, its overflow rule and its +// -1 stand exactly as they are for every save. +struct TableRetainIds +{ + TableIds known; + int32_t known_slot[ TableIds::kCapacity ]; + TableRetain * retain; + int32_t count; + bool overflow; + bool lost; // a retained id past the caller's capacity: the record is dropped + + TableRetainIds( TableRetain * to_retain ) : retain( to_retain ), count( 0 ), overflow( false ), lost( false ) {} + + // an id this build CAN name, which is every id the generated codec writes + uint64_t ref( uint64_t id ) + { + const int32_t before = known.count; + const uint64_t k = known.ref( id ); + if ( known.overflow ) { overflow = true; return 1; } + if ( known.count != before ) { known_slot[ (int32_t) k - 1 ] = ++count; } + return (uint64_t) known_slot[ (int32_t) k - 1 ]; + } + + // an id from INSIDE a retained record. A retained id takes its entry from + // the caller's list, and one past the capacity sets lost: the record is + // dropped, nothing else about the save changes, and the save is never + // refused (§6.6). + uint64_t record_ref( uint64_t id ) + { + if ( TableRetainNameable( id ) ) { return ref( id ); } + if ( retain == NULL ) { lost = true; return 0; } + for ( int32_t i = 0; i < retain->id_used; i++ ) + { + if ( retain->ids[i].id == id ) { return (uint64_t) retain->ids[i].slot; } + } + if ( retain->id_used >= retain->id_capacity ) { lost = true; return 0; } + retain->ids[ retain->id_used ].id = id; + retain->ids[ retain->id_used ].slot = ++count; + retain->id_used++; + return (uint64_t) count; + } + + // undo every entry taken since mark, in either store. Both are appended in + // slot order, so an entry removed is the last one of its store. + void truncate( int32_t mark ) + { + while ( known.count > 0 && known_slot[ known.count - 1 ] > mark ) { known.truncate( known.count - 1 ); } + while ( retain != NULL && retain->id_used > 0 && retain->ids[ retain->id_used - 1 ].slot > mark ) { retain->id_used--; } + count = mark; + } +}; + +// THE FILE STILL CARRIES ONE ID TABLE (§3): the split is the writer's storage +// rather than the wire's, and the trailer is one merge of two slot-ordered +// stores. +inline int64_t TableRetainIdsBytes( const TableRetainIds & ids ) { return int64_t( ids.count ) * 8 + 8; } + +// A TWO-WAY MERGE over the stores' own slot order, and not a scan for each +// slot. Every entry either store holds took its slot from the same counter, so +// the two runs interleave to exactly the slots 1 to count and the merge has no +// case for a slot neither store took. +inline void TableRetainIdsWrite( TableWriter & w, const TableRetainIds & ids ) +{ + const int32_t retained = ids.retain != NULL ? ids.retain->id_used : 0; + int32_t i = 0, j = 0; + while ( i < ids.known.count || j < retained ) + { + if ( j >= retained || ( i < ids.known.count && ids.known_slot[i] < ids.retain->ids[j].slot ) ) + { + w.put64( ids.known.ids[i] ); + i++; + continue; + } + w.put64( ids.retain->ids[j].id ); + j++; + } + w.put64( uint64_t( ids.count ) ); +} + +// ---- THE RESOLVING WALK (§6.6) ---- +// +// A reference names a SLOT of the file's id table, so a verbatim copy +// re-emitted into a file whose table is ordered differently would point at +// other names in silence. A retained record therefore holds the field with +// every reference replaced by the sixty-four-bit id it names, and every length +// that frames a rewritten reference recomputed. +// +// THE WALK IS AN INTERPRETATION, AND ITS VERDICT IS STATED: it reads kind +// bytes, lengths and references and nothing else. No value is decoded, no +// bound is checked, no branch is taken on a payload byte, and anything it +// cannot frame DROPS THE RECORD, counts one retain_lost, and never raises +// malformed on the plain read. +// +// THE WALK IS ONE PASS EACH WAY, and its cost is linear in the record's own +// bytes. Every length that frames a content in the resolved form is a fixed +// slot rather than a canonical LEB128, so the capture reserves it, writes the +// content, and fills the slot in behind it. A spelling that had to know the +// resolved size before writing it would have to walk each content twice, once +// at every level, and the file chooses the nesting. +// +// A retained record's inner nesting is the WRITER's and not this build's, so +// it is the one depth on this path a file can drive. The cap counts NESTED +// BODIES, and a record past it is dropped on the same rule as any other shape +// the walk cannot take. Time no longer rests on it: it is a small stated +// constant and nothing more. +static const int32_t kTableRetainWalkDepthMax = 64; + +// the three RESERVED ids (§3.1, §3.3). One inside a retained record's payload +// would be re-emitted into a nested body, where it is malformed, so meeting +// one drops the record. +inline bool TableRetainReservedId( uint64_t id ) +{ + return id == kTableNodeTableFieldId || id == kTableBuildVersionFieldId || id == kTableMessageVocabularyFieldId; +} + +struct TableRetainIn +{ + const uint8_t * in; + int64_t size; + int64_t at; + const TableIdTable * ids; + uint8_t * out; // NULL: measuring, and nothing is written + int64_t out_at; +}; + +inline void TableRetainInRaw( TableRetainIn & s, const uint8_t * from, int64_t bytes ) +{ + if ( s.out != NULL ) { memcpy( s.out + s.out_at, from, (size_t) bytes ); } + s.out_at += bytes; +} + +inline void TableRetainInLeb( TableRetainIn & s, uint64_t v ) +{ + uint8_t b[10]; + int64_t n = 0; + while ( v >= 0x80 ) { b[n++] = uint8_t( v ) | 0x80; v >>= 7; } + b[n++] = uint8_t( v ); + TableRetainInRaw( s, b, n ); +} + +inline void TableRetainInId( TableRetainIn & s, uint64_t id ) +{ + uint8_t b[8]; + TableRetainWrite64( b, id ); + TableRetainInRaw( s, b, 8 ); +} + +inline bool TableRetainInLebRead( TableRetainIn & s, uint64_t & value ) +{ + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( s.at >= s.size ) { return false; } + const uint8_t b = s.in[ s.at++ ]; + if ( i == 9 && b > 1 ) { return false; } + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) { return i == 0 || b != 0; } + shift += 7; + } + return false; +} + +// one REFERENCE resolved to the id it names. A zero reference is the wire's +// own "no id", the enum's None and the union's empty arm, and rides as the +// id zero. A reference above the entry count, a reference at an id-table entry +// of zero, and a reference at a reserved id are each damage the plain read +// never looked at, and each drops the record. +inline bool TableRetainInRef( TableRetainIn & s, bool zero_allowed ) +{ + uint64_t ref = 0; + if ( !TableRetainInLebRead( s, ref ) ) { return false; } + if ( ref == 0 ) + { + if ( !zero_allowed ) { return false; } + TableRetainInId( s, 0 ); + return true; + } + if ( s.ids == NULL || ref > (uint64_t) s.ids->count ) { return false; } + const uint64_t id = s.ids->at( ref ); + if ( id == 0 || TableRetainReservedId( id ) ) { return false; } + TableRetainInId( s, id ); + return true; +} + +inline int64_t TableRetainInPayload( TableRetainIn & s, uint8_t kind, int32_t depth ); +inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ); + +// ONE FRAMED LENGTH in the resolved form: a pair of fixed u32. The first is +// the RESOLVED byte count of the content it frames, reserved here and written +// once the content is out. The second is the SAVE's scratch, left zero by the +// capture and filled by the walk that emits. +// +// The record is the reader's own storage and nothing outside this family ever +// reads it, so a length may be written after the bytes it measures. That is +// the whole of what makes the walk one pass. +static const int64_t kTableRetainSlotBytes = 8; + +inline int64_t TableRetainInSlot( TableRetainIn & s ) +{ + const int64_t at = s.out_at; + const uint8_t zero[ kTableRetainSlotBytes ] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + TableRetainInRaw( s, zero, kTableRetainSlotBytes ); + return at; +} + +inline void TableRetainInPatch( TableRetainIn & s, int64_t slot, int64_t resolved ) +{ + if ( s.out != NULL ) { TableRetainWrite32( s.out + slot, (uint32_t) resolved ); } +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it. +// The measuring pass takes the same path and reserves the same fixed width, so +// the size it answers is the size the writing pass lays down. +inline int64_t TableRetainInFramed( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s ); + const int64_t resolved = TableRetainInContent( s, kind, length, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s, slot, resolved ); + return resolved; +} + +inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + if ( length < 0 || s.at + length > s.size ) { return -1; } + const int64_t end = s.at + length; + const int64_t began = s.out_at; + switch ( kind ) + { + case 13: // a table BODY: fields, then the zero reference + { + for ( ;; ) + { + uint64_t ref = 0; + const int64_t mark = s.at; + if ( !TableRetainInLebRead( s, ref ) ) { return -1; } + // THE TERMINATOR IS A REFERENCE, and a reference in the + // resolved form is a fixed eight-byte id: the zero that ends a + // body rides at the width every other one does. + if ( ref == 0 ) { TableRetainInId( s, 0 ); break; } + s.at = mark; + if ( !TableRetainInRef( s, false ) ) { return -1; } + if ( s.at >= end ) { return -1; } + const uint8_t field_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &field_kind, 1 ); + if ( TableRetainInPayload( s, field_kind, depth ) < 0 ) { return -1; } + if ( s.at > end ) { return -1; } + } + break; + } + case 14: // an ARRAY body: the element kind, the count, then the elements + { + if ( s.at >= end ) { return -1; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainInLebRead( s, n ) ) { return -1; } + TableRetainInLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( TableRetainInPayload( s, elem_kind, depth ) < 0 ) { return -1; } + if ( s.at > end ) { return -1; } + } + break; + } + case 16: // an ENUM-KEYED body: N triples of a KEY REFERENCE, an L and the element + { + if ( s.at >= end ) { return -1; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainInLebRead( s, n ) ) { return -1; } + TableRetainInLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + // A KEYED BODY'S KEYS RESOLVE AT EVERY ELEMENT KIND (§6.6, §3.2) + if ( !TableRetainInRef( s, false ) ) { return -1; } + uint64_t slot_bytes = 0; + if ( !TableRetainInLebRead( s, slot_bytes ) ) { return -1; } + if ( slot_bytes > (uint64_t) ( end - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, elem_kind, (int64_t) slot_bytes, depth + 1 ) < 0 ) { return -1; } + } + break; + } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + default: + // every other content is bytes: a string, wide text, an escape, a + // payload-free kind, a scalar under a keyed slot's own length + TableRetainInRaw( s, s.in + s.at, length ); + s.at += length; + break; + } + if ( s.at != end ) { return -1; } + return s.out_at - began; +} + +// the depth a payload carries is its enclosing body's: only a framed CONTENT +// is a level, and TableRetainInContent is the one place the cap is read. +inline int64_t TableRetainInPayload( TableRetainIn & s, uint8_t kind, int32_t depth ) +{ + const int64_t began = s.out_at; + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: // the fixed-width kinds, by width + case 3: case 7: case 21: case 26: + case 4: case 8: case 10: case 22: case 27: + case 5: case 9: case 11: case 23: case 28: + case 18: case 19: case 24: case 29: + { + int64_t width = 1; + switch ( kind ) + { + case 3: case 7: case 21: case 26: width = 2; break; + case 4: case 8: case 10: case 22: case 27: width = 4; break; + case 5: case 9: case 11: case 23: case 28: width = 8; break; + case 18: case 19: case 24: case 29: width = 16; break; + default: width = 1; break; + } + if ( s.at + width > s.size ) { return -1; } + TableRetainInRaw( s, s.in + s.at, width ); + s.at += width; + break; + } + case 12: case 31: case 32: case 33: // L, then L bytes, nothing framed inside + { + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + TableRetainInLeb( s, length ); + TableRetainInRaw( s, s.in + s.at, (int64_t) length ); + s.at += (int64_t) length; + break; + } + case 13: case 14: case 16: // L, then a body the walk resolves + { + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, kind, (int64_t) length, depth + 1 ) < 0 ) { return -1; } + break; + } + case 15: // a UNION: the arm id reference, and when it is not zero its kind, L and payload + { + const int64_t mark = s.at; + uint64_t arm = 0; + if ( !TableRetainInLebRead( s, arm ) ) { return -1; } + s.at = mark; + if ( !TableRetainInRef( s, true ) ) { return -1; } + if ( arm == 0 ) { break; } + if ( s.at >= s.size ) { return -1; } + const uint8_t arm_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &arm_kind, 1 ); + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, arm_kind, (int64_t) length, depth + 1 ) < 0 ) { return -1; } + break; + } + case 30: // an ENUM's variant reference, zero for None + { + if ( !TableRetainInRef( s, true ) ) { return -1; } + break; + } + case 17: return -1; // A NODE INDEX (§3.1): the whole record goes with it + default: return -1; // a kind this walk cannot frame + } + return s.out_at - began; +} + +// ---- CAPTURE: the load side (§6.6) ---- +// +// The field is skipped by its framing exactly as it always was and counted +// unknown exactly as it always was, so a full buffer degrades to the default +// behavior one field at a time. False is what r.skip( kind ) answers false +// for, and nothing else: retention can lose a field, it can never turn a good +// read into a bad one. +inline bool TableRetainCapture( TableRetain * retain, TableReader & r, const TableRetainPath & path, + uint64_t field_id, uint8_t kind ) +{ + const int64_t start = r.offset; + if ( !r.skip( kind ) ) { return false; } + if ( retain == NULL ) { return true; } + const int64_t wire_bytes = r.offset - start; + + TableRetainIn probe; + probe.in = r.buffer + start; + probe.size = wire_bytes; + probe.at = 0; + probe.ids = r.ids; + probe.out = NULL; + probe.out_at = 0; + const int64_t payload = TableRetainInPayload( probe, kind, 0 ); + if ( payload < 0 || probe.at != wire_bytes ) { r.report->retain_lost++; return true; } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL: the buffer never holds a + // truncated field, and the read continues (§6.6) + r.report->retain_lost++; + return true; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, field_id ); + record[24] = kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableRetainIn write; + write.in = r.buffer + start; + write.size = wire_bytes; + write.at = 0; + write.ids = r.ids; + write.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out_at = 0; + if ( TableRetainInPayload( write, kind, 0 ) < 0 ) { r.report->retain_lost++; return true; } + retain->used += need; + retain->count++; + r.report->retained++; + return true; +} + +// LoadRetain RESETS BOTH STORES and writes into neither list (§6.6): a +// retained record carries its field's identity in the record itself, with +// every reference resolved. +inline void TableRetainReset( TableRetain * retain, const TableNodeMap & nodes, const uint8_t * region ) +{ + if ( retain == NULL ) { return; } + retain->used = 0; + retain->id_used = 0; + retain->count = 0; + retain->base = region; + retain->directory = nodes.entries; + retain->directory_count = nodes.count; +} + +// ---- RECORD LIFETIME (docs/SPEC-TABLES.md §6.6) ---- +// +// A RETAINED RECORD BELONGS TO THE BODY OCCURRENCE THAT CARRIED IT, AND DIES +// WITH IT. Legal input can carry a known child body twice, and the later +// occurrence resets the child and wins whole (§3, §4): the records retained +// under the earlier occurrence go with the values it held. The discard moves +// NEITHER counter. The writer superseded the data, so nothing was lost that +// the load could have kept, and retained counted the record when its bytes +// were kept and does not fall when they are let go. +// +// The occurrences are four, and each is a body the wire lets a writer put down +// again: a repeated TABLE field, by value or under ?, a UNION whose arm is +// written again, a MAP's duplicate key, and a KEYED-ARRAY slot written again. +// The FIELD form covers the three where the field itself is read again, arm +// switches and shrinking arrays included; the BODY form covers a duplicate key +// inside one occurrence of a map, where the field is read once and the entry +// twice. +inline bool TableRetainUnder( const uint8_t * record, const void * at, const TableRetain & retain, + const TableRetainPath & path, bool field, uint32_t ordinal ) +{ + if ( TableRetainRecordAt( retain, record ) != at ) { return false; } + const int32_t depth = TableRetainRecordDepth( record ); + if ( field ) + { + if ( depth <= path.depth ) { return false; } + } + else if ( depth < path.depth ) { return false; } + const uint8_t * steps = TableRetainRecordSteps( record ); + for ( int32_t i = 0; i < path.depth; i++ ) + { + if ( TableRetainRead32( steps + 8 * i ) != path.steps[i].ordinal ) { return false; } + if ( TableRetainRead32( steps + 8 * i + 4 ) != path.steps[i].index ) { return false; } + } + if ( field && TableRetainRead32( steps + 8 * path.depth ) != ordinal ) { return false; } + return true; +} + +inline void TableRetainDiscard( TableRetain * retain, const TableRetainPath & path, bool field, uint32_t ordinal ) +{ + if ( retain == NULL || retain->count == 0 ) { return; } + int64_t read = 0, write = 0; + int32_t kept = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + const int64_t bytes = TableRetainRecordBytes( retain->bytes + read ); + if ( !TableRetainUnder( retain->bytes + read, path.at, *retain, path, field, ordinal ) ) + { + if ( write != read ) { memmove( retain->bytes + write, retain->bytes + read, (size_t) bytes ); } + write += bytes; + kept++; + } + read += bytes; + } + retain->used = write; + retain->count = kept; +} + +inline void TableRetainDiscardBody( TableRetain * retain, const TableRetainPath & path ) +{ + TableRetainDiscard( retain, path, false, 0 ); +} + +inline void TableRetainDiscardField( TableRetain * retain, const TableRetainPath & path, uint32_t ordinal ) +{ + TableRetainDiscard( retain, path, true, ordinal ); +} + +// ---- EMIT: the save side (§6.6) ---- +// +// The record read back the other way: every resolved id becomes the reference +// the trailer being written gives it, and every length is recomputed against +// the references' new widths. The walk is the capture's mirror and the same +// damage rules apply, except that damage cannot be met: these bytes are the +// reader's own. +// +// A WIRE LENGTH IS CANONICAL LEB128 AND RIDES BEFORE ITS CONTENT, so this side +// cannot fill a slot in behind the bytes the way the capture does. It takes +// one POST-ORDER pass instead: measuring computes each content's wire size and +// leaves it in that content's own scratch slot, and the emit reads the size +// there rather than walking for it. Measuring runs immediately before the +// emit, on the same record and the same id table, which is what makes the two +// readings one walk. +struct TableRetainOut +{ + uint8_t * in; // the record: only a framed length's scratch half is written + int64_t size; + int64_t at; + TableRetainIds * ids; + TableWriter * w; // NULL: measuring, and the scratch slots are being filled + int64_t bytes; +}; + +inline void TableRetainOutRaw( TableRetainOut & s, const uint8_t * from, int64_t bytes ) +{ + if ( s.w != NULL ) { s.w->raw( from, bytes ); } + s.bytes += bytes; +} + +inline void TableRetainOutLeb( TableRetainOut & s, uint64_t v ) +{ + if ( s.w != NULL ) { s.w->putleb( v ); } + s.bytes += TableLebBytes( v ); +} + +inline bool TableRetainOutLebRead( TableRetainOut & s, uint64_t & value ) +{ + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( s.at >= s.size ) { return false; } + const uint8_t b = s.in[ s.at++ ]; + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) { return true; } + shift += 7; + } + return false; +} + +inline bool TableRetainOutRef( TableRetainOut & s ) +{ + if ( s.at + 8 > s.size ) { return false; } + const uint64_t id = TableRetainRead64( s.in + s.at ); + s.at += 8; + if ( id == 0 ) { TableRetainOutLeb( s, 0 ); return true; } // the wire's own no-id + const uint64_t ref = s.ids->record_ref( id ); + if ( s.ids->lost || s.ids->overflow ) { return false; } + TableRetainOutLeb( s, ref ); + return true; +} + +inline bool TableRetainOutPayload( TableRetainOut & s, uint8_t kind, int32_t depth ); +inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t length, int32_t depth ); + +// ONE FRAMED CONTENT, read out of the record's fixed slot and written with the +// canonical LEB128 length this wire wants. The ids it names are interned on +// the way past, which is what makes measure and save one walk in two readings, +// exactly as every other body on this wire is. +// +// MEASURING walks the content, then leaves the wire size it found in the +// scratch half of the slot. EMITTING reads that size, writes it, and CHECKS +// the content against it: a size no measure of this save left there is a +// record refused rather than a length that does not frame what follows. +inline bool TableRetainOutFramed( TableRetainOut & s, uint8_t kind, int32_t depth ) +{ + if ( s.at + kTableRetainSlotBytes > s.size ) { return false; } + uint8_t * const slot = s.in + s.at; + const int64_t resolved = (int64_t) TableRetainRead32( slot ); + s.at += kTableRetainSlotBytes; + if ( s.w == NULL ) + { + const int64_t began = s.bytes; + if ( !TableRetainOutContent( s, kind, resolved, depth ) ) { return false; } + const int64_t wire = s.bytes - began; + if ( wire > 0xFFFFFFFFll ) { return false; } + TableRetainWrite32( slot + 4, (uint32_t) wire ); + s.bytes += TableLebBytes( (uint64_t) wire ); + return true; + } + const int64_t wire = (int64_t) TableRetainRead32( slot + 4 ); + TableRetainOutLeb( s, (uint64_t) wire ); + const int64_t began = s.bytes; + if ( !TableRetainOutContent( s, kind, resolved, depth ) ) { return false; } + return s.bytes - began == wire; +} + +inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t length, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return false; } + if ( length < 0 || s.at + length > s.size ) { return false; } + const int64_t end = s.at + length; + switch ( kind ) + { + case 13: + { + for ( ;; ) + { + if ( s.at + 8 > end ) { return false; } + const uint64_t id = TableRetainRead64( s.in + s.at ); + if ( id == 0 ) { s.at += 8; TableRetainOutLeb( s, 0 ); break; } + if ( !TableRetainOutRef( s ) ) { return false; } + if ( s.at >= end ) { return false; } + const uint8_t field_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &field_kind, 1 ); + if ( !TableRetainOutPayload( s, field_kind, depth ) ) { return false; } + } + break; + } + case 14: + { + if ( s.at >= end ) { return false; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainOutLebRead( s, n ) ) { return false; } + TableRetainOutLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( !TableRetainOutPayload( s, elem_kind, depth ) ) { return false; } + } + break; + } + case 16: + { + if ( s.at >= end ) { return false; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainOutLebRead( s, n ) ) { return false; } + TableRetainOutLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( !TableRetainOutRef( s ) ) { return false; } + if ( !TableRetainOutFramed( s, elem_kind, depth + 1 ) ) { return false; } + } + break; + } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; + default: + TableRetainOutRaw( s, s.in + s.at, length ); + s.at += length; + break; + } + return s.at == end; +} + +// the depth a payload carries is its enclosing body's, exactly as on the +// capture side: only a framed CONTENT is a level. +inline bool TableRetainOutPayload( TableRetainOut & s, uint8_t kind, int32_t depth ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: + case 3: case 7: case 21: case 26: + case 4: case 8: case 10: case 22: case 27: + case 5: case 9: case 11: case 23: case 28: + case 18: case 19: case 24: case 29: + { + int64_t width = 1; + switch ( kind ) + { + case 3: case 7: case 21: case 26: width = 2; break; + case 4: case 8: case 10: case 22: case 27: width = 4; break; + case 5: case 9: case 11: case 23: case 28: width = 8; break; + case 18: case 19: case 24: case 29: width = 16; break; + default: width = 1; break; + } + if ( s.at + width > s.size ) { return false; } + TableRetainOutRaw( s, s.in + s.at, width ); + s.at += width; + break; + } + case 12: case 31: case 32: case 33: + { + uint64_t length = 0; + if ( !TableRetainOutLebRead( s, length ) ) { return false; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return false; } + TableRetainOutLeb( s, length ); + TableRetainOutRaw( s, s.in + s.at, (int64_t) length ); + s.at += (int64_t) length; + break; + } + case 13: case 14: case 16: + { + if ( !TableRetainOutFramed( s, kind, depth + 1 ) ) { return false; } + break; + } + case 15: + { + if ( s.at + 8 > s.size ) { return false; } + const uint64_t arm = TableRetainRead64( s.in + s.at ); + if ( !TableRetainOutRef( s ) ) { return false; } + if ( arm == 0 ) { break; } + if ( s.at >= s.size ) { return false; } + const uint8_t arm_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &arm_kind, 1 ); + if ( !TableRetainOutFramed( s, arm_kind, depth + 1 ) ) { return false; } + break; + } + case 30: + { + if ( !TableRetainOutRef( s ) ) { return false; } + break; + } + default: return false; + } + return true; +} + +// ---- THE RETAINED TAIL: where the records go back (§6.6) ---- +// +// AT THE END OF THEIR OWN BODY, IN THE ORDER RETAINED. Position carries +// nothing on this wire, so appending is chosen for three properties: it is a +// write with no splice, the retained order is preserved, and the result is +// IDEMPOTENT after the first save. +// +// A RETAINED ID PAST THE CAPACITY COUNTS ONE retain_lost AND ITS RECORD IS +// DROPPED, and the save is never refused. MeasureRetain and SaveRetain drop +// the same records under the same walk, so the measure sees the same overflow +// and its answer is the size the save writes. + +// one record's WIRE bytes under the trailer being written, and -1 for a record +// this save cannot place: an id the caller's list had no room for, or a +// resolved form the walk cannot read back. The ids it names are interned on +// the way past, which is what makes measure and save one rule read twice. +// +// THIS IS THE MEASURING PASS, and it leaves every framed content's wire size +// in that content's own scratch slot. The record is the caller's buffer and +// the pass writes nothing else into it. +inline int64_t TableRetainRecordWire( uint8_t * record, TableRetainIds & ids, uint64_t & ref ) +{ + const int32_t mark = ids.count; + ids.lost = false; + ref = ids.record_ref( TableRetainRecordId( record ) ); + if ( !ids.lost && !ids.overflow ) + { + TableRetainOut s; + s.in = TableRetainRecordPayload( record ); + s.size = TableRetainRecordPayloadBytes( record ); + s.at = 0; + s.ids = &ids; + s.w = NULL; + s.bytes = 0; + if ( TableRetainOutPayload( s, TableRetainRecordKind( record ), 0 ) && s.at == s.size ) + { + return TableLebBytes( ref ) + 1 + s.bytes; + } + } + // the record is not written at all, and nothing else about the save + // changes: a full id list degrades to the default behavior one record at a + // time, and the entries this attempt took are given back + ids.truncate( mark ); + ids.lost = false; + return -1; +} + +inline int64_t TableRetainTailMeasure( TableRetain * retain, TableRetainIds & ids, const TableRetainPath & path ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return 0; } + int64_t bytes = 0; + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordHere( *retain, record, path ) ) { continue; } + uint64_t ref = 0; + const int64_t wire = TableRetainRecordWire( record, ids, ref ); + if ( wire < 0 ) { continue; } + bytes += wire; + } + return bytes; +} + +inline bool TableRetainTailSave( TableRetain * retain, TableRetainIds & ids, TableWriter & w, const TableRetainPath & path ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return true; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordHere( *retain, record, path ) ) { continue; } + uint64_t ref = 0; + if ( TableRetainRecordWire( record, ids, ref ) < 0 ) { continue; } + w.putleb( ref ); + w.put8( TableRetainRecordKind( record ) ); + TableRetainOut s; + s.in = TableRetainRecordPayload( record ); + s.size = TableRetainRecordPayloadBytes( record ); + s.at = 0; + s.ids = &ids; + s.w = &w; + s.bytes = 0; + if ( !TableRetainOutPayload( s, TableRetainRecordKind( record ), 0 ) ) { return false; } + record[25] = 1; // PLACED: the one mark the save leaves on the buffer + } + return !w.overflow; +} + +// THE SAVE'S OWN SHARE OF retain_lost, counted ONCE and read after the save +// (§6.6): every record the walk did not place. A record whose path no longer +// names a body, one the caller's id list had no room for, and one the walk +// could not read back are one number here, because the check a caller reads is +// one number. A record is marked as it is written, so this cannot double-count +// a body measured twice. +inline void TableRetainClearPlaced( TableRetain * retain ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + retain->bytes[ at + 25 ] = 0; + at += TableRetainRecordBytes( retain->bytes + at ); + } +} + +inline void TableRetainCountLost( const TableRetain * retain, TableReport * report ) +{ + if ( retain == NULL || retain->bytes == NULL || report == NULL ) { return; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + const uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordPlaced( record ) ) { report->retain_lost++; } + } +} + +// THE NODE TABLE under retention (§3.1, §6.6): the same fill rule the plain +// pair derives, with the retain family's ids and each record's own body +// reached through a dispatch the CALL supplies rather than a second pair of +// thunks on the numbering. A store per node on the PLAIN save path would be a +// cost this feature is not allowed to have. +template +inline int64_t TableNodeTablePayloadRetain( const Ctx & ctx, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure ) +{ + int64_t payload = TableLebBytes( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + payload += TableLebBytes( ids.ref( n.entries[k].type_id ) ); + const int64_t body = measure( ctx, n, ids, n.entries[k].type_id, n.entries[k].node, retain ); + if ( body < 0 ) { return -1; } + payload += TableLebBytes( (uint64_t) body ) + body; + } + return payload; +} + +template +inline int64_t TableNodeTableMeasureRetain( const Ctx & ctx, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure ) +{ + if ( n.count == 0 ) { return 0; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayloadRetain( ctx, ids, n, retain, measure ); + if ( payload < 0 ) { return -1; } + return TableLebBytes( ref ) + 1 + TableLebBytes( (uint64_t) payload ) + payload; +} + +template +inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure, Save save ) +{ + if ( n.count == 0 ) { return true; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayloadRetain( ctx, ids, n, retain, measure ); + if ( payload < 0 ) { return false; } + w.putleb( ref ); + w.put8( 12 ); // kind 12 is the opaque byte payload, exactly as the plain save writes it + w.putleb( (uint64_t) payload ); + w.putleb( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.putleb( ids.ref( n.entries[k].type_id ) ); + const int64_t body = measure( ctx, n, ids, n.entries[k].type_id, n.entries[k].node, retain ); + if ( body < 0 ) { return false; } + w.putleb( (uint64_t) body ); + if ( !save( ctx, n, w, ids, n.entries[k].type_id, n.entries[k].node, retain ) ) { return false; } + } + return true; +} +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_RETAIN + +#ifndef MAPDEMO_SCHEMA_TABLE_EXTENT +#define MAPDEMO_SCHEMA_TABLE_EXTENT + +namespace mapdemo { + +// ---- the NODE EXTENT: where a map's entries and a list's elements live (§2.8, §2.9) ---- + +// TableExtentCarve is a node's extent cursor, PRE-ORDER: a container's whole +// array first, then, element by element in the container's own order, the +// arrays of any list or map an element holds by value. The cursor is the node +// map's, because the generated decoder is threaded with that and not with a +// region. +struct TableExtentCarve +{ + uint8_t * at = NULL; // the region path: the node's extent, unspent + int64_t left = 0; + TableWorker * worker = NULL; // the TOOL's path: the arrays come from the arena +}; + +// AN UNREACHED SLOT MUST HOLD NO LIST OR MAP WITH ELEMENTS IN IT (§2.8, §2.9, +// §7.6). An empty one takes no bytes, so a record whose extent measures ZERO is +// a record whose every by-value list and map is empty. A measure that REFUSED +// answers non-zero here too, and refusing on it is the same answer one level up. +inline bool TableExtentUnreachedEmpty( int64_t extent ) { return extent == 0; } + +// ---- LoadMeasure's framing walk (§6.5) ---- +// +// The measure reads no field value: it walks each record's field headers, +// skipping every payload by its framing, to reach each N at every depth. A +// false is a REFUSAL, and it carries its reason (§6.5). +typedef bool ( * TableWireExtentFn )( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ); + +// the framing walk over an ARRAY OF TABLES held by value: its elements' own +// lists and maps are part of this node's extent too +inline bool TableWireExtentElements( const uint8_t * body, int64_t length, int64_t & at, TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } + if ( r.get8() != 13 ) { return true; } + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +// and over an ENUM-KEYED array, whose triples carry a key REFERENCE before each +// length-prefixed element (docs/SPEC-TABLES.md §3.2) +inline bool TableWireExtentKeyed( const uint8_t * body, int64_t length, int64_t & at, TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } + if ( r.get8() != 13 ) { return true; } + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t key = 0; + if ( !r.getleb( key ) ) { return true; } + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_EXTENT + +#ifndef MAPDEMO_SCHEMA_TABLE_MAP +#define MAPDEMO_SCHEMA_TABLE_MAP + +namespace mapdemo { + +// ---- a MAP: a sorted entry array, and the lookup over it (§2.8) ---- +// +// On the wire, in a region and in a cook a map is an array of one generated +// ENTRY table held in ascending key order. What this adds is Find — a binary +// search over that array where it lies — and a builder that inserts, replaces +// and erases by key. Nothing here is stored: a region and a cook carry the +// array and the count, and not one byte about a hash or a probe. + +// entries carved from ONE call to the allocator pair; a new segment is +// appended when the current one fills, and nothing ever moves (§6.4) +static const int32_t kTableMapSegmentEntries = 32; + +// TableDeclRef names a type in an unevaluated context and is never defined — +// what 's declval is for, without the include the generated corpus +// refuses to pay for (the iterator_traits note, §13.9). +template T & TableDeclRef(); + +// THE ORDER IS TOTAL, AND IT IS THE SAME IN NINE LANGUAGES (§2.8). Integers +// compare by VALUE, signed for the signed kinds and unsigned for the unsigned. +// Strings compare by BYTES, unsigned, a shorter string that is a prefix of a +// longer one first: memcmp over the common length, then the lengths. Never a +// locale, never a code point, never a case fold. +inline int TableKeyOrder( uint64_t a, uint64_t b ) { return a < b ? -1 : ( a > b ? 1 : 0 ); } +inline int TableKeyOrder( int64_t a, int64_t b ) { return a < b ? -1 : ( a > b ? 1 : 0 ); } +inline int TableKeyOrder( const char * a, int32_t a_length, const char * b, int32_t b_length ) +{ + const int32_t common = a_length < b_length ? a_length : b_length; + if ( common > 0 ) + { + const int order = memcmp( (const void *) a, (const void *) b, (size_t) common ); + if ( order != 0 ) { return order < 0 ? -1 : 1; } + } + return a_length < b_length ? -1 : ( a_length > b_length ? 1 : 0 ); +} + +// the length of a NUL-terminated key at a call site, bounded by the storage it +// has to fit: a key one byte longer than the bound is refused, never truncated +inline int32_t TableKeyLength( const char * key, int32_t bound ) +{ + if ( key == NULL ) { return 0; } + for ( int32_t i = 0; i <= bound; i++ ) { if ( key[i] == 0 ) { return i; } } + return bound + 1; // longer than the bound: the caller refuses it +} + +// A KEY IS DATA AND A LENGTH, and the length is CARRIED, never recomputed +// (§2.8, §3). A string(N) key holds any byte a wire or a text can spell, +// U+0000 included, so a lookup that measures to the first NUL answers that "a" +// and "a", 0, "b" are the same key: the first entry is found, RESET, and +// relabeled with the second key, which deletes an entry the report never +// mentions. Every internal lookup and every insertion takes this pair, and the +// public const char * surface builds one and is a wrapper over it. +struct TableMapKeyRef +{ + const char * data; + int32_t length; +}; + +// ---- the storage: SIXTEEN BYTES in the holder's record (§2.8, §7.2) ---- +// +// An int64 self-relative reference to the entry array and an int32 count, then +// padding to eight. The reference is a TableRef like a pointer's: in the arena +// it names the builder's HEAD, in a region it is the delta from the slot to +// the first entry, and 0 is the empty map in both. +template struct TableMap +{ + TableRef entries; + int32_t count = 0; // the LIVE count, in both forms + int32_t padding = 0; // named, so the record has no unwritten byte in it + + // ---- the CONST form: a locked region, a loaded one, an opened cook ---- + // + // One surface over one encoding (§6.3). A region reference resolves from + // the slot's own address, so every one of these is a member and needs no + // base and no context. + const Entry * Entries() const + { + return entries.value != 0 ? (const Entry *) ( (const uint8_t *) &entries + entries.value ) : NULL; + } + int32_t size() const { return count; } + + // FIND: floor( log2 n ) + 1 key compares, in place, no allocation. NULL + // when absent, and on a map[K]*T the RESOLVED pointer, which is what a + // pointer field's accessor answers. + template const Entry * FindEntry( Key key ) const + { + const Entry * base = Entries(); + int32_t low = 0, high = count; + while ( low < high ) + { + const int32_t mid = low + ( high - low ) / 2; + const int order = TableEntryOrder( base[mid], key ); + if ( order == 0 ) { return base + mid; } + if ( order < 0 ) { low = mid + 1; } else { high = mid; } + } + return NULL; + } + // the return type is DEDUCED, so it is worked out when a call site + // instantiates Find and not when the holder's record declares the slot — + // which is what lets the entry's own overloads be declared after it + template auto Find( Key key ) const + { + return TableEntryFound( FindEntry( key ) ); + } + + // ---- iteration: ASCENDING key order, the key beside the value ---- + // + // A proxy BY VALUE, the keyed array's shape (§2.4): for ( auto [ key, + // value ] : map ). It carries no iterator_traits, for the reason + // TableKeyed's does not (§13.9). + struct ConstEntry + { + decltype( TableEntryKey( TableDeclRef() ) ) key; + decltype( TableEntryFound( (const Entry *) NULL ) ) value; + }; + + struct ConstIterator + { + const Entry * at; + ConstEntry operator*() const { return ConstEntry{ TableEntryKey( *at ), TableEntryFound( at ) }; } + ConstIterator & operator++() { at++; return *this; } + bool operator==( const ConstIterator & other ) const { return at == other.at; } + bool operator!=( const ConstIterator & other ) const { return at != other.at; } + }; + + ConstIterator begin() const { return ConstIterator{ Entries() }; } + ConstIterator end() const { return ConstIterator{ Entries() + count }; } +}; + +// ---- the BUILDER's side: a head, and segments that never move (§2.8, §6.4) ---- +// +// The head is a small node in the arena holding the segment chain, the live +// count and the dead count, allocated when the first entry is inserted. Each +// segment is a fixed number of entries carved from one call to the allocator +// pair. An entry's address is stable for the arena's life, so a value handed +// back by an insert stays valid while other entries arrive. +struct TableMapHead +{ + TableRef first; // the arena offset of the first segment + TableRef last; // and of the one an insert appends into + int32_t live; + int32_t dead; +}; + +template struct TableMapSegment +{ + TableRef next; + int32_t used; // entries carved from this segment + int32_t padding; + uint32_t dead[ ( kTableMapSegmentEntries + 31 ) / 32 ]; // Erase marks one bit, never the entry + Entry entries[ kTableMapSegmentEntries ]; +}; + +inline bool TableMapSegmentDead( const uint32_t * dead, int32_t index ) +{ + return ( dead[ index / 32 ] & ( 1u << ( index % 32 ) ) ) != 0; +} + +// ---- the ORDERED CURSOR the four writing walks read (§2.8) ---- +// +// Measure, Save, Lock and Cook each write a map's entries in ascending key +// order with no key twice, deriving the order from the builder's entries as +// each walk derives the numbering (§3.1). Nothing passes between them, so +// measure == save over a map is a real check on two sorts agreeing. +// +// A REGION is already sorted, so its cursor is the array in place and +// allocates nothing. The BUILDER's is the sort: an array of entry pointers +// allocated through the pair and released before the walk returns, because +// sorting the segments themselves would move entries whose addresses a caller +// holds. +template struct TableMapCursor +{ + const Entry * const * order = NULL; // the builder's form: sorted pointers + const Entry * entries = NULL; // the region's form: the array in place + int32_t count = 0; + TableAllocator allocator; + bool ok = false; + const Entry * operator[]( int32_t index ) const + { + return order != NULL ? order[index] : entries + index; + } +}; + +// heapsort: O( n log n ) once per map, no recursion, no allocation past the +// pointer array the caller already paid for +template inline void TableMapSort( const Entry ** order, int32_t count ) +{ + for ( int32_t start = count / 2 - 1; start >= 0; start-- ) + { + int32_t root = start; + for ( ;; ) + { + int32_t child = 2 * root + 1; + if ( child >= count ) { break; } + if ( child + 1 < count && TableEntryOrder( *order[child], *order[child + 1] ) < 0 ) { child++; } + if ( TableEntryOrder( *order[root], *order[child] ) >= 0 ) { break; } + const Entry * swap = order[root]; order[root] = order[child]; order[child] = swap; + root = child; + } + } + for ( int32_t end = count - 1; end > 0; end-- ) + { + const Entry * swap = order[0]; order[0] = order[end]; order[end] = swap; + int32_t root = 0; + for ( ;; ) + { + int32_t child = 2 * root + 1; + if ( child >= end ) { break; } + if ( child + 1 < end && TableEntryOrder( *order[child], *order[child + 1] ) < 0 ) { child++; } + if ( TableEntryOrder( *order[root], *order[child] ) >= 0 ) { break; } + const Entry * hold = order[root]; order[root] = order[child]; order[child] = hold; + root = child; + } + } +} + +// the REGION form: the array is already sorted, so the cursor is the array +template +inline TableMapCursor TableMapOrder( const TableRegionCtx &, const TableMap & map ) +{ + TableMapCursor cursor; + cursor.entries = map.Entries(); + cursor.count = map.count; + cursor.ok = true; + return cursor; +} + +// the BUILDER's form: gather the LIVE entries out of the segment chain in +// insertion order, then sort. A dead entry costs nothing on any wire (§2.8). +template +inline TableMapCursor TableMapOrder( const TableArena & arena, const TableMap & map ) +{ + TableMapCursor cursor; + cursor.allocator = arena.allocator; + cursor.count = map.count; + if ( map.entries.value == 0 || map.count <= 0 ) { cursor.ok = map.count == 0; cursor.count = 0; return cursor; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + if ( head->live != map.count ) { return cursor; } // the slot and the head disagree: refused, never guessed + const Entry ** order = (const Entry **) arena.allocator.alloc( arena.allocator.context, (int64_t) map.count * (int64_t) sizeof( const Entry * ) ); + if ( order == NULL ) { return cursor; } + int32_t at = 0; + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 && at < map.count ) + { + const TableMapSegment * segment = (const TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used && at < map.count; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + order[at++] = segment->entries + i; + } + segment_ref = segment->next; + } + if ( at != map.count ) + { + arena.allocator.free( arena.allocator.context, order ); + return cursor; + } + TableMapSort( order, map.count ); + cursor.order = order; + cursor.ok = true; + return cursor; +} + +template +inline TableMapCursor TableMapOrder( const TableArenaCtx & ctx, const TableMap & map ) +{ + return TableMapOrder( *ctx.arena, map ); +} + +template inline void TableMapRelease( TableMapCursor & cursor ) +{ + if ( cursor.order != NULL ) { cursor.allocator.free( cursor.allocator.context, (void *) cursor.order ); } + cursor.order = NULL; +} + +// ---- the builder's five (§2.8) ---- +// +// Insert APPENDS after one LINEAR SCAN of the live entries for the key it may +// replace, Find is that same scan, and Erase is the scan and one bit. The +// builder builds NO INDEX, and that is a rule: the sort happens once, at Lock, +// Save or Cook, and every lookup that matters runs over the sorted region. + +// the head, allocated when the first entry is inserted +template +inline TableMapHead * TableMapReach( TableWorker & worker, TableMap & map ) +{ + if ( worker.arena == NULL || worker.arena->locked ) { return NULL; } + if ( map.entries.value != 0 ) { return (TableMapHead *) TableArenaAt( *worker.arena, (uint32_t) map.entries.value ); } + uint32_t at = 0; + TableMapHead * head = (TableMapHead *) worker.AllocRaw( (int64_t) sizeof( TableMapHead ), (int64_t) alignof( TableMapHead ), at ); + if ( head == NULL ) { return NULL; } + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + map.entries.value = (int64_t) at; + return head; +} + +// one entry's storage, appended: the current segment when it has room, a new +// one carved from one call to the pair when it does not +template +inline Entry * TableMapAppend( TableWorker & worker, TableMapHead * head, TableMap & map ) +{ + TableMapSegment * segment = NULL; + if ( head->last.value != 0 ) + { + segment = (TableMapSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + if ( segment->used >= kTableMapSegmentEntries ) { segment = NULL; } + } + if ( segment == NULL ) + { + uint32_t at = 0; + segment = (TableMapSegment *) worker.AllocRaw( (int64_t) sizeof( TableMapSegment ), (int64_t) alignof( TableMapSegment ), at ); + if ( segment == NULL ) { return NULL; } // the arena could not carve another segment + segment->next.value = 0; + segment->used = 0; + segment->padding = 0; + for ( int32_t i = 0; i < (int32_t) ( sizeof( segment->dead ) / sizeof( segment->dead[0] ) ); i++ ) { segment->dead[i] = 0; } + if ( head->last.value != 0 ) + { + TableMapSegment * previous = (TableMapSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + previous->next.value = (int64_t) at; + } + else + { + head->first.value = (int64_t) at; + } + head->last.value = (int64_t) at; + } + Entry * entry = segment->entries + segment->used; + segment->used++; + head->live++; + map.count++; + return entry; +} + +// the LINEAR SCAN: the live entries in insertion order, O( n ) key compares +template +inline Entry * TableMapScan( const TableArena & arena, const TableMap & map, Key key ) +{ + if ( map.entries.value == 0 ) { return NULL; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( TableEntryOrder( segment->entries[i], key ) == 0 ) { return segment->entries + i; } + } + segment_ref = segment->next; + } + return NULL; +} + +// ERASE marks the entry DEAD, one bit in the segment's slot and not in the +// entry table, and decrements the live count. Its storage is reclaimed at +// RESET and never reused mid-build, because reusing a slot would make "an +// entry's address is stable" false for exactly one case. +template +inline bool TableMapErase( TableArena & arena, TableMap & map, Key key ) +{ + if ( map.entries.value == 0 ) { return false; } + TableMapHead * head = (TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( TableEntryOrder( segment->entries[i], key ) != 0 ) { continue; } + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + map.count--; + return true; + } + segment_ref = segment->next; + } + return false; +} + +// ---- iterate on the BUILDER: INSERTION order, live entries only (§2.8) ---- +template struct TableMapEach +{ + const TableArena * arena; + TableRef first; + + struct Iterator + { + const TableArena * arena; + TableMapSegment * segment; + int32_t index; + + void Skip() + { + for ( ;; ) + { + if ( segment == NULL ) { return; } + if ( index >= segment->used ) + { + segment = segment->next.value != 0 ? (TableMapSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + index = 0; + continue; + } + if ( TableMapSegmentDead( segment->dead, index ) ) { index++; continue; } + return; + } + } + auto operator*() const { return TableEntryEach( segment->entries + index ); } + Iterator & operator++() { index++; Skip(); return *this; } + bool operator==( const Iterator & other ) const { return segment == other.segment && index == other.index; } + bool operator!=( const Iterator & other ) const { return !( *this == other ); } + }; + + Iterator begin() const + { + Iterator it = { arena, first.value != 0 ? (TableMapSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL, 0 }; + it.Skip(); + return it; + } + Iterator end() const { Iterator it = { arena, NULL, 0 }; return it; } +}; + +template +inline TableMapEach TableMapEachOf( const TableArena & arena, const TableMap & map ) +{ + TableMapEach each = { &arena, TableRef() }; + if ( map.entries.value != 0 ) + { + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + each.first = head->first; + } + return each; +} + +// ---- the LOAD side: where a decoded entry lands (§2.8) ---- +// +// THE READER TRUSTS NOTHING and spends one compare per entry. Every load path +// applies the same rules and produces one report (§4), so the region load of +// §6.5 and LoadBuilder never disagree about a wire. These two shapes are what +// makes that true with one generated decoder: a REGION carves the entry array +// out of the holder node's own extent, and the TOOL's path appends into the +// builder's arena, and the decoder above them cannot tell which it has. + +// The node's extent cursor is TableExtentCarve, the extent runtime's (§2.8, +// §2.9): a map's whole entry array is carved first, then, entry by entry in +// key order, the arrays of any list or map an entry's value holds by value. + +// TableMapFill is one map field being decoded: where the next entry lands, and +// the entry that last LANDED, which is what the ascending check compares +// against. +template struct TableMapFill +{ + TableMap * map = NULL; + Entry * array = NULL; // the region path: the carved array + int32_t capacity = 0; + TableWorker * worker = NULL; // the TOOL's path + bool ok = false; +}; + +template +inline TableMapFill TableMapFillBegin( const TableNodeMap & nodes, TableMap & map, uint32_t n ) +{ + TableMapFill fill; + fill.map = ↦ + map.entries.value = 0; + map.count = 0; + if ( nodes.carve == NULL ) { return fill; } + if ( nodes.carve->worker != NULL ) + { + fill.worker = nodes.carve->worker; // the tool's path: the arena carves + fill.ok = true; + return fill; + } + const int64_t align = (int64_t) alignof( Entry ); + uint8_t * base = (uint8_t *) ( ( (uintptr_t) nodes.carve->at + (uintptr_t) ( align - 1 ) ) & ~( (uintptr_t) ( align - 1 ) ) ); + const int64_t bytes = (int64_t) n * (int64_t) sizeof( Entry ); + const int64_t pad = (int64_t) ( base - nodes.carve->at ); + if ( pad + bytes > nodes.carve->left ) { return fill; } // the measure and the load disagree: refused + nodes.carve->at = base + bytes; + nodes.carve->left -= pad + bytes; + fill.array = (Entry *) base; + fill.capacity = (int32_t) n; + map.entries.value = (int64_t) ( base - (const uint8_t *) &map.entries ); + fill.ok = true; + return fill; +} + +// the entry that last LANDED — NULL before the first +template inline Entry * TableMapFillLast( TableMapFill & fill ) +{ + if ( fill.map->count <= 0 ) { return NULL; } + if ( fill.array != NULL ) { return fill.array + ( fill.map->count - 1 ); } + return TableMapLive( *fill.worker->arena, *fill.map, fill.map->count - 1 ); +} + +// the next slot, at the entry type's declared defaults +template inline Entry * TableMapFillNext( TableMapFill & fill ) +{ + if ( fill.array != NULL ) + { + if ( fill.map->count >= fill.capacity ) { return NULL; } + Entry * entry = fill.array + fill.map->count; + TableReset( *entry ); + fill.map->count++; + return entry; + } + TableMapHead * head = TableMapReach( *fill.worker, *fill.map ); + if ( head == NULL ) { return NULL; } + Entry * entry = TableMapAppend( *fill.worker, head, *fill.map ); + if ( entry != NULL ) { TableReset( *entry ); } + return entry; +} + +// A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): at the first entry whose key +// kind disagrees with the reader's declaration the map resets to EMPTY, one +// kind_mismatch is counted for the map, and its remaining bytes are skipped. +template inline void TableMapFillReset( TableMapFill & fill ) +{ + if ( fill.array != NULL ) + { + fill.map->entries.value = 0; + fill.map->count = 0; + return; + } + if ( fill.map->entries.value != 0 ) + { + TableMapHead * head = (TableMapHead *) TableArenaAt( *fill.worker->arena, (uint32_t) fill.map->entries.value ); + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + } + fill.map->count = 0; +} + +// an EMPTY map's reference is null in both encodings, so a load that placed +// nothing leaves the slot exactly as a Reset does +template inline void TableMapFillEnd( TableMapFill & fill ) +{ + if ( fill.array != NULL && fill.map->count == 0 ) { fill.map->entries.value = 0; } +} + +// the k-th LIVE entry of a builder map, in insertion order — what the tool +// path's ascending check compares against +template +inline Entry * TableMapLive( const TableArena & arena, const TableMap & map, int32_t index ) +{ + if ( map.entries.value == 0 ) { return NULL; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + int32_t at = 0; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( at == index ) { return segment->entries + i; } + at++; + } + segment_ref = segment->next; + } + return NULL; +} + +// ---- LoadMeasure's term, from the FRAMING alone (§2.8, §6.5) ---- +// +// LoadMeasure's term for a map is N x sizeof( Entry ) rounded to +// alignof( Entry ), AT EVERY DEPTH. N is framing and not a value, so this +// reads no field: it walks the map's own header and, where an entry's value +// holds a map or a list of its own, the entries' headers under it. The caller +// owns the allocation precisely so it can refuse a number it did not expect. +// Every -1 carries its REASON (§6.5): the int32 cap first, because a count +// past it cannot fit any body, and then the body's own L, the one rule a +// list's term answers by. +// A MAP ENTRY'S SMALLEST WIRE FOOTPRINT that commands one storage unit is its +// own L and the body's terminator, and under this form's variable lengths that +// footprint is TWO BYTES (docs/SPEC-TABLES.md §4.2). It is what bounds the N a +// map's L can carry, and therefore what a LoadMeasure may be asked for. +static const int64_t kTableMapEntryFloor = 2; + +inline bool TableMapWireExtent( const uint8_t * body, int64_t length, int64_t & at, + int64_t entry_size, int64_t entry_align, TableWireExtentFn inner, + const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } // no array header: nothing rides + if ( r.get8() != 13 ) { return true; } // not an array of tables: §4's ordinary kind mismatch + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + if ( n > (uint64_t) INT32_MAX ) { reason = count_over_extent_cap; return false; } + const int64_t rest = length - r.offset; + if ( n > (uint64_t) ( rest / kTableMapEntryFloor ) ) { reason = count_over_length; return false; } // an N the map's L cannot carry + at = ( at + entry_align - 1 ) & ~( entry_align - 1 ); + at += (int64_t) n * entry_size; + if ( inner == NULL ) { return true; } // no map below an entry: one depth is the whole term + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } // framing damage: the load reports it + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +// ---- the TEXT form's placement (docs/SPEC-TABLES.md §2.8, §16) ---- +// +// The text is a plain JSON object keyed by the KEY, and the generic walk fills +// it through the ENTRY'S OWN descriptor — so all it needs from here is one +// entry at one key, handed back at its defaults. It is the builder's Insert +// with the ENTRY returned rather than its value, because the walk writes the +// value through a field row and not through a typed pointer. +// +// THE ONE INSERTION PRIMITIVE. Lookup, reset, allocation and the KEY COPY are +// all here, so no caller mutates an entry this did not create and no caller +// relabels one it found. A key is copied only when an entry is created, which +// is what makes a duplicate key leave the identity it matched untouched. NULL +// is one thing and one thing only: the arena refused. +template +inline Entry * TableMapPlace( TableWorker & worker, TableMap & map, Key key ) +{ + if ( worker.arena == NULL ) { return NULL; } + Entry * found = TableMapScan( *worker.arena, map, key ); + if ( found != NULL ) + { + TableResetMapValue( *found ); // a repeated key is LAST-WINS, whole + return found; + } + TableMapHead * head = TableMapReach( worker, map ); + if ( head == NULL ) { return NULL; } + Entry * entry = TableMapAppend( worker, head, map ); + if ( entry == NULL ) { return NULL; } + TableReset( *entry ); + TableEntrySetKey( *entry, key ); + return entry; +} + +// ---- the OPTIONAL RUNTIME INDEX (§2.8) ---- +// +// Open addressing with LINEAR PROBING over the sorted array, built AT LOAD for +// a map large enough that log n compares over a cold array cost more than one +// hash and a probe. IT IS NEVER STORED: the caller measures it, owns its +// storage, builds it in one pass and releases it whenever. +// +// ITS HASH AND ITS LOAD FACTOR ARE NOT A CROSS-PORT CONTRACT, and that is a +// rule. What a port is held to is the CONTRACT of the lookup: the same value +// the sorted array's Find returns for the same key, and no allocation past the +// storage the caller handed in. +struct TableMapIndex +{ + int32_t * slots = NULL; // entry indices, +1; 0 is an empty slot + int32_t capacity = 0; + bool good = false; +}; + +// this runtime's own, and no port reproduces it: fnv1a64 over the key's bytes +inline uint64_t TableMapHash( const void * bytes, int32_t length ) +{ + uint64_t hash = 0xCBF29CE484222325ull; + const uint8_t * at = (const uint8_t *) bytes; + for ( int32_t i = 0; i < length; i++ ) { hash ^= (uint64_t) at[i]; hash *= 0x100000001B3ull; } + return hash; +} +inline uint64_t TableMapHash( uint64_t key ) { return TableMapHash( (const void *) &key, (int32_t) sizeof( key ) ); } + +// this runtime's own load factor, and no port reproduces it either: the next +// power of two at or above twice the count, so a probe run stays short +inline int32_t TableMapIndexSlots( int32_t count ) +{ + int32_t slots = 8; + while ( slots < count * 2 ) { slots *= 2; } + return slots; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_MAP + +#ifndef MAPDEMO_SCHEMA_TABLE_LIST +#define MAPDEMO_SCHEMA_TABLE_LIST + +namespace mapdemo { + +// ---- an UNBOUNDED ARRAY: a counted array whose count the data decides (§2.9) ---- +// +// On the wire, in a region and in a cook a list is the kind 14 body a [..N]T +// writes, its elements by-value records inside the holder's node extent. What +// this adds is the slot, a builder that appends into segments that never +// move, and a const surface that indexes and iterates in place. There is no +// sort, no key and no lookup: the order is INSERTION order, and it is +// identity the way position is identity in a fixed array. + +// elements carved from ONE call to the allocator pair. A new segment is +// appended when the current one fills, and nothing ever moves (§6.4) +static const int32_t kTableListSegmentElements = 32; + +// THE ELEMENT STORAGE: T itself, and a TableRef slot for a []*T, whose +// elements are references exactly as a pointer field's slot is (§2.1) +template struct TableListStorage { typedef T Element; }; +template struct TableListStorage { typedef TableRef Element; }; + +// WHAT THE CONST FORM ANSWERS: the element by reference, and on a []*T the +// RESOLVED pointer, one add on the self-relative delta, NULL for a null slot, +// exactly as At answers it (§6.2, §6.3) +template struct TableListConst +{ + typedef const T & Result; + static Result At( const T * element ) { return *element; } +}; +template struct TableListConst +{ + typedef const T * Result; + static Result At( const TableRef * element ) + { + return element->value != 0 ? (const T *) ( (const uint8_t *) element + element->value ) : NULL; + } +}; + +// ---- the storage: SIXTEEN BYTES in the holder's record (§2.9, §7.2) ---- +// +// An int64 self-relative reference to the element array and an int32 count, +// then padding to eight. The reference is a TableRef like a pointer's: in the +// arena it names the builder's HEAD, in a region it is the delta from the slot +// to the first element, and 0 is the empty list in both. It is the map's slot +// exactly, because it is the same two facts. +template struct TableList +{ + typedef typename TableListStorage::Element Element; + + TableRef elements; + int32_t count = 0; // the LIVE count, in both forms + int32_t padding = 0; // named, so the record has no unwritten byte in it + + // ---- the CONST form: a locked region, a loaded one, an opened cook ---- + // + // One surface over one encoding (§6.3). A region reference resolves from + // the slot's own address, so every one of these is a member and needs no + // base and no context. + const Element * Elements() const + { + return elements.value != 0 ? (const Element *) ( (const uint8_t *) &elements + elements.value ) : NULL; + } + int32_t size() const { return count; } + + // INDEXING IS BOUNDS-CHECKED IN EVERY BUILD (§2.4, §2.9): the extent is a + // number that CAME FROM A FILE, so an index past it is not a mistake a + // release build gets to make cheaply. There is no undefined-behavior path + // here in any configuration. The assert carries the message where a + // debugger can read it and NDEBUG removes that. The fatal is what stands + // after it. Both go through the hooks: define schema_assert and + // schema_fatal and this refusal lands in your own handler. + void RefuseIndex( int32_t index ) const + { + if ( (uint32_t) index >= (uint32_t) count ) + { + schema_assert( false && "an unbounded array is indexed inside its count, which came from a file" ); + schema_fatal(); + } + } + typename TableListConst::Result operator[]( int32_t index ) const + { + RefuseIndex( index ); + return TableListConst::At( Elements() + index ); + } + + // ---- iteration: INDEX order, the element and no key ---- + // + // It carries no iterator_traits, for the reason TableKeyed's does not + // (§13.9). + struct ConstIterator + { + const Element * at; + typename TableListConst::Result operator*() const { return TableListConst::At( at ); } + ConstIterator & operator++() { at++; return *this; } + bool operator==( const ConstIterator & other ) const { return at == other.at; } + bool operator!=( const ConstIterator & other ) const { return at != other.at; } + }; + + ConstIterator begin() const { return ConstIterator{ Elements() }; } + ConstIterator end() const { return ConstIterator{ Elements() + count }; } +}; + +// ---- the BUILDER's side: a head, and segments that never move (§2.9, §6.4) ---- +// +// The head is a small node in the arena holding the segment chain, the live +// count and the dead count, allocated when the first element is added. Each +// segment is a fixed number of elements carved from one call to the allocator +// pair. An element's address is stable for the arena's life, so a T * handed +// back by Add stays valid while other elements arrive. +struct TableListHead +{ + TableRef first; // the arena offset of the first segment + TableRef last; // and of the one an Add appends into + int32_t live; + int32_t dead; +}; + +template struct TableListSegment +{ + TableRef next; + int32_t used; // elements carved from this segment + int32_t padding; + uint32_t dead[ ( kTableListSegmentElements + 31 ) / 32 ]; // Erase marks one bit, never the element + Element elements[ kTableListSegmentElements ]; +}; + +inline bool TableListSegmentDead( const uint32_t * dead, int32_t index ) +{ + return ( dead[ index / 32 ] & ( 1u << ( index % 32 ) ) ) != 0; +} + +// the head, allocated when the first element is added +template +inline TableListHead * TableListReach( TableWorker & worker, TableList & list ) +{ + if ( worker.arena == NULL || worker.arena->locked ) { return NULL; } + if ( list.elements.value != 0 ) { return (TableListHead *) TableArenaAt( *worker.arena, (uint32_t) list.elements.value ); } + uint32_t at = 0; + TableListHead * head = (TableListHead *) worker.AllocRaw( (int64_t) sizeof( TableListHead ), (int64_t) alignof( TableListHead ), at ); + if ( head == NULL ) { return NULL; } + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + list.elements.value = (int64_t) at; + return head; +} + +// one element's storage, appended: the current segment when it has room, a +// new one carved from one call to the pair when it does not. NULL means NOT +// ADDED: an arena that cannot carve another segment, or a count at the int32 +// cap (§2.2, §2.9). +template +inline typename TableList::Element * TableListAppend( TableWorker & worker, TableListHead * head, TableList & list ) +{ + typedef typename TableList::Element Element; + if ( list.count >= INT32_MAX ) { return NULL; } // the int32 storage cap + TableListSegment * segment = NULL; + if ( head->last.value != 0 ) + { + segment = (TableListSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + if ( segment->used >= kTableListSegmentElements ) { segment = NULL; } + } + if ( segment == NULL ) + { + uint32_t at = 0; + segment = (TableListSegment *) worker.AllocRaw( (int64_t) sizeof( TableListSegment ), (int64_t) alignof( TableListSegment ), at ); + if ( segment == NULL ) { return NULL; } // the arena could not carve another segment + segment->next.value = 0; + segment->used = 0; + segment->padding = 0; + for ( int32_t i = 0; i < (int32_t) ( sizeof( segment->dead ) / sizeof( segment->dead[0] ) ); i++ ) { segment->dead[i] = 0; } + if ( head->last.value != 0 ) + { + TableListSegment * previous = (TableListSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + previous->next.value = (int64_t) at; + } + else + { + head->first.value = (int64_t) at; + } + head->last.value = (int64_t) at; + } + Element * element = segment->elements + segment->used; + segment->used++; + head->live++; + list.count++; + return element; +} + +// ADD, whole: the head, the append, and the element at its declared defaults +// (§2.9). The text form's placement is this same call, because a list has no +// key to place under (§16). +template +inline typename TableList::Element * TableListPlace( TableWorker & worker, TableList & list ) +{ + typedef typename TableList::Element Element; + TableListHead * head = TableListReach( worker, list ); + if ( head == NULL ) { return NULL; } + Element * element = TableListAppend( worker, head, list ); + if ( element == NULL ) { return NULL; } + new ( element ) Element(); // value-init: the declared defaults, and null for a slot + return element; +} + +// ERASE, ADDRESSED BY THE POINTER (§2.9): the element Add handed back is the +// handle, because a list has no key and the address is the one thing the +// builder promises never moves (§6.4). It marks the element DEAD, one bit in +// the segment's slot and not in the element storage, and decrements the live +// count. False when the pointer is not this list's. Its storage is reclaimed +// at RESET and never reused mid-build, the map's rule for the map's reason. +template +inline bool TableListErase( TableArena & arena, TableList & list, const typename TableList::Element * element ) +{ + typedef typename TableList::Element Element; + if ( list.elements.value == 0 || element == NULL ) { return false; } + TableListHead * head = (TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableListSegment * segment = (TableListSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + if ( element >= segment->elements && element < segment->elements + segment->used ) + { + const int32_t i = (int32_t) ( element - segment->elements ); + if ( TableListSegmentDead( segment->dead, i ) ) { return false; } // already erased + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + list.count--; + return true; + } + segment_ref = segment->next; + } + return false; +} + +// ---- iterate on the BUILDER: INDEX order, live elements only (§2.9) ---- +template struct TableListEach +{ + typedef typename TableList::Element Element; + const TableArena * arena; + TableRef first; + + struct Iterator + { + const TableArena * arena; + TableListSegment * segment; + int32_t index; + + void Skip() + { + for ( ;; ) + { + if ( segment == NULL ) { return; } + if ( index >= segment->used ) + { + segment = segment->next.value != 0 ? (TableListSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + index = 0; + continue; + } + if ( TableListSegmentDead( segment->dead, index ) ) { index++; continue; } + return; + } + } + Element * operator*() const { return segment->elements + index; } + Iterator & operator++() { index++; Skip(); return *this; } + bool operator==( const Iterator & other ) const { return segment == other.segment && index == other.index; } + bool operator!=( const Iterator & other ) const { return !( *this == other ); } + }; + + Iterator begin() const + { + Iterator it = { arena, first.value != 0 ? (TableListSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL, 0 }; + it.Skip(); + return it; + } + Iterator end() const { Iterator it = { arena, NULL, 0 }; return it; } +}; + +template +inline TableListEach TableListEachOf( const TableArena & arena, const TableList & list ) +{ + TableListEach each = { &arena, TableRef() }; + if ( list.elements.value != 0 ) + { + const TableListHead * head = (const TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + each.first = head->first; + } + return each; +} + +// ---- the INDEX-ORDER CURSOR the four writing walks read (§2.9) ---- +// +// Measure, Save, Lock and Cook each visit a list's live elements in the order +// they were added, and they allocate nothing to do it: a region's cursor is +// the array in place, and the builder's walks the segment chain. Indexing the +// builder's form is SEQUENTIAL by construction, every walk steps i, i + 1, +// i + 2, so the cursor remembers where the last access landed and moves one +// live slot per step. An access behind the memo restarts from the first +// segment, which no walk here does. +template struct TableListCursor +{ + const Element * elements = NULL; // the region's form: the array in place + const TableArena * arena = NULL; // the builder's form: the segments + TableRef first; + int32_t count = 0; + bool ok = false; + // the memo: the segment and slot the last access landed on, and the live + // index that slot holds + mutable const TableListSegment * segment = NULL; + mutable int32_t within = -1; + mutable int32_t logical = -1; + + const Element * At( int32_t index ) const + { + if ( elements != NULL ) { return elements + index; } + if ( segment == NULL || index < logical ) + { + segment = first.value != 0 ? (const TableListSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL; + within = -1; + logical = -1; + } + while ( logical < index ) + { + for ( ;; ) + { + within++; + while ( segment != NULL && within >= segment->used ) + { + segment = segment->next.value != 0 ? (const TableListSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + within = 0; + } + if ( segment == NULL ) { return NULL; } // the slot and the head disagree + if ( !TableListSegmentDead( segment->dead, within ) ) { break; } + } + logical++; + } + return segment->elements + within; + } + const Element & operator[]( int32_t index ) const { return *At( index ); } +}; + +// the REGION form: the array is the cursor +template +inline TableListCursor::Element> TableListElements( const TableRegionCtx &, const TableList & list ) +{ + TableListCursor::Element> cursor; + cursor.elements = list.Elements(); + cursor.count = list.count; + cursor.ok = true; + return cursor; +} + +// the BUILDER's form: the live elements out of the segment chain, in the +// order they were added. A dead element costs nothing on any wire (§2.9). +template +inline TableListCursor::Element> TableListElements( const TableArena & arena, const TableList & list ) +{ + TableListCursor::Element> cursor; + cursor.arena = &arena; + cursor.count = list.count; + if ( list.elements.value == 0 || list.count <= 0 ) { cursor.ok = list.count == 0; cursor.count = 0; return cursor; } + const TableListHead * head = (const TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + if ( head->live != list.count ) { return cursor; } // the slot and the head disagree: refused, never guessed + cursor.first = head->first; + cursor.ok = true; + return cursor; +} + +template +inline TableListCursor::Element> TableListElements( const TableArenaCtx & ctx, const TableList & list ) +{ + return TableListElements( *ctx.arena, list ); +} + +// ---- the LOAD side: where a decoded element lands (§2.9) ---- +// +// The same two shapes the map's fill takes, because the decoder above them +// cannot tell which it has: a REGION carves the element array out of the +// holder node's own extent, PRE-ORDER, and the TOOL's path appends into the +// builder's arena. +template struct TableListFill +{ + typedef typename TableList::Element Element; + TableList * list = NULL; + Element * array = NULL; // the region path: the carved array + int32_t capacity = 0; + TableWorker * worker = NULL; // the TOOL's path + bool ok = false; + bool refused = false; // a count above the int32 cap on the tool's path: LoadBuilder answers NULL +}; + +template +inline TableListFill TableListFillBegin( const TableNodeMap & nodes, TableList & list, uint64_t n ) +{ + typedef typename TableList::Element Element; + TableListFill fill; + fill.list = &list; + list.elements.value = 0; + list.count = 0; + if ( nodes.carve == NULL ) { return fill; } + if ( n > (uint64_t) INT32_MAX ) + { + // A COUNT ABOVE THE int32 STORAGE CAP (§2.2, §2.9): into a region it was + // refused by LoadMeasure before this ran, and into a builder it is the + // refusal LoadBuilder answers NULL for, moving no counter + fill.refused = nodes.carve->worker != NULL; + return fill; + } + if ( nodes.carve->worker != NULL ) + { + fill.worker = nodes.carve->worker; // the tool's path: the arena carves + fill.ok = true; + return fill; + } + const int64_t align = (int64_t) alignof( Element ); + uint8_t * base = (uint8_t *) ( ( (uintptr_t) nodes.carve->at + (uintptr_t) ( align - 1 ) ) & ~( (uintptr_t) ( align - 1 ) ) ); + const int64_t bytes = (int64_t) n * (int64_t) sizeof( Element ); + const int64_t pad = (int64_t) ( base - nodes.carve->at ); + if ( pad + bytes > nodes.carve->left ) { return fill; } // the measure and the load disagree: refused + nodes.carve->at = base + bytes; + nodes.carve->left -= pad + bytes; + fill.array = (Element *) base; + fill.capacity = (int32_t) n; + list.elements.value = (int64_t) ( base - (const uint8_t *) &list.elements ); + fill.ok = true; + return fill; +} + +// the next slot, at the element's declared defaults. NULL when the arena +// could not carve, which the decoder reports as framing damage +template inline typename TableList::Element * TableListFillNext( TableListFill & fill ) +{ + typedef typename TableList::Element Element; + if ( fill.array != NULL ) + { + if ( fill.list->count >= fill.capacity ) { return NULL; } + Element * element = fill.array + fill.list->count; + new ( element ) Element(); + fill.list->count++; + return element; + } + return TableListPlace( *fill.worker, *fill.list ); +} + +// A SLOT WHOSE ELEMENT NEVER LANDED is given back (§2.9, §4): the array keeps +// what it decoded, and an element whose own framing gave out before one byte +// of it decoded was not decoded. The region's form uncounts it, and the builder's +// marks it dead, which is what the storage rule allows mid-build. +template inline void TableListFillDrop( TableListFill & fill ) +{ + typedef typename TableList::Element Element; + if ( fill.array != NULL ) + { + if ( fill.list->count > 0 ) { fill.list->count--; } + return; + } + if ( fill.list->elements.value == 0 ) { return; } + TableListHead * head = (TableListHead *) TableArenaAt( *fill.worker->arena, (uint32_t) fill.list->elements.value ); + if ( head->last.value == 0 ) { return; } + TableListSegment * segment = (TableListSegment *) TableArenaAt( *fill.worker->arena, (uint32_t) head->last.value ); + if ( segment->used <= 0 ) { return; } + const int32_t i = segment->used - 1; + if ( TableListSegmentDead( segment->dead, i ) ) { return; } + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + fill.list->count--; +} + +// an EMPTY list's reference is null in both encodings, so a load that placed +// nothing leaves the slot exactly as a Reset does +template inline void TableListFillEnd( TableListFill & fill ) +{ + if ( fill.array != NULL && fill.list->count == 0 ) { fill.list->elements.value = 0; } +} + +// ---- LoadMeasure's term, from the FRAMING alone (§2.9, §6.5) ---- +// +// N x sizeof( T ) rounded to alignof( T ), AT EVERY DEPTH. N is framing and +// not a value, so this reads no field: it walks the list's own header and, +// where a table element holds a list or a map of its own, the elements' +// headers under it. Every -1 carries its REASON (§6.5): the int32 cap first, +// because a count past it cannot fit any body, and then the body's own L. +inline bool TableListWireExtent( const uint8_t * body, int64_t length, int64_t & at, + int64_t elem_size, int64_t elem_align, uint8_t elem_kind, int64_t elem_floor, + TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } // no array header: nothing rides + if ( r.get8() != elem_kind ) { return true; } // another element kind: §4's ordinary kind mismatch, the field reads empty + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + if ( n > (uint64_t) INT32_MAX ) { reason = count_over_extent_cap; return false; } + const int64_t rest = length - r.offset; + if ( n > (uint64_t) ( rest / elem_floor ) ) { reason = count_over_length; return false; } // an N the list's L cannot carry + at = ( at + elem_align - 1 ) & ~( elem_align - 1 ); + at += (int64_t) n * elem_size; + if ( inner == NULL ) { return true; } // nothing below an element: one depth is the whole term + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } // framing damage: the load reports it + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_LIST + +#ifndef MAPDEMO_SCHEMA_BUILD_VERSION +#define MAPDEMO_SCHEMA_BUILD_VERSION + +namespace mapdemo { + +// THE BUILD VERSION (docs/SPEC-TABLES.md §20): one digest over every fact the bytes +// this build produces depend on — the type wire's protocol id, every record's +// layout as the compiler's own C ABI model computes it, and the facts that +// decide what a load PUTS in those slots. It is the number a cook's header +// carries and the number Open compares, and the number a block's prologue +// carries and BlockOpen compares: a build version answers "which build?" and +// not "which form?", and what separates the two forms is their MAGIC. +// +// There are TWO ids in the design and they are not interchangeable: the +// PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is +// what everything cooked or blocked is keyed by. A table edit moves this and +// never the protocol id; a type edit moves both. +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_BUILD_VERSION + +#ifndef MAPDEMO_SCHEMA_TABLE_COOK +#define MAPDEMO_SCHEMA_TABLE_COOK + +namespace mapdemo { + +// ---- the cooked form (docs/SPEC-TABLES.md §7) ---- +// +// A cooked file is a HEADER, a DATA part and an ATTRIBUTION part, in that +// order. Every word of the header is a u64 written in the byte order the cook +// was produced in, and the header is 64 bytes: +// +// 0 magic 0x4b4f4f434d484353, read BYTEWISE before anything else +// 8 build_version the unit's id (docs/SPEC-TABLES.md §20) +// 16 byte_order 1 little, 2 big — the order that WROTE the file +// 24 data_length the region's bytes, rounded up to alignment +// 32 attribution_length the directory's bytes, or 0 +// 40 alignment the region's alignment, never below eight +// 48 reserved zero +// 56 reserved zero +// +// The DATA part is Lock's region written verbatim (§7.2) — the root at its +// base — and it is what a runtime points at. The ATTRIBUTION part is the node +// directory (§6.3), and NOTHING THAT READS THE STRUCTURE TOUCHES IT: it is +// written beside the data for schema cook-check, so a build that ships no +// tooling need not carry it at all. +static const int64_t kTableCookHeaderBytes = 64; + +// THE MAGIC'S VALUE, and a consumer written from the page needs the constant +// rather than a description of one. It is "SCHMCOOK" read as ASCII in the byte +// order a little-endian store produces — the same shape the block form's +// SCHMABLK takes, so a hex dump of a little-endian cook is legible and the two +// accelerators sit in one vocabulary. +// +// IT IS STORED IN THE PRODUCER'S ORDER, which is what makes it the byte-order +// check as well as the form check: a consumer reads back this build's +// constant, or that constant byte-reversed — which identifies a cook of the +// OTHER order — or something that is not a cook. All three answers but the +// first refuse, and a cook and a BLOCK are separated here too, because a +// form's identity belongs in its magic rather than in a second digest. +static const uint64_t TableCookMagic = 0x4b4f4f434d484353ull; + +// THIS BUILD's byte order, as the header's own word carries it. The magic is +// what REFUSES a foreign order; this word is what RECORDS which order wrote +// the file, so a refusal names the order rather than inferring it and a tool +// dumping a cook reads the fact. A file whose magic matched and whose order +// word did not is corrupt, and there is no reading that recovers it. +// +// The BUILD VERSION cannot do either job: §20.1 digests byteorder as a +// GENERATION input, little for every target schema generates for today, so +// two builds of one schema for two orders emit the same id. +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +static const uint64_t TableCookByteOrder = 2; // big +#else +static const uint64_t TableCookByteOrder = 1; // little +#endif + +// The greatest region alignment a cooked file may name. The DATA part begins +// at align_up( 64, alignment ), which is 64 for every unit this language can +// declare — the largest alignment it has is sixteen — so a word past this cap +// describes a file no build of this schema wrote (docs/SPEC-TABLES.md §7.1). +static const uint64_t TableCookMaxAlign = 64; + +// The header read, BYTEWISE. memcpy is the portable spelling of "these eight +// bytes, in this machine's order"; every compiler this repo builds under folds +// it to one load, and it is the only read in the whole of Open that is not a +// comparison. +inline uint64_t table_cook_read64( const uint8_t * p ) +{ + uint64_t v; + memcpy( &v, p, sizeof( v ) ); + return v; +} + +// TableCookOpen: THE WHOLE CHECK, in one place, because §7 states the +// enumeration once and every generated Open is that one enumeration plus +// its own root's two layout facts. +// +// THE CHECK, in order: the magic read bytewise, the byte order it establishes, +// the build version against this build's own, both RESERVED words zero, the +// region alignment the header names, the two part lengths against the length +// the caller passed — a truncated file and a file with trailing bytes are the +// same refusal — the root's own storage inside the data part, and the +// alignment of the base. +// +// AND THAT IS ALL OF IT. On a match the bytes ARE what this build wrote, in +// this build's layout and this build's byte order, so there is nothing to +// validate and nothing to fix up: the caller gets the root. Nothing per node +// happens here, which is what makes open O(1) in the file's size; a walk of +// any shape would forfeit that, and validating an untrusted file is schema +// cook-check's job and a person's decision (§7.4). +// +// EVERY NUMBER BELOW COMES OUT OF THE FILE, so the arithmetic is unsigned and +// each term is BOUNDED BEFORE IT IS ADDED: a forged length near 2^64 must +// refuse, and an addition that wrapped would be the defect the comparison +// after it was supposed to catch. Nothing past length is read on any path, +// including every refusing one. +// A REFUSAL NAMES ITSELF, beside the null (docs/SPEC-TABLES.md §7): the reason +// is written on the refusal path only, so a match costs nothing and a caller +// that passed no out-parameter pays nothing. +inline const uint8_t * TableCookRefuse( TableRefuseReason * reason, TableRefuseReason why ) +{ + if ( reason != NULL ) { *reason = why; } + return NULL; +} + +inline uint64_t table_cook_byteswap64( uint64_t v ) +{ + return ( v >> 56 ) | ( ( v >> 40 ) & 0xff00ull ) | ( ( v >> 24 ) & 0xff0000ull ) | ( ( v >> 8 ) & 0xff000000ull ) + | ( ( v << 8 ) & 0xff00000000ull ) | ( ( v << 24 ) & 0xff0000000000ull ) | ( ( v << 40 ) & 0xff000000000000ull ) + | ( v << 56 ); +} + +inline const uint8_t * TableCookOpen( const void * bytes, uint64_t length, uint64_t root_size, uint64_t root_align, TableRefuseReason * reason ) +{ + // a null buffer is the CALLER's defect, as an unaligned base is; a buffer + // shorter than the header has no header to read and is truncated + if ( bytes == NULL ) { return TableCookRefuse( reason, unaligned_base ); } + if ( length < (uint64_t) kTableCookHeaderBytes ) { return TableCookRefuse( reason, truncated ); } + const uint8_t * raw = (const uint8_t *) bytes; + // the MAGIC, bytewise and first: it is what establishes the byte order + // every other header word is read in, so nothing else may be read before + // it. A byte-reversed constant is a cook of the other order and refuses + // here, which is why the order never reaches a fix-up pass; anything else + // is not a cook at all, a BLOCK's magic included. + const uint64_t magic = table_cook_read64( raw ); + if ( magic != TableCookMagic ) + { + return TableCookRefuse( reason, magic == table_cook_byteswap64( TableCookMagic ) ? foreign_order : not_a_cook ); + } + // a byte-order word that contradicts its own magic describes no cook in + // EITHER order, so it shares the magic's own value (§7.1) + if ( table_cook_read64( raw + 16 ) != TableCookByteOrder ) { return TableCookRefuse( reason, not_a_cook ); } + if ( table_cook_read64( raw + 8 ) != BuildVersion ) { return TableCookRefuse( reason, wrong_build_version ); } + // the RESERVED words: a non-zero one means a writer used a form this build + // does not understand, and Open refuses rather than ignoring it. + if ( table_cook_read64( raw + 48 ) != 0 ) { return TableCookRefuse( reason, reserved_not_zero ); } + if ( table_cook_read64( raw + 56 ) != 0 ) { return TableCookRefuse( reason, reserved_not_zero ); } + const uint64_t data_length = table_cook_read64( raw + 24 ); + const uint64_t attribution_length = table_cook_read64( raw + 32 ); + const uint64_t alignment = table_cook_read64( raw + 40 ); + // THE ALIGNMENT WORD IS DATA, and it is the one header field the rest of + // the check does arithmetic WITH rather than only comparison against. A + // region's alignment is a power of two, never below eight (the floor that + // puts the attribution part on an eight-byte boundary without a second + // padding rule) and never past the cap above; a word that is none of those + // rounds nothing and aligns nothing, so it is refused before it is used, + // which is why bad_alignment precedes both truncated clauses (§7). + if ( alignment < 8 || alignment > TableCookMaxAlign ) { return TableCookRefuse( reason, bad_alignment ); } + if ( ( alignment & ( alignment - 1 ) ) != 0 ) { return TableCookRefuse( reason, bad_alignment ); } + // and it must be an alignment THE ROOT CAN SIT AT, since the root is at + // the region's base: both are powers of two, so "at least the root's" + // is one division. + if ( ( alignment % root_align ) != 0 ) { return TableCookRefuse( reason, bad_alignment ); } + // The DATA part begins at align_up( 64, alignment ). It is DERIVED and not + // a header field, because a fact a reader computes is a fact two writers + // cannot disagree about. + const uint64_t data_offset = ( (uint64_t) kTableCookHeaderBytes + alignment - 1 ) & ~( alignment - 1 ); + if ( length < data_offset ) { return TableCookRefuse( reason, truncated ); } + // the two part lengths against the length the caller passed. The whole + // file is data_offset + data_length + attribution_length, and a length + // that is not EXACTLY that refuses — truncation and trailing bytes are one + // refusal, and both terms are subtracted rather than added so no sum can + // carry. + if ( data_length > length - data_offset ) { return TableCookRefuse( reason, truncated ); } + if ( attribution_length != length - data_offset - data_length ) { return TableCookRefuse( reason, truncated ); } + // the ROOT sits at the region's base, so the region has to hold it: a + // shorter data part describes a root partly outside the file, which is the + // one way a match-and-point reader could hand back storage it never + // received. It is the second clause on truncated (§7). + if ( data_length < root_size ) { return TableCookRefuse( reason, truncated ); } + const uint8_t * base = raw + data_offset; + // the alignment of the BASE, LAST, because it is the only clause that reads + // nothing out of the file. The header pads the data part to the region's + // alignment, so a base an allocator or mmap gave you is already aligned — + // mmap gives page alignment for free — and a base that is not is a caller's + // buffer this form cannot be read out of: the caller's defect, not the file's. + if ( ( (uintptr_t) base % (uintptr_t) alignment ) != 0 ) { return TableCookRefuse( reason, unaligned_base ); } + return base; +} + +// ---- the cooked form, the WRITE side (docs/SPEC-TABLES.md §7.6) ---- +// +// THE BYTE ORDER IS THE TARGET'S, NOT THE HOST'S. A cook is produced in the +// byte order of the build that will read it (§7), so the fixing happens here — +// offline, once, on the writing side — and never at Open. Passing +// TableByteOrder::Big on a little-endian machine produces a big-endian build's +// file, and nothing about the writing host reaches the bytes. +enum class TableByteOrder +{ + Little = 1, // the header's byte_order word, and the order every scalar is written in + Big = 2, +}; + +// One store, width as an argument. Every call site passes a literal width, so +// the loop folds to a store (and a byte swap on the foreign order); a name per +// width would claim four §11 names to save nothing. +inline void table_cook_put( uint8_t * at, uint64_t value, int32_t width, TableByteOrder order ) +{ + if ( order == TableByteOrder::Little ) + { + for ( int32_t i = 0; i < width; i++ ) { at[i] = (uint8_t) ( value >> ( 8 * i ) ); } + } + else + { + for ( int32_t i = 0; i < width; i++ ) { at[i] = (uint8_t) ( value >> ( 8 * ( width - 1 - i ) ) ); } + } +} + +// A 128-bit store as two lanes: sixteen bytes, the low lane first in the +// little order and the high lane first — each lane big-endian — in the big +// order, exactly as a u64 is one lane of eight (docs/SPEC-TABLES.md §7.2). +inline void table_cook_put128( uint8_t * at, uint64_t lo, uint64_t hi, TableByteOrder order ) +{ + if ( order == TableByteOrder::Little ) { table_cook_put( at, lo, 8, order ); table_cook_put( at + 8, hi, 8, order ); } + else { table_cook_put( at, hi, 8, order ); table_cook_put( at + 8, lo, 8, order ); } +} + +// A buffer piece: the USED bytes and nothing else. The tail is already zero — +// the whole extent was zeroed before any field was written — so this copies the +// used prefix and leaves the rest, which is what makes a string's unused tail a +// consequence of one memset rather than a rule per buffer. A used length past +// the buffer, or below zero, is a value no reader could have produced and it is +// clamped rather than trusted: this writes inside the caller's buffer on every +// input. +inline void table_cook_bytes( uint8_t * at, const void * source, int64_t used, int64_t capacity ) +{ + if ( used <= 0 ) { return; } + const int64_t n = used < capacity ? used : capacity; + memcpy( at, source, (size_t) n ); +} + +// A WIDE TEXT buffer piece (docs/SPEC-TABLES.md §7.2): the USED code units, +// each a TWO-BYTE SCALAR in the cook's byte order. A record is written piece +// by piece and never memcpy'd, and a swap has to know where every scalar +// begins — a char16_t is one, so the units go one store each rather than as +// bytes. The tail is already zero, as the narrow twin's is, so the terminating +// zero unit at index used costs nothing here. +inline void table_cook_units( uint8_t * at, const char16_t * source, int64_t used, int64_t capacity, TableByteOrder order ) +{ + if ( used <= 0 ) { return; } + const int64_t n = used < capacity ? used : capacity; + for ( int64_t i = 0; i < n; i++ ) { table_cook_put( at + i * 2, (uint64_t) (uint16_t) source[i], 2, order ); } +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_COOK + +#ifndef MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE +#define MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE + +namespace mapdemo { + +// ---- the cooked form's WRITE side for a POINTERED root (docs/SPEC-TABLES.md §7.6) ---- +// +// A pointered root's cook is the region of §7.2: every node the numbering +// reached (§3.1), once, at its own type's alignment, in index order, the root +// at offset zero. This is that region while it is being laid out and written — +// the tool's own Layout and Write, in one struct. +// +// The OFFSETS are one per node, the root's zero at position 0 and node index k +// at position k - 1, which is the directory's own order (§6.3); they are the +// one allocation the write makes beyond the numbering, and they go through the +// same pair. A measure needs no offsets and leaves the pointer NULL. +struct TableCookRegion +{ + const TableNumbering * numbering = NULL; // node -> index, from the walk that placed it + int64_t * offsets = NULL; // index - 1 -> the node's region offset; NULL while measuring + int64_t count = 0; // nodes, the root included + int64_t bytes = 0; // the data part's length, rounded to align + int64_t align = 0; // the region's alignment: the nodes' greatest, never below eight + uint8_t * base = NULL; // where the data part is being written; NULL while measuring +}; + +// A reference slot: the SELF-RELATIVE delta from the slot's own address to the +// node's start (§6.3), and zero for null. The node is found by the address the +// numbering keyed it under, which is the same address the walk resolved through +// the same context — so a reference the numbering does not carry is a slot the +// walk never reached (a counted array's slot past its count, an absent +// optional's value) holding a node the region will not hold, and it is refused +// rather than written as a delta to nowhere. +inline bool table_cook_ref( const TableCookRegion & region, uint8_t * at, const void * pointee, TableByteOrder order ) +{ + if ( pointee == NULL ) { table_cook_put( at, 0, 8, order ); return true; } + uint64_t index = 0; + if ( !TableNumberingIndex( *region.numbering, pointee, index ) ) { return false; } + if ( index == 0 || index > (uint64_t) region.count ) { return false; } + const int64_t delta = region.offsets[index - 1] - (int64_t) ( at - region.base ); + table_cook_put( at, (uint64_t) delta, 8, order ); + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE + +namespace mapdemo { + +// table TrailsStepsEntry — TABLE-wire storage: relocatable, bounded, defaults in the +// member initializers (docs/SPEC-TABLES.md) +struct TrailsStepsEntry { + uint32_t key = 0; + TableList value; // Item: the element array, empty until an Add +}; + +// table Trails — TABLE-wire storage: relocatable, bounded, defaults in the +// member initializers (docs/SPEC-TABLES.md) +struct Trails { + TableMap steps; // map[uint32]*Item — the sorted entry array, empty until an insert + int32_t after = 0; +}; + +// ---- prefill: the declared defaults, in place (docs/SPEC-TABLES.md) ---- + +inline void TrailsStepsEntryReset( TrailsStepsEntry & value ); +inline void TrailsReset( Trails & value ); + +inline void TrailsStepsEntryReset( TrailsStepsEntry & value ) +{ + value.key = 0; + value.value.elements.value = 0; // Item: empty + value.value.count = 0; + value.value.padding = 0; +} + +inline void TrailsReset( Trails & value ) +{ + value.steps.entries.value = 0; // map[uint32]*Item: empty + value.steps.count = 0; + value.steps.padding = 0; + value.after = 0; +} + +template inline int64_t TrailsStepsEntryMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const TrailsStepsEntry & value ); +template inline bool TrailsStepsEntrySaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const TrailsStepsEntry & value ); +inline bool TrailsStepsEntryLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, TrailsStepsEntry & value ); +inline bool TrailsStepsEntryMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ); +template inline int64_t TrailsMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Trails & value ); +template inline bool TrailsSaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Trails & value ); +inline bool TrailsLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Trails & value ); +inline bool TrailsMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ); + +// TrailsStepsEntryMessageKeyRead: the key of one entry on the message wire, before the +// slot is chosen (docs/SPEC-TABLES.md §2.8, §3.3), and the bit the entry's +// body ends at. Field order inside a body is not contractual, so this scans +// the whole body by its announced shapes rather than assuming a position. +struct TrailsStepsEntryMessageKeyRead +{ + uint32_t key; + int64_t end; // the bit after the entry's own zero reference + bool found; // the body carried the key's id + bool kind_bad; // it carried it under another kind: the MAP's event + bool widened; // under a kind the declaration WIDENS (§4): decoded exactly, the MAP counts one + bool over; // longer than this reader's bound: the ENTRY is dropped + bool malformed; // the entry's framing gave out +}; + +inline TrailsStepsEntryMessageKeyRead TrailsStepsEntryMessageReadKey( TableBitReader r, const TableVocabulary & vocabulary, int64_t index_bits ) +{ + TrailsStepsEntryMessageKeyRead out = { 0, 0, false, false, false, false, false }; + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { out.malformed = true; return out; } + if ( ref == 0 ) { out.end = r.offset; return out; } // the terminator: no key field is the key's DEFAULT + if ( ref > (uint64_t) vocabulary.count ) { out.malformed = true; return out; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { out.malformed = true; return out; } + if ( entry.id == 0x3dc94a19365b10ecull ) // `key`, the ordinary hash of an ordinary name + { + const bool kind_bad = entry.kind != 8 && !TableKindWidens( entry.kind, 8 ); // THE KEY KIND IS THE READER'S DECLARATION + out.kind_bad = kind_bad; + out.found = !kind_bad; + out.widened = entry.kind != 8; + if ( kind_bad ) + { + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { out.malformed = true; return out; } + continue; + } + { + uint64_t raw = 0; + const int64_t width = entry.value_bits; + if ( width < 0 || !r.get( raw, width ) ) { out.malformed = true; return out; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + out.key = (uint32_t) decoded_wide; + } + continue; // the LAST occurrence is the one §3 keeps + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { out.malformed = true; return out; } + } +} + +// ---- the arena's reset hook (docs/SPEC-TABLES.md §6) ---- +// +// TableWorker::Alloc is a template and cannot name a member's Reset, so +// the arena reaches it through this overload set by argument-dependent +// lookup. It is how a node born in raw arena storage comes to hold the +// declared defaults without value-initialising the whole aggregate. + +inline void TableReset( TrailsStepsEntry & value ) { TrailsStepsEntryReset( value ); } +inline void TableReset( Trails & value ) { TrailsReset( value ); } + +// ---- pointer targets: allocation and resolution (docs/SPEC-TABLES.md §2) ---- +// +// A reference resolves differently in the two forms, and the CONTEXT says +// which: in the arena it is an offset; in a region it is a self-relative +// delta, so the const deref below is one add and needs no base pointer. + +// ---- codecs: measure/save/load per closure member ---- + +template inline int64_t TrailsStepsEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const TrailsStepsEntry & value ); +template inline bool TrailsStepsEntrySaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const TrailsStepsEntry & value ); +template inline bool TrailsStepsEntrySaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const TrailsStepsEntry & value ); +inline bool TrailsStepsEntryLoadBody( TableReader & r, const TableNodeMap & nodes, TrailsStepsEntry & value ); +template inline int64_t TrailsMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Trails & value ); +template inline bool TrailsSaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Trails & value ); +template inline bool TrailsSaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Trails & value ); +inline bool TrailsLoadBody( TableReader & r, const TableNodeMap & nodes, Trails & value ); + +// ---- pointer-graph walkers: number (measure/save), pack (Lock) ---- + +template inline bool TrailsStepsEntryNumber( const Ctx & ctx, TableNumbering & numbering, const TrailsStepsEntry & value ); +template inline int64_t TrailsStepsEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const TrailsStepsEntry & value ); +template inline bool TrailsStepsEntryPack( const Ctx & ctx, TablePackMap & seen, const TrailsStepsEntry & src, TrailsStepsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); +template inline bool TrailsNumber( const Ctx & ctx, TableNumbering & numbering, const Trails & value ); +template inline int64_t TrailsPackMeasure( const Ctx & ctx, TablePackMap & seen, const Trails & value ); +template inline bool TrailsPack( const Ctx & ctx, TablePackMap & seen, const Trails & src, Trails & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +// ---- the numbering's bridge to each member's codec (docs/SPEC-TABLES.md §3.1) ---- + +template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const TrailsStepsEntry & value ) { return TrailsStepsEntryMeasureBody( ctx, numbering, ids, value ); } +template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const TrailsStepsEntry & value ) { return TrailsStepsEntrySaveBody( ctx, numbering, w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const TrailsStepsEntry & value ) { return TrailsStepsEntryMeasureMessageBody( ctx, numbering, index_bits, at, value ); } +template inline bool TableNodeMessageSave( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const TrailsStepsEntry & value ) { return TrailsStepsEntrySaveMessageBody( ctx, numbering, index_bits, w, value ); } +template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Trails & value ) { return TrailsMeasureBody( ctx, numbering, ids, value ); } +template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Trails & value ) { return TrailsSaveBody( ctx, numbering, w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Trails & value ) { return TrailsMeasureMessageBody( ctx, numbering, index_bits, at, value ); } +template inline bool TableNodeMessageSave( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Trails & value ) { return TrailsSaveMessageBody( ctx, numbering, index_bits, w, value ); } + +// ---- TrailsStepsEntry: the order, the key and the value (docs/SPEC-TABLES.md §2.8) ---- +// +// The four overloads the map runtime's templates reach by argument-dependent +// lookup. Nothing outside this file names them. +static_assert( alignof( TrailsStepsEntry ) <= kTableAlign, "a map entry's alignment must fit the arena's" ); + +inline int TableEntryOrder( const TrailsStepsEntry & a, const TrailsStepsEntry & b ) +{ + return TableKeyOrder( (uint64_t) a.key, (uint64_t) b.key ); // integers compare by VALUE, unsigned here +} +inline int TableEntryOrder( const TrailsStepsEntry & entry, uint32_t key ) +{ + return TableKeyOrder( (uint64_t) entry.key, (uint64_t) key ); +} +inline uint32_t TableEntryKey( const TrailsStepsEntry & entry ) { return entry.key; } +inline void TableEntrySetKey( TrailsStepsEntry & entry, uint32_t key ) { entry.key = key; } +inline const TableList * TableEntryFound( const TrailsStepsEntry * entry ) { return entry != NULL ? &entry->value : NULL; } +inline TableList * TableEntryValue( TrailsStepsEntry * entry ) { return &entry->value; } +struct TrailsStepsEntryEach { uint32_t key; decltype( TableEntryValue( (TrailsStepsEntry *) NULL ) ) value; }; +inline TrailsStepsEntryEach TableEntryEach( TrailsStepsEntry * entry ) { return TrailsStepsEntryEach{ TableEntryKey( *entry ), TableEntryValue( entry ) }; } +inline void TableResetMapValue( TrailsStepsEntry & value ) +{ + value.value.elements.value = 0; // Item: empty + value.value.count = 0; + value.value.padding = 0; +} + +// TrailsStepsEntryReadKey: the key, before the slot is chosen (docs/SPEC-TABLES.md §2.8). +// Field order inside a body is not contractual (§3), so this scans rather +// than assuming a position — and this implementation writes the key first, +// so on any wire it wrote the scan ends at the first header. +struct TrailsStepsEntryKeyRead +{ + uint32_t key; + bool found; // the body carried the key's id + bool kind_bad; // it carried it under another kind: the MAP's event + bool widened; // under a kind the declaration WIDENS (§4): decoded exactly, the MAP counts one + bool over; // longer than this reader's bound: the ENTRY is dropped + bool malformed; // the entry's framing gave out +}; + +inline TrailsStepsEntryKeyRead TrailsStepsEntryReadKey( const uint8_t * body, int64_t length, const TableIdTable * ids ) +{ + TrailsStepsEntryKeyRead out = { 0, false, false, false, false, false }; + TableReport scratch; // the scan's own framing damage is the MAP's, raised by the caller + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { out.malformed = true; return out; } + if ( field_ref == 0 ) { return out; } // the terminator: no key field is the key's DEFAULT + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { out.malformed = true; return out; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { out.malformed = true; return out; } + uint8_t field_kind = r.get8(); + if ( field_id == 0x3dc94a19365b10ecull ) // `key`, the ordinary hash of an ordinary name + { + if ( field_kind != 8 && TableKindWidens( field_kind, 8 ) ) + { + out.widened = true; + out.found = true; + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, field_kind, widened_v ) ) { out.malformed = true; return out; } + out.key = (uint32_t) widened_v; + continue; // the LAST occurrence is the one §3 keeps + } + out.kind_bad = field_kind != 8; // THE KEY KIND IS THE READER'S DECLARATION + out.found = !out.kind_bad; + if ( !out.kind_bad ) + { + if ( !r.has( 4 ) ) { out.malformed = true; return out; } + out.key = (uint32_t) r.get32(); + continue; // the LAST occurrence is the one §3 keeps + } + } + if ( !r.skip( field_kind ) ) { out.malformed = true; return out; } + } +} + +// ---- retain-unknown: the second family (docs/SPEC-TABLES.md §6.6) ---- +// +// The same walks, with the PATH threaded and the unknown arm capturing. The +// three above are untouched and cost nothing for these being here: a caller +// that does not ask instantiates none of them. + +template inline int64_t TrailsStepsEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool TrailsStepsEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool TrailsStepsEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TrailsStepsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline int64_t TrailsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool TrailsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool TrailsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TrailsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Trails & value, TableRetain * retain, const TableRetainPath & path ); + +template +inline int64_t TrailsStepsEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const TrailsStepsEntry & value ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + { + // value: a kind 14 array of kind 17 elements, INDEX order (§2.9) + TableListCursor cursor_value = TableListElements( ctx, value.value ); + if ( !cursor_value.ok ) { return -1; } // the slot and the head disagree + if ( cursor_value.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( cursor_value.count ) ); // the element kind byte and the count + for ( int32_t elem_i_value = 0; elem_i_value < cursor_value.count; elem_i_value++ ) + { + { + const Item * slot_pointee_value = ItemAt( ctx, cursor_value[elem_i_value] ); + uint64_t slot_index_value = 0; + if ( slot_pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee_value, slot_index_value ) ) { return -1; } + body_value += TableLebBytes( slot_index_value ); + } + } + bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); + } + } + return bytes; +} + +template +inline bool TrailsStepsEntrySaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const TrailsStepsEntry & value ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 8 ); // key + w.put32( uint32_t( value.key ) ); + } + { + TableListCursor cursor_value = TableListElements( ctx, value.value ); // value + if ( !cursor_value.ok ) { return false; } + if ( cursor_value.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( cursor_value.count ) ); // the element kind byte and the count + for ( int32_t elem_i_value = 0; elem_i_value < cursor_value.count; elem_i_value++ ) + { + { + const Item * slot_pointee_value = ItemAt( ctx, cursor_value[elem_i_value] ); + uint64_t slot_index_value = 0; + if ( slot_pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee_value, slot_index_value ) ) { return false; } + body_value += TableLebBytes( slot_index_value ); + } + } + w.putleb( ref_value ); w.put8( 14 ); w.putleb( (uint64_t) body_value ); // value + w.put8( 17 ); w.putleb( (uint64_t) ( cursor_value.count ) ); + for ( int32_t elem_i_value = 0; elem_i_value < cursor_value.count; elem_i_value++ ) + { + { + const Item * slot_pointee_value = ItemAt( ctx, cursor_value[elem_i_value] ); + uint64_t slot_index_value = 0; + if ( slot_pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee_value, slot_index_value ) ) { return false; } + w.putleb( slot_index_value ); + } + } + } + } + return !w.overflow; +} + +template +inline bool TrailsStepsEntrySaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const TrailsStepsEntry & value ) +{ + if ( !TrailsStepsEntrySaveBodyFields( ctx, numbering, w, ids, value ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool TrailsStepsEntryLoadBody( TableReader & r, const TableNodeMap & nodes, TrailsStepsEntry & value ) +{ + TrailsStepsEntryReset( value ); // prefill declared defaults in place, then overlay + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x3dc94a19365b10ecull: // key + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.key = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.key = decoded_v; + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER is INERT (§4): the field keeps + // the value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + if ( !counted_ok ) { r.report->malformed = true; } + // AN ELEMENT KIND THAT DISAGREES with the reader's declaration is §3's + // element-kind rule: the field reads EMPTY and one kind_mismatch counts + else if ( elem_kind != 17 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + // THE COUNT IS THE DATA'S (§2.9): there is no bound, so clamped + // cannot fire on it. A count above the int32 storage cap is the + // fill's refusal, and it moves no counter. + TableListFill fill = TableListFillBegin( nodes, value.value, count ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + // elements are BOUNDED by the field body: a count the length cannot + // cover keeps the decoded prefix, flags malformed, and the parent + // continues at the next field + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + for ( uint64_t i = 0; i < count; i++ ) + { + TableRef * slot = TableListFillNext( fill ); + if ( slot == NULL ) { r.report->malformed = true; break; } // the arena could not carve + bool landed = false; + do + { + { + uint64_t node_index_value = 0; + if ( !sub.getleb( node_index_value ) ) { r.report->malformed = true; break; } + TableNodeResolve( nodes, ( *slot ), node_index_value, 0x52cfa1d198476806ull, r.report ); // *Item + } + landed = true; + } while ( 0 ); + if ( !landed ) { TableListFillDrop( fill ); break; } // the element's own framing gave out before it decoded + } + TableListFillEnd( fill ); + } + } + r.offset = body_end; // excess bytes and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's cost, in BITS (docs/SPEC-TABLES.md §3.3). `at` is the +// body's own bit position in the batch, because a `string(N)` ALIGNS before +// its bytes and an align costs what the position says it costs. +template +inline int64_t TrailsStepsEntryMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const TrailsStepsEntry & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + int64_t bits = 0; + if ( value.key != 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + } + { + TableListCursor cursor_value = TableListElements( ctx, value.value ); // value + if ( !cursor_value.ok ) { return -1; } // the slot and the head disagree + if ( cursor_value.count > 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + bits += (int64_t) ( cursor_value.count ) * index_bits; + } + } + bits += kTableMessageRefBitsHere; // the ZERO REFERENCE that ends the body + (void) at; + return bits; +} + +// The BITPACKED body: the fields, then the ZERO REFERENCE that ends it. No +// kind byte rides at all, and no length frames a nested body, because a +// body is self-delimiting: it is written where the file form put an L. +template +inline bool TrailsStepsEntrySaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const TrailsStepsEntry & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + if ( value.key != 0 ) + { + w.put( 9, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.key ), 32 ); + } + { + TableListCursor cursor_value = TableListElements( ctx, value.value ); // value + if ( !cursor_value.ok ) { return false; } // the slot and the head disagree + if ( cursor_value.count > 0 ) + { + w.put( 49, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( cursor_value.count ) - 0, 32 ); + for ( int32_t i = 0; i < cursor_value.count; i++ ) + { + const Item * pointee_value = ItemAt( ctx, cursor_value[i] ); // *Item + uint64_t index_value = 0; + if ( pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) pointee_value, index_value ) ) { return false; } + w.put( index_value, index_bits ); + } + } + } + w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +// TrailsStepsEntryMessageExtent: the extent TrailsStepsEntry's maps command on the message wire, from +// the FRAMING alone (docs/SPEC-TABLES.md §2.8, §3.3, §6.5). +inline bool TrailsStepsEntryMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { return false; } + if ( entry.id == 0x7ce4fd9430e80ceaull && entry.kind == 14 && entry.elem_kind == 17 ) // value: an unbounded array + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { return false; } + n += (uint64_t) entry.min; + if ( n > (uint64_t) INT32_MAX ) { return false; } // above the int32 storage cap (§2.9) + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( TableRef ) + at += (int64_t) n * (int64_t) sizeof( TableRef ); // the whole array FIRST + const int64_t run = TableMessageElementRunBits( vocabulary, entry ); + if ( run >= 0 ) { if ( !TableMessageSkipRun( r, n, run ) ) { return false; } } + else { for ( uint64_t i = 0; i < n; i++ ) { if ( !TableMessageSkipElement( r, vocabulary, index_bits, entry ) ) { return false; } } } + continue; + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { return false; } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +inline bool TrailsStepsEntryLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, TrailsStepsEntry & value ) +{ + (void) nodes; (void) index_bits; + TrailsStepsEntryReset( value ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.value, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + TableRef * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( *slot ), node_index_2, 0x52cfa1d198476806ull, report ); // *Item + } + } + TableListFillEnd( fill ); + } + break; + } + default: + report->unknown++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + } +} + +template +inline int64_t TrailsMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Trails & value ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // steps: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); + if ( !order_steps.ok ) { return -1; } // the sort could not run + if ( order_steps.count > 0 ) + { + const uint64_t ref_steps = ids.ref( 0x124250ad5a5b6d14ull ); + int64_t body_steps = 1 + TableLebBytes( (uint64_t) order_steps.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_steps = TrailsStepsEntryMeasureBody( ctx, numbering, ids, *order_steps[i] ); + if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return -1; } + body_steps += TableLebBytes( (uint64_t) ( elem_steps ) ) + ( elem_steps ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_steps ) + 1 + TableLebBytes( (uint64_t) ( body_steps ) ) + ( body_steps ); + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + return bytes; +} + +template +inline bool TrailsSaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Trails & value ) +{ + { + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); // steps + if ( !order_steps.ok ) { return false; } + if ( order_steps.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_steps = ids.ref( 0x124250ad5a5b6d14ull ); + int64_t body_steps = 1 + TableLebBytes( (uint64_t) order_steps.count ); + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_steps = TrailsStepsEntryMeasureBody( ctx, numbering, ids, *order_steps[i] ); + if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return false; } + body_steps += TableLebBytes( (uint64_t) ( elem_steps ) ) + ( elem_steps ); + } + w.putleb( ref_steps ); w.put8( 14 ); w.putleb( (uint64_t) body_steps ); + w.put8( 13 ); w.putleb( (uint64_t) order_steps.count ); + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_len_steps = TrailsStepsEntryMeasureBody( ctx, numbering, ids, *order_steps[i] ); + if ( elem_len_steps < 0 ) { TableMapRelease( order_steps ); return false; } + w.putleb( (uint64_t) elem_len_steps ); + if ( !TrailsStepsEntrySaveBody( ctx, numbering, w, ids, *order_steps[i] ) ) { TableMapRelease( order_steps ); return false; } + } + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + return !w.overflow; +} + +template +inline bool TrailsSaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Trails & value ) +{ + if ( !TrailsSaveBodyFields( ctx, numbering, w, ids, value ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool TrailsLoadBody( TableReader & r, const TableNodeMap & nodes, Trails & value ) +{ + TrailsReset( value ); // prefill declared defaults in place, then overlay + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x124250ad5a5b6d14ull: // steps + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + TableMapFill fill = TableMapFillBegin( nodes, value.steps, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + TrailsStepsEntryKeyRead read = TrailsStepsEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + TrailsStepsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + TrailsStepsEntryLoadBody( elem, nodes, *slot ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's cost, in BITS (docs/SPEC-TABLES.md §3.3). `at` is the +// body's own bit position in the batch, because a `string(N)` ALIGNS before +// its bytes and an align costs what the position says it costs. +template +inline int64_t TrailsMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Trails & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + int64_t bits = 0; + { + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); // steps + if ( !order_steps.ok ) { return -1; } // the sort could not run + if ( order_steps.count > 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; // the count the data decides + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_steps = TrailsStepsEntryMeasureMessageBody( ctx, numbering, index_bits, at + bits, *order_steps[i] ); + if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return -1; } + bits += elem_steps; // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + } + bits += kTableMessageRefBitsHere; // the ZERO REFERENCE that ends the body + (void) at; + return bits; +} + +// The BITPACKED body: the fields, then the ZERO REFERENCE that ends it. No +// kind byte rides at all, and no length frames a nested body, because a +// body is self-delimiting: it is written where the file form put an L. +template +inline bool TrailsSaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Trails & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + { + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); // steps + if ( !order_steps.ok ) { return false; } // the sort could not run + if ( order_steps.count > 0 ) + { + w.put( 48, kTableMessageRefBitsHere ); + w.put( (uint64_t) order_steps.count, 32 ); // the count the data decides + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + if ( !TrailsStepsEntrySaveMessageBody( ctx, numbering, index_bits, w, *order_steps[i] ) ) { TableMapRelease( order_steps ); return false; } + } + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) + { + w.put( 18, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.after ), 32 ); + } + w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +// TrailsMessageExtent: the extent Trails's maps command on the message wire, from +// the FRAMING alone (docs/SPEC-TABLES.md §2.8, §3.3, §6.5). +inline bool TrailsMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { return false; } + if ( entry.id == 0x124250ad5a5b6d14ull && entry.kind == 14 && entry.elem_kind == 13 ) // steps + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { return false; } + n += (uint64_t) entry.min; + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( TrailsStepsEntry ) + at += (int64_t) n * (int64_t) sizeof( TrailsStepsEntry ); // the whole array FIRST + for ( uint64_t i = 0; i < n; i++ ) // then, entry by entry in key order + { + if ( !TrailsStepsEntryMessageExtent( r, vocabulary, index_bits, at ) ) { return false; } + } + continue; + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { return false; } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +inline bool TrailsLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Trails & value ) +{ + (void) nodes; (void) index_bits; + TrailsReset( value ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x124250ad5a5b6d14ull: // steps + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + TableMapFill fill = TableMapFillBegin( nodes, value.steps, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const TrailsStepsEntryMessageKeyRead read = TrailsStepsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + TrailsStepsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !TrailsStepsEntryLoadMessageBody( r, vocabulary, report, nodes, index_bits, *slot ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + } +} + +// TrailsStepsEntryWireExtent: the extent TrailsStepsEntry's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool TrailsStepsEntryWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; // the scan's framing damage is the LOAD's to report + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { return true; } + if ( field_ref == 0 ) { return true; } + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { return true; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { return true; } + uint8_t field_kind = r.get8(); + if ( field_id == 0x7ce4fd9430e80ceaull && field_kind == 14 ) // value: an unbounded array + { + uint64_t list_len = 0; + if ( !r.getleb( list_len ) || !r.room( list_len ) ) { return true; } + const uint8_t * list_body = r.buffer + r.offset; + r.offset += (int64_t) list_len; + if ( !TableListWireExtent( list_body, (int64_t) list_len, at, (int64_t) sizeof( TableRef ), (int64_t) alignof( TableRef ), 17, 1, NULL, ids, reason ) ) { return false; } + continue; + } + if ( !r.skip( field_kind ) ) { return true; } + } +} + +// TrailsStepsEntryExtentAt: the node extent TrailsStepsEntry's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as TrailsStepsEntryExtentPack advances it (§2.8, §2.9). +template +inline bool TrailsStepsEntryExtentAt( const Ctx & ctx, const TrailsStepsEntry & value, int64_t & at ) +{ + { + TableListCursor cursor = TableListElements( ctx, value.value ); + if ( !cursor.ok ) { return false; } + at = ( at + (int64_t) alignof( TableRef ) - 1 ) & ~( (int64_t) alignof( TableRef ) - 1 ); + at += (int64_t) cursor.count * (int64_t) sizeof( TableRef ); // the whole array FIRST + } + return true; +} + +// the whole extent of one node, from a fresh offset: what a pack reserves +// for it beside the record's own storage. +template +inline int64_t TrailsStepsEntryExtent( const Ctx & ctx, const TrailsStepsEntry & value ) +{ + int64_t at = 0; + if ( !TrailsStepsEntryExtentAt( ctx, value, at ) ) { return -1; } + return at; +} + +// TrailsStepsEntryExtentPack: carve TrailsStepsEntry's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset TrailsStepsEntryExtentAt advances (§2.8, §2.9). +template +inline bool TrailsStepsEntryExtentPack( const Ctx & ctx, const TrailsStepsEntry & src, TrailsStepsEntry & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + { + TableListCursor cursor = TableListElements( ctx, src.value ); + if ( !cursor.ok ) { return false; } + at = ( at + (int64_t) alignof( TableRef ) - 1 ) & ~( (int64_t) alignof( TableRef ) - 1 ); + const int64_t bytes = (int64_t) cursor.count * (int64_t) sizeof( TableRef ); + if ( at + bytes > capacity ) { return false; } + TableRef * placed = (TableRef *) ( extent + at ); + at += bytes; + dst.value.count = cursor.count; + dst.value.padding = 0; + dst.value.elements.value = cursor.count > 0 ? (int64_t) ( (uint8_t *) placed - (const uint8_t *) &dst.value.elements ) : 0; + for ( int32_t i = 0; i < cursor.count; i++ ) // INDEX order, live elements only + { + memcpy( (void *) ( placed + i ), (const void *) &cursor[i], sizeof( TableRef ) ); // trivially copyable, by construction + } + } + return true; +} + +// TrailsWireExtent: the extent Trails's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool TrailsWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; // the scan's framing damage is the LOAD's to report + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { return true; } + if ( field_ref == 0 ) { return true; } + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { return true; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { return true; } + uint8_t field_kind = r.get8(); + if ( field_id == 0x124250ad5a5b6d14ull && field_kind == 14 ) // steps + { + uint64_t map_len = 0; + if ( !r.getleb( map_len ) || !r.room( map_len ) ) { return true; } + const uint8_t * map_body = r.buffer + r.offset; + r.offset += (int64_t) map_len; + if ( !TableMapWireExtent( map_body, (int64_t) map_len, at, (int64_t) sizeof( TrailsStepsEntry ), (int64_t) alignof( TrailsStepsEntry ), &TrailsStepsEntryWireExtent, ids, reason ) ) { return false; } + continue; + } + if ( !r.skip( field_kind ) ) { return true; } + } +} + +// TrailsExtentAt: the node extent Trails's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as TrailsExtentPack advances it (§2.8, §2.9). +template +inline bool TrailsExtentAt( const Ctx & ctx, const Trails & value, int64_t & at ) +{ + { + TableMapCursor cursor = TableMapOrder( ctx, value.steps ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( TrailsStepsEntry ) + at += (int64_t) cursor.count * (int64_t) sizeof( TrailsStepsEntry ); // the whole array FIRST + for ( int32_t i = 0; i < cursor.count; i++ ) // then, entry by entry in key order + { + if ( !TrailsStepsEntryExtentAt( ctx, *cursor[i], at ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// the whole extent of one node, from a fresh offset: what a pack reserves +// for it beside the record's own storage. +template +inline int64_t TrailsExtent( const Ctx & ctx, const Trails & value ) +{ + int64_t at = 0; + if ( !TrailsExtentAt( ctx, value, at ) ) { return -1; } + return at; +} + +// TrailsExtentPack: carve Trails's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset TrailsExtentAt advances (§2.8, §2.9). +template +inline bool TrailsExtentPack( const Ctx & ctx, const Trails & src, Trails & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + { + TableMapCursor cursor = TableMapOrder( ctx, src.steps ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; + const int64_t bytes = (int64_t) cursor.count * (int64_t) sizeof( TrailsStepsEntry ); + if ( at + bytes > capacity ) { TableMapRelease( cursor ); return false; } + TrailsStepsEntry * placed = (TrailsStepsEntry *) ( extent + at ); + at += bytes; + dst.steps.count = cursor.count; + dst.steps.padding = 0; + dst.steps.entries.value = cursor.count > 0 ? (int64_t) ( (uint8_t *) placed - (const uint8_t *) &dst.steps.entries ) : 0; + for ( int32_t i = 0; i < cursor.count; i++ ) + { + memcpy( (void *) ( placed + i ), (const void *) cursor[i], sizeof( TrailsStepsEntry ) ); // trivially copyable, by construction + } + for ( int32_t i = 0; i < cursor.count; i++ ) + { + if ( !TrailsStepsEntryExtentPack( ctx, *cursor[i], placed[i], extent, at, capacity ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// ---- Trails.steps: the builder's five and the side index (§2.8) ---- + +// INSERT: the key is copied, the value is handed back at its defaults to +// fill. A DUPLICATE key REPLACES — the value is reset and the same entry +// handed back, key and address unchanged — so a caller that wants to know +// writes Find first. NULL is NOT INSERTED: a key longer than the bound, +// because a truncated key would be a merged entry, and an arena that +// cannot carve another segment, alike. +// +// It is a WRAPPER: TableMapPlace owns the lookup, the reset, the +// allocation and the key copy, and this half is the bound and the +// const char * key's length. Nothing here mutates an entry (§2.8). +inline TableList * TrailsStepsInsert( TableWorker & worker, TableMap & map, uint32_t key ) +{ + TrailsStepsEntry * entry = TableMapPlace( worker, map, key ); + return entry != NULL ? TableEntryValue( entry ) : NULL; +} + +// FIND on the builder: the same linear scan, O( n ) key compares over the +// segments in insertion order. NULL when absent. The builder builds NO +// INDEX, and that is a rule — the sort happens once, at Lock, Save or +// Cook, and every lookup that matters runs over the sorted region. +inline TableList * TrailsStepsFind( TableArena & arena, TableMap & map, uint32_t key ) +{ + TrailsStepsEntry * found = TableMapScan( arena, map, key ); + return found != NULL ? TableEntryValue( found ) : NULL; +} + +// ERASE: marks the entry DEAD, one bit in the segment's slot and not in the +// entry table. False when absent. Its storage is held until the builder +// resets and never reused mid-build, because reusing a slot would make "an +// entry's address is stable" false for exactly one case. +inline bool TrailsStepsErase( TableArena & arena, TableMap & map, uint32_t key ) +{ + return TableMapErase( arena, map, key ); +} + +// EACH on the builder: INSERTION order, live entries only. +inline TableMapEach TrailsStepsEach( const TableArena & arena, const TableMap & map ) +{ + return TableMapEachOf( arena, map ); +} + +// ---- the OPTIONAL INDEX: caller-owned, built at load, never stored ---- +// +// Open addressing with linear probing over the sorted array, for a map large +// enough that log n compares over a cold array cost more than one hash and a +// probe. ITS HASH AND ITS LOAD FACTOR ARE NOT A CROSS-PORT CONTRACT: the +// index is never stored, so no golden, no cook-check rule and no +// build-version line ever names either. What a port is held to is the +// CONTRACT of the lookup — the same value the sorted array's Find returns +// for the same key, and no allocation past the storage the caller handed in. +inline int64_t TrailsStepsIndexMeasure( const TableMap & map ) +{ + return (int64_t) TableMapIndexSlots( map.count ) * (int64_t) sizeof( int32_t ); +} + +inline TableMapIndex TrailsStepsIndex( const TableMap & map, void * storage, int64_t bytes ) +{ + TableMapIndex index; + const int32_t slots = TableMapIndexSlots( map.count ); + if ( storage == NULL || bytes < (int64_t) slots * (int64_t) sizeof( int32_t ) ) { return index; } + index.slots = (int32_t *) storage; + index.capacity = slots; + for ( int32_t i = 0; i < slots; i++ ) { index.slots[i] = 0; } + const TrailsStepsEntry * entries = map.Entries(); + for ( int32_t i = 0; i < map.count; i++ ) // ONE PASS over the sorted array + { + int32_t at = (int32_t) ( TableMapHash( (uint64_t) entries[i].key ) & (uint64_t) ( slots - 1 ) ); + while ( index.slots[at] != 0 ) { at = ( at + 1 ) & ( slots - 1 ); } + index.slots[at] = i + 1; // slots are ENTRY INDICES; 0 is an empty slot + } + index.good = true; + return index; +} + +inline const TableList * TrailsStepsIndexFind( const TableMapIndex & index, const TableMap & map, uint32_t key ) +{ + if ( !index.good ) { return map.Find( key ); } // an index that did not build is not a wrong answer + const TrailsStepsEntry * entries = map.Entries(); + int32_t at = (int32_t) ( TableMapHash( (uint64_t) key ) & (uint64_t) ( index.capacity - 1 ) ); + for ( int32_t probe = 0; probe < index.capacity; probe++ ) + { + const int32_t slot = index.slots[at]; + if ( slot == 0 ) { return NULL; } + if ( TableEntryOrder( entries[slot - 1], key ) == 0 ) { return TableEntryFound( entries + slot - 1 ); } + at = ( at + 1 ) & ( index.capacity - 1 ); + } + return NULL; +} + +// ---- TrailsStepsEntry.value: the builder's three (§2.9) ---- + +// ADD: the element is appended and handed back to fill. On a []*T that is +// the SLOT at null, which ItemEmplace fills as it fills any pointer slot, +// and a second slot may hold the same reference: two slots, one node. +// NULL means NOT ADDED: an arena that cannot carve another segment, or a +// count at the int32 cap. A caller that needs the reason checks size(). +inline TableRef * TrailsStepsEntryValueAdd( TableWorker & worker, TableList & list ) +{ + return TableListPlace( worker, list ); +} + +// ERASE, by the element's own pointer: marks it DEAD, one bit in the +// segment's slot and not in the element storage. False when the pointer is +// not this list's. Storage is held until the builder resets. INDICES ARE +// NOT STABLE ACROSS AN ERASE: what was index 3 is index 2 in the next Save. +inline bool TrailsStepsEntryValueErase( TableArena & arena, TableList & list, const TableRef * element ) +{ + return TableListErase( arena, list, element ); +} + +// EACH on the builder: INDEX order, live elements only, yielding the +// element Add handed back. +inline TableListEach TrailsStepsEntryValueEach( const TableArena & arena, const TableList & list ) +{ + return TableListEachOf( arena, list ); +} + +// TrailsStepsEntryNumber: number everything TrailsStepsEntry POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool TrailsStepsEntryNumber( const Ctx & ctx, TableNumbering & numbering, const TrailsStepsEntry & value ) +{ + { // value: a by-value edge, elements in INDEX order (§2.9, §3.1) + TableListCursor cursor_value = TableListElements( ctx, value.value ); + if ( !cursor_value.ok ) { return false; } + for ( int32_t i = 0; i < cursor_value.count; i++ ) + { + { + const Item * pointee = ItemAt( ctx, cursor_value[i] ); // value + if ( pointee != NULL ) + { + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( numbering.seen, (const void *) pointee, + (int64_t) ( numbering.count + 2 ), taken, slot ); // its index, if this is its first visit + if ( entry == NULL ) { return false; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return false; } // a data cycle + } + else + { + TableNodeEntry node; + node.node = (const void *) pointee; + node.type_id = 0x52cfa1d198476806ull; // fnv1a64( "Item" ) + node.type_slot = 65; // its slot in the unit's vocabulary (§3.3) + node.measure = &TableNodeMeasureThunk; + node.save = &TableNodeSaveThunk; + node.message_measure = &TableNodeMessageMeasureThunk; + node.message_save = &TableNodeMessageSaveThunk; + if ( !TableNumberingAppend( numbering, node ) ) { return false; } + if ( !ItemNumber( ctx, numbering, *pointee ) ) { return false; } + TablePackMapClose( numbering.seen, (const void *) pointee, slot ); + } + } + } + } + } + return true; +} + +// TrailsStepsEntryPackMeasure: the packed region bytes of everything TrailsStepsEntry POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t TrailsStepsEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const TrailsStepsEntry & value ) +{ + int64_t bytes = 0; + { // value: a by-value edge, elements in INDEX order (§2.9, §3.1) + TableListCursor cursor_value = TableListElements( ctx, value.value ); + if ( !cursor_value.ok ) { return -1; } + for ( int32_t i = 0; i < cursor_value.count; i++ ) + { + { + const Item * pointee = ItemAt( ctx, cursor_value[i] ); // value + if ( pointee != NULL ) + { + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( seen, (const void *) pointee, 0, taken, slot ); + if ( entry == NULL ) { return -1; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return -1; } // a data cycle + } + else + { + int64_t inner = ItemPackMeasure( ctx, seen, *pointee ); + if ( inner < 0 ) { return -1; } + TablePackMapClose( seen, (const void *) pointee, slot ); + bytes += TableAlignUp64( (int64_t) sizeof( Item ) ) + inner; + } + } + } + } + } + return bytes; +} + +// TrailsStepsEntryPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool TrailsStepsEntryPackEdges( const Ctx & ctx, TablePackMap & seen, const TrailsStepsEntry & src, TrailsStepsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool TrailsStepsEntryPack( const Ctx & ctx, TablePackMap & seen, const TrailsStepsEntry & src, TrailsStepsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( TrailsStepsEntry ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( TrailsStepsEntry ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !TrailsStepsEntryExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return TrailsStepsEntryPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool TrailsStepsEntryPackEdges( const Ctx & ctx, TablePackMap & seen, const TrailsStepsEntry & src, TrailsStepsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + { // value: a by-value edge, elements in INDEX order (§2.9, §3.1) + TableListCursor cursor_value = TableListElements( ctx, src.value ); + if ( !cursor_value.ok ) { return false; } + TableRef * placed_value = (TableRef *) ( dst.value.elements.value != 0 ? ( (uint8_t *) &dst.value.elements + dst.value.elements.value ) : NULL ); + for ( int32_t i = 0; i < cursor_value.count; i++ ) + { + { + placed_value[i].value = 0; // value + const Item * pointee = ItemAt( ctx, cursor_value[i] ); + if ( pointee != NULL ) + { + int64_t at = TableAlignUp64( used ); // where it WOULD land, if this is its first visit + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( seen, (const void *) pointee, at, taken, slot ); + if ( entry == NULL ) { return false; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return false; } // a data cycle + placed_value[i].value = (int64_t) ( ( base + entry->offset ) - (const uint8_t *) &placed_value[i] ); // the one body it already has + } + else + { + if ( at + (int64_t) sizeof( Item ) > capacity ) { return false; } + used = at + TableAlignUp64( (int64_t) sizeof( Item ) ); + Item * child = new ( base + at ) Item; // lifetime only: the Pack below memcpy's the whole node over it + placed_value[i].value = (int64_t) ( ( base + at ) - (const uint8_t *) &placed_value[i] ); + if ( !ItemPack( ctx, seen, *pointee, *child, base, capacity, used ) ) { return false; } + TablePackMapClose( seen, (const void *) pointee, slot ); + } + } + } + } + } + return true; +} + +// TrailsNumber: number everything Trails POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool TrailsNumber( const Ctx & ctx, TableNumbering & numbering, const Trails & value ) +{ + { // steps: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_steps = TableMapOrder( ctx, value.steps ); + if ( !cursor_steps.ok ) { return false; } + for ( int32_t i = 0; i < cursor_steps.count; i++ ) + { + if ( !TrailsStepsEntryNumber( ctx, numbering, *cursor_steps[i] ) ) { TableMapRelease( cursor_steps ); return false; } + } + TableMapRelease( cursor_steps ); + } + return true; +} + +// TrailsPackMeasure: the packed region bytes of everything Trails POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t TrailsPackMeasure( const Ctx & ctx, TablePackMap & seen, const Trails & value ) +{ + int64_t bytes = 0; + { // steps: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_steps = TableMapOrder( ctx, value.steps ); + if ( !cursor_steps.ok ) { return -1; } + for ( int32_t i = 0; i < cursor_steps.count; i++ ) + { + int64_t inner = TrailsStepsEntryPackMeasure( ctx, seen, *cursor_steps[i] ); + if ( inner < 0 ) { TableMapRelease( cursor_steps ); return -1; } + bytes += inner; + } + TableMapRelease( cursor_steps ); + } + return bytes; +} + +// TrailsPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool TrailsPackEdges( const Ctx & ctx, TablePackMap & seen, const Trails & src, Trails & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool TrailsPack( const Ctx & ctx, TablePackMap & seen, const Trails & src, Trails & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( Trails ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( Trails ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !TrailsExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return TrailsPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool TrailsPackEdges( const Ctx & ctx, TablePackMap & seen, const Trails & src, Trails & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + { // steps: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_steps = TableMapOrder( ctx, src.steps ); + if ( !cursor_steps.ok ) { return false; } + TrailsStepsEntry * placed_steps = (TrailsStepsEntry *) ( dst.steps.entries.value != 0 ? ( (uint8_t *) &dst.steps.entries + dst.steps.entries.value ) : NULL ); + for ( int32_t i = 0; i < cursor_steps.count; i++ ) + { + if ( !TrailsStepsEntryPackEdges( ctx, seen, *cursor_steps[i], placed_steps[i], base, capacity, used ) ) { TableMapRelease( cursor_steps ); return false; } + } + TableMapRelease( cursor_steps ); + } + return true; +} + +// ---- Trails: the variable-length life (docs/SPEC-TABLES.md §2, §6, §9) ---- +// +// MUTABLE: TrailsBuilder — allocate nodes, wire them together, then Lock. +// CONST: one packed region, root at its base. Lock produces it and Load +// produces it, so a locked structure and a loaded one are the +// SAME representation with one view API. There is no unlock: +// re-editing means loading the const form into a fresh builder. +// Trails is never held by value — a file-format-scale structure is a region +// and a root pointer, not a struct you copy. + +struct TrailsBuilder +{ + TableArena arena; + TableWorker main; // the calling thread's allocation front + TableRef root_ref; + uint8_t * region = NULL; // the packed const form, produced by Lock() + int64_t region_bytes = 0; + + // THE ALLOCATOR IS THE BUILDER'S, and everything this structure ever + // allocates goes through it: the arena's segments, Lock's identity map, + // the packed region, the wire walks' numbering, and the tool path's node + // directory. Name your own and a profiler sees every byte under it. + TrailsBuilder( TableAllocator allocator = TableDefaultAllocator() ) + { + TableArenaInit( arena, allocator ); + main.arena = &arena; + TableSlot slot = main.Alloc(); + root_ref = slot.ref; + } + ~TrailsBuilder() { TableArenaShutdown( arena ); arena.allocator.free( arena.allocator.context, region ); } + TrailsBuilder( const TrailsBuilder & ) = delete; + TrailsBuilder & operator=( const TrailsBuilder & ) = delete; + + // Alloc a node in THIS thread's slab: no lock, no atomic per node. + // The result is usable both as the node pointer and as the reference + // to store in a pointer field. + template TableSlot Alloc() { return main.Alloc(); } + // a BYTE BUFFER's node of exactly `length` bytes (docs/SPEC-TABLES.md §2.5): + // the bytes to write through, and the reference to store in a *bytes + // or *string slot; a blob past a slab takes a span of its own + TableBytesSlot AllocBytes( int64_t length ) { return main.AllocBytes( length ); } + TableStringSlot AllocString( int64_t length ) { return main.AllocString( length ); } + // one worker per thread; allocate on your own, and synchronize your own + // writes to nodes another worker allocated + TableWorker Worker() { TableWorker worker; worker.arena = &arena; return worker; } + + // GetRoot/AsConst, not Root/Const: a member function hides the type + // name it shares, and `table Root` is this spec's own canonical + // example. The checker refuses a table named after any member here, + // so the remaining spellings cannot collide either. + Trails * GetRoot() { return arena.locked ? NULL : (Trails *) TableArenaAt( arena, (uint32_t) root_ref.value ); } + bool Locked() const { return arena.locked; } + const Trails * AsConst() const { return (const Trails *) region; } + const uint8_t * Region() const { return region; } + int64_t RegionBytes() const { return region_bytes; } + + // Lock is ONE WAY and it is the compaction: the segmented arena becomes + // one exact-packed region with zero slack, references rewritten + // self-relative, and the mutable life released. Single-threaded: call + // it after the workers have joined. + bool Lock(); +}; + +inline bool TrailsBuilder::Lock() +{ + if ( arena.locked ) { return region != NULL; } + if ( root_ref.null() ) { return false; } + TableArenaCtx ctx = { &arena }; + const Trails & root = *(const Trails *) TableArenaAt( arena, (uint32_t) root_ref.value ); + // The ROOT takes the map's first entry: it is packed at offset 0, and its + // descent is open for the whole walk (docs/SPEC-TABLES.md §3.1). + TablePackMap seen; + TablePackMapInit( seen, arena.allocator ); + bool root_taken = false; + int64_t root_slot = 0; + int64_t below = -1; + if ( TablePackMapReach( seen, (const void *) &root, 0, root_taken, root_slot ) != NULL ) + { + below = TrailsPackMeasure( ctx, seen, root ); + } + if ( below < 0 ) { TablePackMapShutdown( seen ); return false; } // a data cycle, named at the reference that closes it + int64_t root_extent = TrailsExtent( ctx, root ); + if ( root_extent < 0 ) { TablePackMapShutdown( seen ); return false; } // the sort could not run + int64_t total = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ) + below; + // the AUTHORING path may allocate (§6.5), and it does so through the + // builder's own pair. The region comes back ZEROED, which is the + // allocator's contract: a packed region carries node padding. + uint8_t * packed = (uint8_t *) arena.allocator.alloc( arena.allocator.context, total ); + if ( packed == NULL ) { TablePackMapShutdown( seen ); return false; } + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + Trails * destination = new ( packed ) Trails; // lifetime only: the Pack below memcpy's the whole node over it + // The pack walk RE-DERIVES the same numbering rather than carrying the + // measure's — nothing passes between them, which is what makes + // `used == total` below a real check and not a tautology (§3.1). The + // map keeps the capacity the measure paid for, so the second walk + // rehashes nothing. + TablePackMapReset( seen ); + if ( TablePackMapReach( seen, (const void *) &root, 0, root_taken, root_slot ) == NULL || + !TrailsPack( ctx, seen, root, *destination, packed, total, used ) || used != total ) + { + TablePackMapShutdown( seen ); + arena.allocator.free( arena.allocator.context, packed ); + return false; + } + TablePackMapShutdown( seen ); + region = packed; + region_bytes = total; + arena.locked = true; // MONOTONIC: there is no unlock + TableArenaShutdown( arena ); + return true; +} + +// ---- Trails on the wire: the FLAT NODE TABLE (docs/SPEC-TABLES.md §3.1) ---- +// +// A pointered save writes every reachable node ONCE, into a node table under +// the reserved id 0xFFFF, and a pointer field rides as a u32 INDEX into it +// under kind 17. No pointer edge is a nesting level, so a chain's length is +// not a depth and two references to one node are one node. + +// TrailsNodeStorage: the region bytes one record commands, or -1 for a type id +// this build cannot name — which keeps its index and reads null. A BYTE +// BUFFER's record commands its header and its bytes (docs/SPEC-TABLES.md §2.5), +// which is the one answer the record's LENGTH decides, and a blob past the +// size cap answers kTableNodeRefused with its reason (§3.1, §6.5). +// A MAP'S ENTRIES RIDE IN THEIR HOLDER'S EXTENT (docs/SPEC-TABLES.md §2.8), +// so a record's storage is its type's PLUS N x sizeof( Entry ) at every +// depth, summed from the FRAMING: N is framing and not a value, and this +// reads no field. kTableNodeRefused is a wire whose N its L cannot carry. +inline int64_t TrailsNodeStorage( uint64_t type_id, int64_t length, TableRefuseReason & reason ) +{ + (void) length; // no byte buffer below this root: every node's storage is its type's + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( (int64_t) sizeof( Item ) ); // Item + default: break; + } + (void) reason; // no blob and no extent below this root: nothing here refuses + return -1; +} + +// TrailsNodePlace: start one record's node's lifetime in the storage pass one +// reserved for it, holding exactly the declared defaults — a byte buffer's +// header holds its length, and its bytes come in pass two. +inline void TrailsNodePlace( uint64_t type_id, uint8_t * at, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: { Item * node = new ( at ) Item; ItemReset( *node ); break; } // Item + default: break; + } +} + +// TrailsNodeRecordBytes: one record's OWN storage, before the extent its maps +// take (docs/SPEC-TABLES.md §2.8) — where a node's extent begins. +inline int64_t TrailsNodeRecordBytes( uint64_t type_id ) +{ + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( (int64_t) sizeof( Item ) ); // Item + default: break; + } + return 0; +} + +// TrailsNodeAlloc: the TOOL's path — one record's node in the builder's arena. +// Zero is the arena's null, and it is also what a type id this build cannot +// name answers. +inline uint32_t TrailsNodeAlloc( uint64_t type_id, TableWorker & worker, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return (uint32_t) worker.Alloc().ref.value; // Item + default: break; + } + return 0; +} + +// TrailsNodeBody: PASS TWO's half — decode one record's body into the storage it +// already owns. +inline void TrailsNodeBody( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = TrailsNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? TrailsNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ItemLoadBody( r, *(Item *) at ); break; // Item + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// TrailsNodeMessageStorage: the region bytes one record commands on the message +// wire, or -1 for a type id this build cannot name. A table's is its own +// storage plus the extent its maps take; a byte buffer's is its header and +// its bytes, which is the one answer the record's LENGTH decides. +inline int64_t TrailsNodeMessageStorage( uint64_t type_id, int64_t extent, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Item ) ) + extent ); // Item + default: break; + } + return -1; +} + +// TrailsNodeMessageExtent: step over one TABLE record's body, tallying the extent +// its maps take where its type has any (§2.8). A type this build cannot +// name is stepped over by its announced shapes and takes no extent. +inline bool TrailsNodeMessageExtent( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & extent ) +{ + extent = 0; + (void) type_id; // no map below any node this root can name + return TableMessageSkipBody( r, vocabulary, index_bits ); +} + +// TrailsNodeMessageBody: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +inline bool TrailsNodeMessageBody( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at ) +{ + TableExtentCarve carve; + carve.at = at + TrailsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !TrailsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemLoadMessageBody( r, vocabulary, report, index_bits, *(Item *) at ); break; // Item + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// The numbering both wire walks derive, and NEITHER CARRIES THE OTHER'S: the +// root takes index 1 and its entry stays open for the whole walk, so a +// reference back at it is the cycle it is (§3.1). +template +inline bool TrailsNumberFrom( const Ctx & ctx, TableNumbering & numbering, const Trails & root ) +{ + bool taken = false; + int64_t slot = 0; + if ( TablePackMapReach( numbering.seen, (const void *) &root, (int64_t) kTableNodeIndexRoot, taken, slot ) == NULL ) { return false; } + return TrailsNumber( ctx, numbering, root ); +} + +template +inline int64_t TrailsMeasureWire( const Ctx & ctx, const Trails & root, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bytes = -1; + if ( TrailsNumberFrom( ctx, numbering, root ) ) + { + TableIds ids; + bytes = TrailsMeasureBody( ctx, numbering, ids, root ); + if ( bytes >= 0 ) + { + const int64_t table = TableNodeTableMeasure( ctx, ids, numbering ); + // the FORM BYTE, the ROOT BODY — its own fields, the node table + // and the terminator — and the ID TABLE (docs/SPEC-TABLES.md §3) + bytes = table < 0 || ids.overflow ? -1 : 1 + bytes + table + TableIdsBytes( ids ); + } + } + TableNumberingShutdown( numbering ); + return bytes; +} + +template +inline int64_t TrailsSaveWire( const Ctx & ctx, const Trails & root, uint8_t * buffer, int64_t capacity, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + if ( !TrailsNumberFrom( ctx, numbering, root ) ) { TableNumberingShutdown( numbering ); return -1; } + TableWriter w( buffer, capacity ); + TableIds ids; + w.put8( kTableWireForm ); // the FORM BYTE is the whole header (§3) + // the root's own fields, then the node table's field, then the + // terminator: a reader that gives up inside the table has already + // decoded the ROOT'S OWN FIELDS (§3.1) + bool ok = TrailsSaveBodyFields( ctx, numbering, w, ids, root ) && TableNodeTableSave( ctx, w, ids, numbering ); + TableNumberingShutdown( numbering ); + if ( !ok || ids.overflow ) { return -1; } + w.put8( 0 ); // the ZERO REFERENCE that ends the root body + TableIdsWrite( w, ids ); + if ( w.overflow ) { return -1; } // the caller's buffer was too small + return w.offset; // == TrailsMeasure( root ) +} + +inline int64_t TrailsMeasure( const Trails * root, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return TrailsMeasureWire( ctx, *root, allocator ); +} + +inline int64_t TrailsSave( const Trails * root, uint8_t * buffer, int64_t capacity, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return TrailsSaveWire( ctx, *root, buffer, capacity, allocator ); +} + +inline int64_t TrailsMeasure( const TrailsBuilder & builder ) +{ + if ( builder.region != NULL ) { return TrailsMeasure( builder.AsConst(), builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return TrailsMeasureWire( ctx, *(const Trails *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), builder.arena.allocator ); +} + +inline int64_t TrailsSave( const TrailsBuilder & builder, uint8_t * buffer, int64_t capacity ) +{ + if ( builder.region != NULL ) { return TrailsSave( builder.AsConst(), buffer, capacity, builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return TrailsSaveWire( ctx, *(const Trails *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), buffer, capacity, builder.arena.allocator ); +} + +// ---- Trails on the MESSAGE wire: the batch over a region (docs/SPEC-TABLES.md §3.3) ---- + +// TrailsMessageBodyBits: one root body's bits at bit position `at` of the batch, +// with the numbering derived from the graph, the node table FIRST, then the +// fields, then the zero reference. Measure derives the numbering and save +// derives the same one, and nothing passes between them (§3.1). +template +inline int64_t TrailsMessageBodyBits( const Ctx & ctx, const Trails & root, int64_t at, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bits = -1; + if ( TrailsNumberFrom( ctx, numbering, root ) ) + { + const int64_t index_bits = TableBitsRequired( 0, numbering.count + 1 ); + const int64_t table = TableMessageNodeTableMeasure( ctx, numbering, index_bits, at ); + if ( table >= 0 ) + { + const int64_t body = TrailsMeasureMessageBody( ctx, numbering, index_bits, at + table, root ); + bits = body < 0 ? -1 : table + body; + } + } + TableNumberingShutdown( numbering ); + return bits; +} + +template +inline bool TrailsMessageBodySave( const Ctx & ctx, const Trails & root, TableBitWriter & w, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + bool ok = false; + if ( TrailsNumberFrom( ctx, numbering, root ) ) + { + const int64_t index_bits = TableBitsRequired( 0, numbering.count + 1 ); + ok = TableMessageNodeTableSave( ctx, numbering, index_bits, w ) && TrailsSaveMessageBody( ctx, numbering, index_bits, w, root ); + } + TableNumberingShutdown( numbering ); + return ok && !w.overflow; +} + +// THE PRIMITIVE IS A BATCH (§3.3): a number of ROOTS in one buffer, one count +// and one continuous bit stream, each body carrying its own numbering. A +// root is a locked region's, `builder.AsConst()`, or a loaded one's. M above +// 256 is a refusal by name, batch_too_large, with nothing written. +inline int64_t TrailsMeasureMessages( const Trails * const * roots, int64_t count, TableReport * report, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( roots == NULL || count < 1 ) { return -1; } + if ( count > kTableMessageBatchMax ) { TableMessageRefuseBatch( report ); return -1; } + TableRegionCtx ctx; + int64_t bits = 8; // the body count + for ( int64_t i = 0; i < count; i++ ) + { + if ( roots[i] == NULL ) { return -1; } + const int64_t body = TrailsMessageBodyBits( ctx, *roots[i], bits, allocator ); + if ( body < 0 ) { return -1; } + bits += body; + } + return 1 + ( bits + 7 ) / 8; +} + +inline int64_t TrailsSaveMessages( const Trails * const * roots, int64_t count, uint8_t * buffer, int64_t capacity, TableReport * report, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( roots == NULL || count < 1 ) { return -1; } + if ( count > kTableMessageBatchMax ) { TableMessageRefuseBatch( report ); return -1; } + TableMessageBatch batch; + if ( !TableMessageBatchBegin( batch, buffer, capacity, count ) ) { return -1; } + TableRegionCtx ctx; + for ( int64_t i = 0; i < count; i++ ) + { + if ( roots[i] == NULL || !TrailsMessageBodySave( ctx, *roots[i], batch.w, allocator ) ) { return -1; } + batch.written++; + } + return TableMessageBatchEnd( batch ); // == TrailsMeasureMessages( roots, count, report, allocator ) +} + +// TrailsMessageRecordScan: one node record's type id and the extent its maps +// take, or a blob's length, the reader left after the record. A type id +// reference of 0, one past E, or one naming anything but a kind-0 entry is +// damage, as §3.1 and §3.3 say. +inline bool TrailsMessageRecordScan( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, uint64_t & type_id, int64_t & extent, int64_t & length ) +{ + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { return false; } + TableMessageEntry type_entry; + if ( !TableMessageNameEntry( vocabulary, type_ref, type_entry ) ) { return false; } + type_id = type_entry.id; + extent = 0; + length = 0; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + // A BLOB RECORD CARRIES A LENGTH AT THIRTY-TWO RAW BITS, then ALIGNS, + // then the bytes verbatim (§3.3) + uint64_t n = 0; + if ( !r.get( n, 32 ) || !r.align() || !r.skip( (int64_t) n * 8 ) ) { return false; } + length = (int64_t) n; + return true; + } + return TrailsNodeMessageExtent( type_id, r, vocabulary, index_bits, extent ); +} + +// TrailsMessageBodyStorage: one body's node count and data bytes from the FRAMING +// alone, the reader left at the next body. The node table is walked record +// by record, a table record's body stepped over by its announced shapes and +// a blob's by its length, then the root's own fields. False is a numbering +// that could not be sized; `complete` false is a ROOT body whose own framing +// gave out, which the load meets as damage inside this body after the +// bodies before it were delivered, so the batch is sized through this body +// and no further (§3.3). +inline bool TrailsMessageBodyStorage( TableBitReader & r, const TableVocabulary & vocabulary, int64_t & records, int64_t & data, bool & complete ) +{ + complete = true; + records = 0; + data = 0; + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { return false; } + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !TrailsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { return false; } + const int64_t storage = TrailsNodeMessageStorage( type_id, extent, length ); + if ( storage > 0 ) { data += storage; } // a type id this build cannot name commands none + records++; + } + int64_t root_extent = 0; + if ( !TrailsMessageExtent( r, vocabulary, index_bits, root_extent ) ) { complete = false; } + data += TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + return true; +} + +// TrailsLoadMeasure's MESSAGE overload: the exact region bytes ONE BATCH needs, +// which is one measurement, one allocation and one bounds check for however +// many bodies ride (§3.3, §6.5). It is a scan by the announced shapes and +// reads no field value. The answer is the data bytes plus the attribution, +// one node directory a body, and -1 for a wire it cannot size: no vocabulary, +// another form, or framing that gives out. +inline int64_t TrailsLoadMeasure( const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, int64_t * attribution_bytes = NULL ) +{ + TableReport ignored; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, &ignored ); + if ( bodies < 0 ) { return -1; } + int64_t data = 0, attribution = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + int64_t records = 0, body_data = 0; + bool complete = true; + if ( !TrailsMessageBodyStorage( br.r, vocabulary, records, body_data, complete ) ) { return -1; } + data += body_data; + attribution += ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( !complete ) { break; } // damage inside this body: the load delivers the ones before it + } + if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; } + return data + attribution; +} + +// TrailsLoadMessageBodyInto: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +inline bool TrailsLoadMessageBodyInto( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Trails * & root_out ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !TrailsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = TrailsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + TrailsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !TrailsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xb4578774a78fb150ull; + Trails * root = new ( region + used ) Trails; // lifetime only: LoadMessageBody's first act is TrailsReset + TrailsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Trails ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + out->unknown += unknown_records; + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !TrailsNodeMessageBody( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return TrailsLoadMessageBody( r, vocabulary, out, nodes, index_bits, *root ); +} + +// TrailsLoadMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +inline bool TrailsLoadMessages( const Trails ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !TrailsLoadMessageBodyInto( br.r, vocabulary, out, region, region_bytes, used, roots[b] ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + +// TrailsLoadMeasure: the exact region bytes a wire buffer will need, and it is +// ONE SCAN — a record's type id gives its storage size, its length gives the +// next record — reading no field value at all, so the caller owns the +// allocation and can refuse a number it did not expect (§6.5). +// +// It reports the DATA bytes and the ATTRIBUTION bytes separately, because the +// attribution is the wire's numbering made resident (§6.3) and a caller may +// release it once Load returns. The answer is their sum. +inline int64_t TrailsLoadMeasure( const uint8_t * wire_file, int64_t wire_file_bytes, int64_t * attribution_bytes = NULL, TableRefuseReason * reason_out = NULL ) +{ + TableReport ignored; + TableIdTable ids_table; + int64_t body_bytes = 0; + // a FORM BYTE this build does not carry is refused by name (§3, §6.5); + // a trailer that cannot be read whole is damage and names no reason + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict == TableOpenRefused ) { if ( reason_out != NULL ) { *reason_out = unknown_form; } return -1; } + if ( verdict != TableOpenOk ) { return -1; } + // ANY BYTE BETWEEN THE ROOT'S TERMINATOR AND THE TABLE'S FIRST ENTRY + // IS MALFORMED (docs/SPEC-TABLES.md §3): the two ends of the file have + // met, nothing is decoded, and no region is sized from it. + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) { return -1; } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &ignored, &ids_table ); + TableRefuseReason reason = count_over_length; + int64_t root_extent = 0; + if ( !TrailsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { if ( reason_out != NULL ) { *reason_out = reason; } return -1; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + int64_t records = 0; + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = TrailsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { if ( reason_out != NULL ) { *reason_out = reason; } return -1; } // an N the record's framing cannot carry, or a blob past the cap (§2.8, §2.9, §3.1) + if ( storage > 0 ) { data += storage; } // a type id this build cannot name commands none + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; } + return data + attribution; +} + +// TrailsLoad: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +inline const Trails * TrailsLoad( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Trails ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !TrailsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = TrailsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0xb4578774a78fb150ull; + Trails * root = new ( region ) Trails; // lifetime only: LoadBody's first act is TrailsReset + TrailsReset( *root ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = TrailsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + TrailsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + if ( nodes.good ) { out->unknown += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + TrailsNodeBody( type_id, sub, nodes, region + directory[k + 1].offset ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Trails ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + TrailsLoadBody( r, nodes, *root ); + return root; +} + +// TrailsLoadBuilder: the TOOL's path — the same tolerant decode into a fresh +// builder, so loaded data can be edited and locked again. The numbering is +// the same one; what differs is where a node lives and therefore what a +// resolved slot holds — an arena offset here, a self-relative delta there. +inline bool TrailsLoadBuilder( TrailsBuilder & builder, const uint8_t * wire_file, int64_t wire_file_bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; } + return false; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return false; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + Trails * root = builder.GetRoot(); + if ( root == NULL ) { out->malformed = true; return false; } + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) { records++; } + } + // the AUTHORING side may allocate (§6.5), and this is the tool's path. + // It goes through the builder's own pair, like everything else the + // builder reaches, and the entries come back zeroed. + const TableAllocator allocator = builder.arena.allocator; + TableNodeDirEntry * directory = (TableNodeDirEntry *) allocator.alloc( allocator.context, ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ) ); + if ( directory == NULL ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) builder.root_ref.value; + directory[0].type_id = 0xb4578774a78fb150ull; + TableNodeMap nodes; + nodes.base = NULL; + nodes.entries = directory; + nodes.count = records + 1; + nodes.arena = true; // a resolved slot holds the node's ARENA OFFSET here + nodes.worker = &builder.main; // and a map's entries and a list's elements are the arena's, not a node extent's (§2.8, §2.9) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + uint32_t at = TrailsNodeAlloc( type_id, builder.main, length ); + if ( at == 0 ) + { + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + } + else + { + directory[k + 1].offset = (uint64_t) at; + } + directory[k + 1].type_id = type_id; + k++; + } + nodes.good = TableNodeScanWhole( scan ); + if ( nodes.good ) { out->unknown += unknown_records; } else { out->malformed = true; } + } + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + TrailsNodeBody( type_id, sub, nodes, TableArenaAt( builder.arena, (uint32_t) directory[k + 1].offset ) ); + } + k++; + } + } + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.worker = &builder.main; + nodes.carve = &root_carve; + bool ok = TrailsLoadBody( r, nodes, *root ); + // A COUNT ABOVE THE int32 CAP is this path's refusal (docs/SPEC-TABLES.md + // §2.9): the partial builder is the caller's to discard, and the report + // holds what it held when the count was met + ok = ok && !nodes.refused; + allocator.free( allocator.context, directory ); + return ok; +} + +template +inline int64_t TrailsStepsEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + { + // value: a kind 14 array of kind 17 elements, INDEX order (§2.9) + TableListCursor cursor_value = TableListElements( ctx, value.value ); + if ( !cursor_value.ok ) { return -1; } // the slot and the head disagree + if ( cursor_value.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( cursor_value.count ) ); // the element kind byte and the count + for ( int32_t elem_i_value = 0; elem_i_value < cursor_value.count; elem_i_value++ ) + { + { + const Item * slot_pointee_value = ItemAt( ctx, cursor_value[elem_i_value] ); + uint64_t slot_index_value = 0; + if ( slot_pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee_value, slot_index_value ) ) { return -1; } + body_value += TableLebBytes( slot_index_value ); + } + } + bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); + } + } + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool TrailsStepsEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 8 ); // key + w.put32( uint32_t( value.key ) ); + } + { + TableListCursor cursor_value = TableListElements( ctx, value.value ); // value + if ( !cursor_value.ok ) { return false; } + if ( cursor_value.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( cursor_value.count ) ); // the element kind byte and the count + for ( int32_t elem_i_value = 0; elem_i_value < cursor_value.count; elem_i_value++ ) + { + { + const Item * slot_pointee_value = ItemAt( ctx, cursor_value[elem_i_value] ); + uint64_t slot_index_value = 0; + if ( slot_pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee_value, slot_index_value ) ) { return false; } + body_value += TableLebBytes( slot_index_value ); + } + } + w.putleb( ref_value ); w.put8( 14 ); w.putleb( (uint64_t) body_value ); // value + w.put8( 17 ); w.putleb( (uint64_t) ( cursor_value.count ) ); + for ( int32_t elem_i_value = 0; elem_i_value < cursor_value.count; elem_i_value++ ) + { + { + const Item * slot_pointee_value = ItemAt( ctx, cursor_value[elem_i_value] ); + uint64_t slot_index_value = 0; + if ( slot_pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee_value, slot_index_value ) ) { return false; } + w.putleb( slot_index_value ); + } + } + } + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool TrailsStepsEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !TrailsStepsEntrySaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool TrailsStepsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + TrailsStepsEntryReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x3dc94a19365b10ecull: // key + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.key = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.key = decoded_v; + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER is INERT (§4): the field keeps + // the value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + if ( !counted_ok ) { r.report->malformed = true; } + // AN ELEMENT KIND THAT DISAGREES with the reader's declaration is §3's + // element-kind rule: the field reads EMPTY and one kind_mismatch counts + else if ( elem_kind != 17 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + // THE COUNT IS THE DATA'S (§2.9): there is no bound, so clamped + // cannot fire on it. A count above the int32 storage cap is the + // fill's refusal, and it moves no counter. + TableListFill fill = TableListFillBegin( nodes, value.value, count ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + // elements are BOUNDED by the field body: a count the length cannot + // cover keeps the decoded prefix, flags malformed, and the parent + // continues at the next field + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + for ( uint64_t i = 0; i < count; i++ ) + { + TableRef * slot = TableListFillNext( fill ); + if ( slot == NULL ) { r.report->malformed = true; break; } // the arena could not carve + bool landed = false; + do + { + { + uint64_t node_index_value = 0; + if ( !sub.getleb( node_index_value ) ) { r.report->malformed = true; break; } + TableNodeResolve( nodes, ( *slot ), node_index_value, 0x52cfa1d198476806ull, r.report ); // *Item + } + landed = true; + } while ( 0 ); + if ( !landed ) { TableListFillDrop( fill ); break; } // the element's own framing gave out before it decoded + } + TableListFillEnd( fill ); + } + } + r.offset = body_end; // excess bytes and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +template +inline int64_t TrailsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // steps: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); + if ( !order_steps.ok ) { return -1; } // the sort could not run + if ( order_steps.count > 0 ) + { + const uint64_t ref_steps = ids.ref( 0x124250ad5a5b6d14ull ); + int64_t body_steps = 1 + TableLebBytes( (uint64_t) order_steps.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_steps = TrailsStepsEntryMeasureBodyRetain( ctx, numbering, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return -1; } + body_steps += TableLebBytes( (uint64_t) ( elem_steps ) ) + ( elem_steps ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_steps ) + 1 + TableLebBytes( (uint64_t) ( body_steps ) ) + ( body_steps ); + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool TrailsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + { + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); // steps + if ( !order_steps.ok ) { return false; } + if ( order_steps.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_steps = ids.ref( 0x124250ad5a5b6d14ull ); + int64_t body_steps = 1 + TableLebBytes( (uint64_t) order_steps.count ); + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_steps = TrailsStepsEntryMeasureBodyRetain( ctx, numbering, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return false; } + body_steps += TableLebBytes( (uint64_t) ( elem_steps ) ) + ( elem_steps ); + } + w.putleb( ref_steps ); w.put8( 14 ); w.putleb( (uint64_t) body_steps ); + w.put8( 13 ); w.putleb( (uint64_t) order_steps.count ); + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_len_steps = TrailsStepsEntryMeasureBodyRetain( ctx, numbering, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_len_steps < 0 ) { TableMapRelease( order_steps ); return false; } + w.putleb( (uint64_t) elem_len_steps ); + if ( !TrailsStepsEntrySaveBodyRetain( ctx, numbering, w, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_steps ); return false; } + } + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool TrailsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !TrailsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool TrailsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + TrailsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x124250ad5a5b6d14ull: // steps + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.steps, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + TrailsStepsEntryKeyRead read = TrailsStepsEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + TrailsStepsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + TrailsStepsEntryLoadBodyRetain( elem, nodes, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// TrailsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void TrailsNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = TrailsNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? TrailsNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ItemLoadBodyRetain( r, *(Item *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Item + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// TrailsLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +// UNDER RETENTION it also fills the caller's two stores with the fields +// this build cannot name, and the report carries what it could not keep +// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the +// reader's data is exactly what it would have been with retention off. +inline const Trails * TrailsLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Trails ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !TrailsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = TrailsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0xb4578774a78fb150ull; + Trails * root = new ( region ) Trails; // lifetime only: LoadBody's first act is TrailsReset + TrailsReset( *root ); + + // LoadRetain RESETS BOTH STORES and writes into neither id list: a + // retained record carries its field's identity in the record itself, + // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer + // belongs to this region from here on. + TableRetainReset( retain, nodes, region ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = TrailsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + TrailsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + TrailsNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Trails ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + TrailsLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// TrailsMeasureRetain and TrailsSaveRetain: the pair, with the retained tail in +// every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same +// records under the same walk, so Measure's answer is the size the save +// writes even where a record could not be placed. +template +inline int64_t TrailsMeasureWireRetain( const Ctx & ctx, const Trails & root, TableRetain * retain, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bytes = -1; + auto retain_measure = []( const Ctx & c, const TableNumbering & nn, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> int64_t + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemMeasureBodyRetain( ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return -1; + }; + if ( TrailsNumberFrom( ctx, numbering, root ) ) + { + TableRetainIds ids( retain ); + if ( retain != NULL ) { retain->id_used = 0; } // one walk fills the list, and the save's own walk refills it + bytes = TrailsMeasureBodyRetain( ctx, numbering, ids, root, retain, TableRetainPathRoot( (const void *) &root, 1 ) ); + if ( bytes >= 0 ) + { + const int64_t table = TableNodeTableMeasureRetain( ctx, ids, numbering, retain, retain_measure ); + bytes = table < 0 || ids.overflow ? -1 : 1 + bytes + table + TableRetainIdsBytes( ids ); + } + } + TableNumberingShutdown( numbering ); + return bytes; +} + +template +inline int64_t TrailsSaveWireRetain( const Ctx & ctx, const Trails & root, TableRetain * retain, uint8_t * buffer, int64_t capacity, TableReport * report ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, TableDefaultAllocator() ); + auto retain_measure = []( const Ctx & c, const TableNumbering & nn, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> int64_t + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemMeasureBodyRetain( ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return -1; + }; + auto retain_save = []( const Ctx & c, const TableNumbering & nn, TableWriter & ww, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> bool + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ww; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemSaveBodyRetain( ww, ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return false; + }; + if ( !TrailsNumberFrom( ctx, numbering, root ) ) { TableNumberingShutdown( numbering ); return -1; } + TableWriter w( buffer, capacity ); + TableRetainIds ids( retain ); + if ( retain != NULL ) { retain->id_used = 0; } + TableRetainClearPlaced( retain ); + w.put8( kTableWireForm ); // the FORM BYTE is the whole header (§3) + // the root's own fields, then the RETAINED TAIL, then the node table's + // field: a retained field is one of the root's own values, and the tail + // is pinned before the large and damage-prone part (§6.6, §3.1) + bool ok = TrailsSaveBodyFieldsRetain( ctx, numbering, w, ids, root, retain, TableRetainPathRoot( (const void *) &root, 1 ) ) && + TableNodeTableSaveRetain( ctx, w, ids, numbering, retain, retain_measure, retain_save ); + TableNumberingShutdown( numbering ); + if ( !ok || ids.overflow ) { return -1; } + w.put8( 0 ); // the ZERO REFERENCE that ends the root body + TableRetainIdsWrite( w, ids ); + if ( w.overflow ) { return -1; } // the caller's buffer was too small + // THE SAVE'S OWN SHARE OF retain_lost, read after the save (§6.6): every + // record the walk did not place, counted once. + TableRetainCountLost( retain, report ); + return w.offset; +} + +inline int64_t TrailsMeasureRetain( const Trails * root, TableRetain * retain, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return TrailsMeasureWireRetain( ctx, *root, retain, allocator ); +} + +// SaveRetain REFUSES A NULL REPORT and returns -1 (docs/SPEC-TABLES.md +// §6.6): the save is the only place a caller learns that a record was +// dropped, so the report is required here where it is optional everywhere +// else. A surface that let a caller retain, save and never find out would +// be a promise it could not check. +inline int64_t TrailsSaveRetain( const Trails * root, TableRetain * retain, uint8_t * buffer, int64_t capacity, TableReport * report ) +{ + if ( root == NULL || report == NULL ) { return -1; } + TableRegionCtx ctx; + return TrailsSaveWireRetain( ctx, *root, retain, buffer, capacity, report ); +} + +// TrailsSaveRetainMessages: RETENTION WRITING FORM 2 IS REFUSED BY NAME +// (docs/SPEC-TABLES.md §3.3). It is a MISUSE refusal on §6.6's own +// precedent and never a silent drop, and the two answers are named: a +// caller that must carry unknowns across a rewrite writes the FILE form, +// which carries its own table and takes §6.6 unchanged, and a RELAY +// forwards the sending peer's announcement and its batch bytes verbatim. +template +inline int64_t TrailsSaveRetainMessages( Args &&... ) +{ + static_assert( sizeof...( Args ) == (size_t) -1, + "Trails: a form 2 writer names entries through slots of a vocabulary the compiler settled, and a retained id is one this build's closure does not contain, so it has neither a slot nor an announced shape. Retention writing the MESSAGE form is refused by name (docs/SPEC-TABLES.md §3.3). Write the FILE form, which carries its own table and takes §6.6 unchanged, or relay the sender's announcement and batch bytes verbatim." ); + return -1; +} + +// ---- the cooked form: point at a cook (docs/SPEC-TABLES.md §7) ---- + +// TrailsOpen: match the header and POINT. On a match the bytes ARE what this +// build wrote, in this build's layout and this build's byte order, so there +// is nothing to validate and nothing to fix up and the root comes back as it +// lies. On ANY refusal it returns NULL and NAMES the refusal in the caller's +// TableRefuseReason, the first failing clause in §7's order (a wrong build +// version is a re-cook, a foreign order a cross-endian cook, a truncated +// file a bad download, an unaligned base the caller's own buffer), and the +// caller falls back to a wire load, which is the path that carries every +// version. The reason is written on the refusal path only; a caller that +// passes nothing gets the null alone. +// +// It is O(1) IN THE FILE'S SIZE — the header and nothing per node — so a one +// megabyte cook and a one gigabyte cook open in the same time, and a mapped +// file's pages are touched only as they are used. That is a property of +// touching nothing at open rather than a separate mechanism. +// +// A REFERENCE INSIDE THE REGION IS DEREFERENCED THROUGH TrailsAt: the slot holds +// the signed self-relative byte delta of §6.3, so a deref is one add and +// needs no base pointer, a whole region relocates by plain memcpy, and a +// delta of zero is null. +// +// There is ONE entry point and no tolerant twin: a build either wrote this +// file or it did not, and the build version is what says which. Validating a +// file whose provenance a person doubts is schema cook-check, offline, +// over the ATTRIBUTION part beside the data — a person's decision, never a +// parameter on a load. +inline const Trails * TrailsOpen( const void * bytes, uint64_t length, TableRefuseReason * reason = NULL ) +{ + return (const Trails *) TableCookOpen( bytes, length, (uint64_t) sizeof( Trails ), (uint64_t) alignof( Trails ), reason ); +} + +// ---- the cooked form: WRITE a cook (docs/SPEC-TABLES.md §7.6) ---- +// +// The bytes are `schema cook`'s, and the tool stays the reference: the two +// writers are held to one file, byte for byte, in both byte orders. A cook is +// content-addressed by (asset hash, build version), so two writers of one +// instance produce ONE artifact or the pair means nothing. + +template inline bool TrailsStepsEntryCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const TrailsStepsEntry & value, TableByteOrder order ); +template inline bool TrailsCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Trails & value, TableByteOrder order ); + +template inline bool TrailsStepsEntryCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const TrailsStepsEntry & value, TableByteOrder order ) +{ + (void) ctx; (void) region; // no reference resolves in this body: a list's and a map's slots are the extent writer's, and the class was decided elsewhere in the closure + table_cook_put( at + 0, (uint64_t) value.key, 4, order ); + table_cook_put( at + 8, 0, 8, order ); // value: the array's delta, filled by the extent writer + table_cook_put( at + 16, 0, 4, order ); // and its count + return true; +} + +template inline bool TrailsCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Trails & value, TableByteOrder order ) +{ + (void) ctx; (void) region; // no reference resolves in this body: a list's and a map's slots are the extent writer's, and the class was decided elsewhere in the closure + table_cook_put( at + 0, 0, 8, order ); // steps: the array's delta, filled by the extent writer + table_cook_put( at + 8, 0, 4, order ); // and its count + table_cook_put( at + 16, (uint64_t) value.after, 4, order ); + return true; +} + +template inline bool TrailsStepsEntryCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const TrailsStepsEntry & value, TableByteOrder order ); +template inline bool TrailsCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const Trails & value, TableByteOrder order ); + +// TrailsStepsEntryCookExtent: TrailsStepsEntry's arrays into the node's extent, PRE-ORDER, a map's entries +// in ASCENDING key order and a list's elements in INDEX order, each through its +// own cook writer (§2.8, §2.9, §7.6). +template inline bool TrailsStepsEntryCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const TrailsStepsEntry & value, TableByteOrder order ) +{ + { // value: an unbounded array + TableListCursor cursor = TableListElements( ctx, value.value ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( TableRef ) + uint8_t * array = extent + at; + at += (int64_t) cursor.count * 8; // the whole array FIRST + // the SIXTEEN BYTES of the slot: the self-relative delta, then the count + table_cook_put( record + 8, cursor.count > 0 ? (uint64_t) (int64_t) ( array - ( record + 8 ) ) : 0, 8, order ); + table_cook_put( record + 16, (uint64_t) (uint32_t) cursor.count, 4, order ); + for ( int32_t i = 0; i < cursor.count; i++ ) // INDEX order, live elements only + { + if ( !table_cook_ref( region, array + i * 8, (const void *) ItemAt( ctx, cursor[i] ), order ) ) { return false; } + } + } + return true; +} + +// TrailsCookExtent: Trails's arrays into the node's extent, PRE-ORDER, a map's entries +// in ASCENDING key order and a list's elements in INDEX order, each through its +// own cook writer (§2.8, §2.9, §7.6). +template inline bool TrailsCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const Trails & value, TableByteOrder order ) +{ + (void) region; // a table element's and an entry's references resolve through their own bodies + { // steps + TableMapCursor cursor = TableMapOrder( ctx, value.steps ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( TrailsStepsEntry ) + uint8_t * array = extent + at; + at += (int64_t) cursor.count * 24; // the whole array FIRST + // the SIXTEEN BYTES of the slot: the self-relative delta, then the count + table_cook_put( record + 0, cursor.count > 0 ? (uint64_t) (int64_t) ( array - ( record + 0 ) ) : 0, 8, order ); + table_cook_put( record + 8, (uint64_t) (uint32_t) cursor.count, 4, order ); + for ( int32_t i = 0; i < cursor.count; i++ ) + { + if ( !TrailsStepsEntryCookBody( ctx, region, array + i * 24, *cursor[i], order ) ) { return false; } + } + for ( int32_t i = 0; i < cursor.count; i++ ) // then, entry by entry in key order + { + if ( !TrailsStepsEntryCookExtent( ctx, region, extent, at, array + i * 24, *cursor[i], order ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// TrailsStepsEntryCookNode: one node, the record, then the extent its lists and maps take (§2.8, §2.9). +template inline bool TrailsStepsEntryCookNode( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const TrailsStepsEntry & value, TableByteOrder order ) +{ + if ( !TrailsStepsEntryCookBody( ctx, region, at, value, order ) ) { return false; } + int64_t extent_at = 0; + if ( !TrailsStepsEntryCookExtent( ctx, region, at + 24, extent_at, at, value, order ) ) { return false; } + return extent_at == TrailsStepsEntryExtent( ctx, value ); // the extent written is the extent measured, or no header is written +} + +// TrailsCookNode: one node, the record, then the extent its lists and maps take (§2.8, §2.9). +template inline bool TrailsCookNode( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Trails & value, TableByteOrder order ) +{ + if ( !TrailsCookBody( ctx, region, at, value, order ) ) { return false; } + int64_t extent_at = 0; + if ( !TrailsCookExtent( ctx, region, at + 24, extent_at, at, value, order ) ) { return false; } + return extent_at == TrailsExtent( ctx, value ); // the extent written is the extent measured, or no header is written +} + +// TrailsCookLayout: the tool's own Layout (docs/SPEC-TABLES.md §7.2) over one +// numbering — the root at zero, then every node in index order at +// align_up( offset, alignof ) for its OWN type, no slack between them, the +// data length rounded to the greatest alignment among them and never below +// eight. The offsets go into the region's table when it has one, and are only +// summed when it does not (a measure). A type id the numbering carries that +// this root cannot name is the two walks disagreeing, and it is refused. +// A NODE'S SIZE DEPENDS ON ITS VALUE where a list or a map rides in its extent +// (docs/SPEC-TABLES.md §2.8), so the layout takes the resolution context +// the numbering walked and reads the same arrays that walk read. +template +inline bool TrailsCookLayout( const Ctx & ctx, const Trails & root, const TableNumbering & numbering, TableCookRegion & region ) +{ + region.numbering = &numbering; + region.count = numbering.count + 1; + const int64_t root_extent = TrailsExtent( ctx, root ); + if ( root_extent < 0 ) { return false; } + int64_t offset = 24 + root_extent; // the root at zero, its extent behind it + int64_t align = 8; + if ( region.offsets != NULL ) { region.offsets[0] = 0; } + for ( int64_t k = 0; k < numbering.count; k++ ) + { + int64_t size = 0; + int64_t node_align = 0; + switch ( numbering.entries[k].type_id ) + { + case 0x52cfa1d198476806ull: size = 4; node_align = 4; break; // Item + default: return false; + } + offset = ( offset + node_align - 1 ) & ~( node_align - 1 ); + if ( region.offsets != NULL ) { region.offsets[k + 1] = offset; } + offset += size; + if ( node_align > align ) { align = node_align; } + } + region.bytes = ( offset + align - 1 ) & ~( align - 1 ); + region.align = align; + return true; +} + +// TrailsCookMeasureFrom: the whole cooked file's bytes for one graph — the header, +// the data part and the attribution part (§7.1). IT DEPENDS ON THE VALUE, +// because the answer is the numbering: the depth-first walk of §3.1 is run +// here and run again by the write, and neither carries the other's (§7.6). A +// data cycle is refused by the walk and answers -1. +template +inline int64_t TrailsCookMeasureFrom( const Ctx & ctx, const Trails & root, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + TableCookRegion region; + int64_t bytes = -1; + if ( TrailsNumberFrom( ctx, numbering, root ) && TrailsCookLayout( ctx, root, numbering, region ) ) + { + const int64_t data_offset = ( kTableCookHeaderBytes + region.align - 1 ) & ~( region.align - 1 ); + bytes = data_offset + region.bytes + region.count * (int64_t) sizeof( TableNodeDirEntry ); + } + TableNumberingShutdown( numbering ); + return bytes; +} + +// TrailsCookFrom: write one cooked file of a pointered graph, in the byte order +// the caller names. The bytes are `schema cook`'s, byte for byte (§7.6). +// +// THE CALLER OWNS THE OUTPUT and nothing is allocated toward it. What is +// allocated is the numbering — the identity map, the entry array and one +// offset per node — through the pair handed in, and released before this +// returns (§6.5, §13.9). A capacity short of the measure writes nothing. +// +// THE HEADER IS WRITTEN LAST. A reference the numbering did not carry is +// found while a body is being written, and a write that refuses there has +// already put bytes in the buffer; with no magic ahead of them, no Open can +// mistake them for a cook. +template +inline bool TrailsCookFrom( const Ctx & ctx, const Trails & root, void * out, uint64_t capacity, TableByteOrder order, TableAllocator allocator ) +{ + if ( out == NULL ) { return false; } + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + TableCookRegion region; + bool ok = TrailsNumberFrom( ctx, numbering, root ); + if ( ok ) + { + region.offsets = (int64_t *) allocator.alloc( allocator.context, ( numbering.count + 1 ) * (int64_t) sizeof( int64_t ) ); + ok = region.offsets != NULL && TrailsCookLayout( ctx, root, numbering, region ); + } + if ( ok ) + { + const int64_t data_offset = ( kTableCookHeaderBytes + region.align - 1 ) & ~( region.align - 1 ); + const int64_t attribution = region.count * (int64_t) sizeof( TableNodeDirEntry ); + const int64_t need = data_offset + region.bytes + attribution; + ok = (uint64_t) need <= capacity; + if ( ok ) + { + uint8_t * raw = (uint8_t *) out; + memset( raw, 0, (size_t) need ); // EVERY BYTE NO FIELD COVERS IS ZERO (§7.2) + region.base = raw + data_offset; + // the DATA part: the root at the region's base, then every numbered + // node at the offset the layout gave it, each through its own writer + ok = TrailsCookNode( ctx, region, region.base, root, order ); + for ( int64_t k = 0; ok && k < numbering.count; k++ ) + { + uint8_t * at = region.base + region.offsets[k + 1]; + const void * node = numbering.entries[k].node; + switch ( numbering.entries[k].type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemCookNode( ctx, region, at, *(const Item *) node, order ); break; // Item + default: ok = false; break; + } + } + // the ATTRIBUTION part: the node directory (§6.3), one entry per node + // in index order, for `schema cook-check` + uint8_t * entry = raw + data_offset + region.bytes; + table_cook_put( entry, 0, 8, order ); + table_cook_put( entry + 8, 0xb4578774a78fb150ull, 8, order ); // the root: fnv1a64( "Trails" ) + for ( int64_t k = 0; k < numbering.count; k++ ) + { + entry += sizeof( TableNodeDirEntry ); + table_cook_put( entry, (uint64_t) region.offsets[k + 1], 8, order ); + table_cook_put( entry + 8, numbering.entries[k].type_id, 8, order ); + } + // and the HEADER (§7.1), every word a u64 in the order the file is + // produced in; the two RESERVED words are the memset's zeros + if ( ok ) + { + table_cook_put( raw + 0, TableCookMagic, 8, order ); + table_cook_put( raw + 8, BuildVersion, 8, order ); + table_cook_put( raw + 16, (uint64_t) ( order == TableByteOrder::Big ? 2 : 1 ), 8, order ); + table_cook_put( raw + 24, (uint64_t) region.bytes, 8, order ); + table_cook_put( raw + 32, (uint64_t) attribution, 8, order ); + table_cook_put( raw + 40, (uint64_t) region.align, 8, order ); + } + } + } + allocator.free( allocator.context, region.offsets ); + TableNumberingShutdown( numbering ); + return ok; +} + +// TrailsCookMeasure / TrailsCook over a REGION root — a locked builder's AsConst, a +// region TrailsLoad produced, or an opened cook — with the pair the numbering +// allocates through as an optional last argument, as the wire's own entries +// take it (§13.9). +inline int64_t TrailsCookMeasure( const Trails * root, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return TrailsCookMeasureFrom( ctx, *root, allocator ); +} + +inline bool TrailsCook( const Trails * root, void * out, uint64_t capacity, TableByteOrder order, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return false; } + TableRegionCtx ctx; + return TrailsCookFrom( ctx, *root, out, capacity, order, allocator ); +} + +// and over a BUILDER, locked or not: the builder's own pair, and the arena +// encoding while it is still mutable (§6.3). +inline int64_t TrailsCookMeasure( const TrailsBuilder & builder ) +{ + if ( builder.region != NULL ) { return TrailsCookMeasure( builder.AsConst(), builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return TrailsCookMeasureFrom( ctx, *(const Trails *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), builder.arena.allocator ); +} + +inline bool TrailsCook( const TrailsBuilder & builder, void * out, uint64_t capacity, TableByteOrder order ) +{ + if ( builder.region != NULL ) { return TrailsCook( builder.AsConst(), out, capacity, order, builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return false; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return TrailsCookFrom( ctx, *(const Trails *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), out, capacity, order, builder.arena.allocator ); +} + +// ---- relocatability, enforced: the wire is a pure length-prefixed +// stream AND the decoded storage is pointer-free — every closure type +// must stay trivially copyable and standard-layout, so instances can be +// memcpy'd, mmap'd, shared across processes, and walked through +// descriptor offsets. A failure here means a pointer, virtual or +// non-trivial member crept into generated storage. +// +// They ask the COMPILER ITSELF, which is what every C++ standard library +// answers the same two questions with — and it costs this header no +// include at all. +// A pointer FIELD is a TableRef — eight bytes and no address — so the +// property holds in BOTH forms: a fixed-size table is one relocatable +// struct, and a packed region is one relocatable block whose references +// are self-relative and therefore survive a plain memcpy. +static_assert( __is_trivially_copyable( TrailsStepsEntry ), "TrailsStepsEntry must stay relocatable" ); +static_assert( __is_standard_layout( TrailsStepsEntry ), "TrailsStepsEntry must stay standard-layout for offsetof" ); +static_assert( __is_trivially_copyable( Trails ), "Trails must stay relocatable" ); +static_assert( __is_standard_layout( Trails ), "Trails must stay standard-layout for offsetof" ); + +// ---- the cook's layout contract (docs/SPEC-TABLES.md §20.3) ---- +// +// The compiler derived every number below from the declaration and folded it +// into the BUILD VERSION; these asserts are this compiler saying whether it +// agrees. The model is not self-evidently right — on 32-bit System V +// alignof(uint64_t) is 4, not 8 — which is precisely why it is asserted +// rather than assumed. +static_assert( sizeof( TrailsStepsEntry ) == 24, "TrailsStepsEntry's sizeof moved: the build version was taken over 24, so a cook of it would not be this build's file (docs/SPEC-TABLES.md §20.3)" ); +static_assert( alignof( TrailsStepsEntry ) == 8, "TrailsStepsEntry's alignof moved: the build version was taken over 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( TrailsStepsEntry, key ) == 0, "TrailsStepsEntry's field key moved: the build version was taken over offset 0 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( TrailsStepsEntry, value ) == 8, "TrailsStepsEntry's field value moved: the build version was taken over offset 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( sizeof( Trails ) == 24, "Trails's sizeof moved: the build version was taken over 24, so a cook of it would not be this build's file (docs/SPEC-TABLES.md §20.3)" ); +static_assert( alignof( Trails ) == 8, "Trails's alignof moved: the build version was taken over 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( Trails, steps ) == 0, "Trails's field steps moved: the build version was taken over offset 0 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( Trails, after ) == 16, "Trails's field after moved: the build version was taken over offset 16 (docs/SPEC-TABLES.md §20.3)" ); + +static_assert( alignof( TableRef ) <= kTableAlign, "TrailsStepsEntry.value: an unbounded array's element alignment must fit the arena's" ); + +// ---- reflection descriptors (tables only, docs/SPEC-TABLES.md) ---- + +inline const TableTypeInfo * TrailsStepsEntryTableType(); +inline const TableTypeInfo * TrailsTableType(); +// The descriptors are CONSTANT-INITIALISED data, and a field's target is +// the ADDRESS of another descriptor. These declarations are what let a +// self- or mutually-referential graph — Node naming itself through *Node — +// be expressed as constant data instead of a lazy link, which could not +// have been written race-free OR recursion-safe. The whole reflection +// surface is therefore immutable: read it from any thread, any time. +extern const TableTypeInfo TrailsStepsEntryTableInfo; +extern const TableTypeInfo TrailsTableInfo; + +inline const TableFieldInfo TrailsStepsEntryTableFields[] = { + { "key", "key", "uint32", 0x3dc94a19365b10ecull, 8, false, false, NULL, NULL, false, false, 0, (uint32_t) offsetof( TrailsStepsEntry, key ), (uint32_t) sizeof( TrailsStepsEntry::key ), 0xffffffffu, 0xffffffffu, NULL, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, + { "value", "value", "Item", 0x7ce4fd9430e80ceaull, 17, true, true, []( const void * slot ) -> const void * { return (const void *) ItemAt( *(const TableRef *) slot ); }, []( TableWorker & worker, void * slot ) -> void * { return (void *) ItemEmplace( worker, *(TableRef *) slot ); }, true, false, 0, (uint32_t) offsetof( TrailsStepsEntry, value ), (uint32_t) sizeof( TableRef ), (uint32_t) offsetof( TrailsStepsEntry, value.count ), 0xffffffffu, &ItemTableInfo, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, []( TableWorker & worker, void * slot, const char *, int32_t, int64_t ) -> void * { return (void *) TableListPlace( worker, *(TableList *) slot ); }, "", TableDocNone, 0, NULL }, +}; +inline const TableTypeInfo TrailsStepsEntryTableInfo = { "TrailsStepsEntry", (uint32_t) sizeof( TrailsStepsEntry ), 2, TrailsStepsEntryTableFields, +[]( void * p ) { TrailsStepsEntryReset( *(TrailsStepsEntry *) p ); }, true, TableDocNone, 0, NULL }; +inline const TableTypeInfo * TrailsStepsEntryTableType() { return &TrailsStepsEntryTableInfo; } + +inline const TableFieldInfo TrailsTableFields[] = { + { "steps", "steps", "map[uint32]*Item", 0x124250ad5a5b6d14ull, 13, true, false, NULL, NULL, true, false, 0, (uint32_t) offsetof( Trails, steps ), (uint32_t) sizeof( TrailsStepsEntry ), (uint32_t) offsetof( Trails, steps.count ), 0xffffffffu, &TrailsStepsEntryTableInfo, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, []( TableWorker & worker, void * slot, const char *, int32_t, int64_t key_value ) -> void * { return (void *) TableMapPlace( worker, *(TableMap *) slot, (uint32_t) key_value ); }, "", TableDocNone, 0, NULL }, + { "after", "after", "int32", 0xbf82010f6f71eae9ull, 4, false, false, NULL, NULL, false, false, 0, (uint32_t) offsetof( Trails, after ), (uint32_t) sizeof( Trails::after ), 0xffffffffu, 0xffffffffu, NULL, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, +}; +inline const TableTypeInfo TrailsTableInfo = { "Trails", (uint32_t) sizeof( Trails ), 2, TrailsTableFields, +[]( void * p ) { TrailsReset( *(Trails *) p ); }, true, TableDocNone, 0, NULL }; +inline const TableTypeInfo * TrailsTableType() { return &TrailsTableInfo; } + +// ---- the text form (docs/SPEC-TABLES.md §16) ---- + +// Trails in and out of a JSON text (docs/SPEC-TABLES.md §16.7): read into a +// builder, written from a region's const root. A node named more than once +// carries `&node` in the text. Defined in TrailsTable.cpp; link it to use them. +bool TrailsFromJson( TrailsBuilder & builder, const char * text, int64_t bytes, TableReport * report ); +int64_t TrailsToJsonMeasure( const Trails * root, TableAllocator allocator = TableDefaultAllocator() ); +int64_t TrailsToJson( const Trails * root, char * buffer, int64_t capacity, TableAllocator allocator = TableDefaultAllocator() ); + +} // namespace mapdemo diff --git a/testdata/wire/tables/map_conn.bin b/testdata/wire/tables/map_conn.bin index edea0d793..1435cfef8 100644 Binary files a/testdata/wire/tables/map_conn.bin and b/testdata/wire/tables/map_conn.bin differ diff --git a/testdata/wire/tables/map_crews.bin b/testdata/wire/tables/map_crews.bin new file mode 100644 index 000000000..26c364574 Binary files /dev/null and b/testdata/wire/tables/map_crews.bin differ diff --git a/testdata/wire/tables/map_full_message.bin b/testdata/wire/tables/map_full_message.bin index af1648afd..ba8b33658 100644 Binary files a/testdata/wire/tables/map_full_message.bin and b/testdata/wire/tables/map_full_message.bin differ diff --git a/testdata/wire/tables/map_pairs.bin b/testdata/wire/tables/map_pairs.bin new file mode 100644 index 000000000..32d8762ed Binary files /dev/null and b/testdata/wire/tables/map_pairs.bin differ diff --git a/testdata/wire/tables/map_trails.bin b/testdata/wire/tables/map_trails.bin new file mode 100644 index 000000000..dad875d02 Binary files /dev/null and b/testdata/wire/tables/map_trails.bin differ