From 5bfd34838302bb4cc36c7b8a224790c037a148d7 Mon Sep 17 00:00:00 2001 From: Rowan Date: Mon, 7 Sep 2026 05:41:11 -0400 Subject: [PATCH 1/5] tables: the map corpus for the three pointer-array value kinds (#666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three units under tables/maps, one per value kind whose element is a POINTER, each a map and a field past it so a stop is visible: Pairs map[uint32][2]*Item, Crews map[uint32][..2]*Item and Trails map[uint32][]*Item. `[E]*T`, `[N]*string` and `[]*bytes` are refused by name (§11), so this is the whole remaining set of #628's class. RED FIRST. Against this commit's emitter each generated header is refused alone, clang at the repo's own table flags over a translation unit that includes it and nothing else: PairsTable.h:6168: no matching function for call to 'ItemAt' candidate not viable: no known conversion from 'const TableRef[2]' to 'const TableRef' for 1st argument PairsTable.h:6169: cannot initialize return object of type 'TableRef *' with an rvalue of type 'TableRef (*)[2]' CrewsTable.h:6170: no matching function for call to 'ItemAt' candidate not viable: no known conversion from 'const TableRef[2]' to 'const TableRef' for 1st argument CrewsTable.h:6171: cannot initialize return object of type 'TableRef *' with an rvalue of type 'TableRef (*)[2]' TrailsTable.h:6171: no matching function for call to 'ItemAt' candidate not viable: no known conversion from 'const TableList' to 'const TableRef' for 1st argument TrailsTable.h:6172: cannot initialize return object of type 'TableRef *' with an rvalue of type 'TableList *' The entry's own storage is right in all three: §2.8 makes the entry a real table whose `value` is an ordinary field, so `TableRef value[2]`, that array beside `value_count`, and `TableList` are all already emitted, and so is the value reset that clears every slot. What has no spelling is the HANDLE: `mapValueIsPointer` tests `Type.Pointer` alone, so an array of pointers takes the `map[K]*T` arm and spells `At` on an array. Co-Authored-By: Claude Opus 5 --- tables/maps/Crews.schema | 14 ++++++++++++++ tables/maps/Pairs.schema | 15 +++++++++++++++ tables/maps/Trails.schema | 15 +++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 tables/maps/Crews.schema create mode 100644 tables/maps/Pairs.schema create mode 100644 tables/maps/Trails.schema diff --git a/tables/maps/Crews.schema b/tables/maps/Crews.schema new file mode 100644 index 000000000..265dacc55 --- /dev/null +++ b/tables/maps/Crews.schema @@ -0,0 +1,14 @@ +package mapdemo + +// A map whose VALUE is a COUNTED ARRAY OF POINTERS (docs/SPEC-TABLES.md §2.1, +// §2.8): "A VALUE is anything a table field can hold ... `[..N]T`", and the +// element is a pointer, so the value is an ordinary field of the generated +// entry and its storage is that field's own row, `TableRef value[N]` beside +// its `int32` used count (§4.2). Two members are not one addressable slot, so +// the handle every map surface hands back is the ENTRY, which reaches both. + +table Crews +{ + members map[uint32][..2]*Item // [..N]*T: pointer slots and the used count beside them + after int32 +} diff --git a/tables/maps/Pairs.schema b/tables/maps/Pairs.schema new file mode 100644 index 000000000..12e341bbb --- /dev/null +++ b/tables/maps/Pairs.schema @@ -0,0 +1,15 @@ +package mapdemo + +// A map whose VALUE is a FIXED ARRAY OF POINTERS (docs/SPEC-TABLES.md §2.1, +// §2.8): "A VALUE is anything a table field can hold ... `[N]T`", and the +// element is a pointer, so the value is an ordinary field of the generated +// entry and its storage is that field's own row, `TableRef value[N]` (§4.2). +// That is ONE member, so the handle points at the ARRAY and keeps the extent. +// Every slot names a node, and two slots may name one node exactly as two +// pointer fields may. + +table Pairs +{ + slots map[uint32][2]*Item // [N]*T: a fixed extent of pointer slots + after int32 +} diff --git a/tables/maps/Trails.schema b/tables/maps/Trails.schema new file mode 100644 index 000000000..c52fee00c --- /dev/null +++ b/tables/maps/Trails.schema @@ -0,0 +1,15 @@ +package mapdemo + +// A map whose VALUE is an UNBOUNDED ARRAY OF POINTERS (docs/SPEC-TABLES.md +// §2.1, §2.8, §2.9): "A VALUE is anything a table field can hold ... and an +// unbounded `[]T` (§2.9)", and the element is a pointer, so the value is an +// ordinary field of the generated entry and its storage is that field's own +// row, the sixteen-byte list slot over `TableRef` elements (§4.2). That is one +// member, so the handle is the slot itself and the list's own surface reaches +// the pointer slots, which ride in the HOLDER'S node extent as any list's do. + +table Trails +{ + steps map[uint32][]*Item // []*T: an unbounded extent of pointer slots + after int32 +} From cfeeeecf1e42baca96cc9642fd9219cc64e26505 Mon Sep 17 00:00:00 2001 From: Rowan Date: Mon, 7 Sep 2026 05:46:51 -0400 Subject: [PATCH 2/5] tables: map values that are arrays of pointers (#666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The handle every map surface hands back now follows the ARRAY FORM and not the element, so `[N]*T`, `[..N]*T` and `[]*T` under a map key emit C++ that compiles (docs/SPEC-TABLES.md §2.1, §2.8, §4.2). The entry's own storage was right in all three, and so was its value reset: §2.8 makes the entry a real table whose `value` is an ordinary field, so `TableRef value[2]`, that array beside `value_count`, and `TableList` were all already emitted, and the codec is the field's own on every walk. What had no spelling was the HANDLE, because `mapValueIsPointer` tested `Type.Pointer` alone and sent all three to the `map[K]*T` arm, which spells `At` on an array. - the predicate is now pointer AND no array AND no enum key, so it reports the ONE slot it was written for; - `[N]*T` takes the fixed-array arm and its element is a `TableRef`, so the handle is `Value *` over `TableRef[N]`; - `[..N]*T` takes the pair arm, because a counted array is a pair whatever its element is: the count beside the slots is the second member, so the handle is the ENTRY; - `[]*T` takes the list arm, whose storage already answered `TableList` through the list emitter's own type argument. `[E]*T`, `[N]*string` and `[]*bytes` are refused by name at the entry's value (§2.4, §15), so this is the whole set. RED FIRST, against this commit's parent, in 5bfd3483's message: clang at the repo's own table flags refused each of the three generated headers alone, on `no matching function for call to 'ItemAt'` and on a `TableRef *` that cannot be initialized from `TableRef (*)[2]` or `TableList *`. BOTH ENGINES. `test/tables/maps_main.cpp` gains three rows, one per unit, each pinning its wire, holding `measure == save`, loading the region, re-saving it byte for byte, and reading the text form back to the same wire. Each instance carries a SHARED node and a slot that names NONE, so the `&node` label and the `null` row of §16.7 both ride through a map value. `test/conformance/harness/maps_test.go` gains the three names, so the compiler's own engine reads the reference's bytes, writes them back byte for byte, renders the text and reads that text to the same bytes; the engine needed no change, because #662's by-value map edge already descends an entry and the entry's value is an ordinary pointer-array field. Co-Authored-By: Claude Opus 5 --- internal/codegen/cpptable/maps.go | 69 +++-- test/conformance/harness/maps_test.go | 4 + test/tables/maps_main.cpp | 350 ++++++++++++++++++++++++++ testdata/wire/tables/map_crews.bin | Bin 0 -> 137 bytes testdata/wire/tables/map_pairs.bin | Bin 0 -> 129 bytes testdata/wire/tables/map_trails.bin | Bin 0 -> 129 bytes 6 files changed, 402 insertions(+), 21 deletions(-) create mode 100644 testdata/wire/tables/map_crews.bin create mode 100644 testdata/wire/tables/map_pairs.bin create mode 100644 testdata/wire/tables/map_trails.bin diff --git a/internal/codegen/cpptable/maps.go b/internal/codegen/cpptable/maps.go index 26d5af7df..71bb628a1 100644 --- a/internal/codegen/cpptable/maps.go +++ b/internal/codegen/cpptable/maps.go @@ -82,24 +82,35 @@ func mapKeyOrderType(f *ir.Field) string { return "uint64_t" } -// mapValueIsPointer reports a `map[K]*T` — the value is a pointer SLOT, so the -// const form's Find answers the resolved `const T *` (docs/SPEC-TABLES.md §2.8). -func mapValueIsPointer(f *ir.Field) bool { return ir.MapValueField(f).Type.Pointer } +// mapValueIsPointer reports a `map[K]*T` — the value is ONE pointer SLOT, so +// the const form's Find answers the resolved `const T *` (docs/SPEC-TABLES.md +// §2.8). AN ARRAY OF POINTERS IS NOT THIS CASE. `[N]*T`, `[..N]*T` and `[]*T` +// store a `TableRef` PER ELEMENT (§2.1, §4.2), so each takes the arm its array +// form takes and carries `TableRef` as the element; a predicate that read +// `Type.Pointer` alone sent all three here and spelled `At` on an array. +// `[E]*T` is refused by name (§11) and reaches no arm at all. +func mapValueIsPointer(f *ir.Field) bool { + value := ir.MapValueField(f) + return value.Type.Pointer && value.Array == ir.ArrayNone && value.KeyEnum == "" +} // mapValueIsPair reports a value whose storage is TWO MEMBERS: the buffer or // the element array, and the `int32` length or count beside it that says how // much of it is used (docs/SPEC-TABLES.md §2.8, §4.2, §7.2). That is -// `string(N)`, `wstring(N)`, `bytes(N)` and `[..N]T`. Each is an ordinary -// field of the generated entry, and two members are not one addressable slot, -// so the handle every map surface hands back is the ENTRY, which reaches both. +// `string(N)`, `wstring(N)`, `bytes(N)`, `[..N]T` and `[..N]*T`. Each is an +// ordinary field of the generated entry, and two members are not one +// addressable slot, so the handle every map surface hands back is the ENTRY, +// which reaches both. A COUNTED ARRAY IS A PAIR WHATEVER ITS ELEMENT IS: a +// pointer element changes the element's storage to a `TableRef` and changes +// nothing about the count beside it. func mapValueIsPair(f *ir.Field) bool { value := ir.MapValueField(f) - if value.Type.Pointer { - return false - } if value.Array == ir.ArrayCounted { return true } + if value.Type.Pointer { + return false + } switch value.Type.Kind { case ir.TString, ir.TWString, ir.TBytes: return true @@ -107,13 +118,13 @@ func mapValueIsPair(f *ir.Field) bool { return false } -// mapValueIsFixedArray reports a `[N]T` value. Its storage is ONE member, an -// array of a fixed extent (docs/SPEC-TABLES.md §4.2), and a pointer to that -// member keeps the extent. `[E]T` is not this case: it stores a +// mapValueIsFixedArray reports a `[N]T` or `[N]*T` value. Its storage is ONE +// member, an array of a fixed extent (docs/SPEC-TABLES.md §4.2), and a pointer +// to that member keeps the extent. `[E]T` is not this case: it stores a // `TableKeyed`, which is an ordinary one-member type spelling. func mapValueIsFixedArray(f *ir.Field) bool { value := ir.MapValueField(f) - return !value.Type.Pointer && value.KeyEnum == "" && value.Array == ir.ArrayFixed + return value.KeyEnum == "" && value.Array == ir.ArrayFixed } // mapValueArrayAlias names the array type a `[N]T` value's handle points at. @@ -121,6 +132,20 @@ func mapValueIsFixedArray(f *ir.Field) bool { // around the function name, so the alias is what makes the handle readable. func mapValueArrayAlias(f *ir.Field) string { return mapEntryOf(f).Name + "Value" } +// mapValueElementType is the C++ type ONE ELEMENT of an array value is stored +// as: a `TableRef` where the element is a pointer, which is the row §4.2 gives +// an array of pointers at a field, and the element's own type otherwise +// (docs/SPEC-TABLES.md §2.1, §4.2). +func (g *tableGen) mapValueElementType(f *ir.Field) string { + value := ir.MapValueField(f) + if value.Type.Pointer { + g.noteRef(value.Type.Name) + return "TableRef" + } + typ, _ := g.cppFieldType(value.Type) + return typ +} + // ---- the runtime (docs/SPEC-TABLES.md §2.8) ---- // tableMapRuntime is the map half of the variable-length runtime: the storage @@ -886,10 +911,10 @@ func (g *tableGen) emitMapEntrySurface(owner *ir.Struct, f *ir.Field) { g.pf("inline const %s * TableEntryFound( const %s * entry ) { return entry != NULL ? %sAt( entry->value ) : NULL; }\n", t, n, t) g.pf("inline TableRef * TableEntryValue( %s * entry ) { return &entry->value; } // the builder hands back the SLOT\n", n) case mapValueIsPair(f): - // a map[K]string(N), map[K]wstring(N), map[K]bytes(N) or map[K][..N]T: - // the value's storage is the buffer or the element array and the int32 - // used length or count beside it (§2.8, §4.2, §7.2), two members, so - // the ENTRY is the handle that reaches both. A caller fills + // a map[K]string(N), map[K]wstring(N), map[K]bytes(N), map[K][..N]T or + // map[K][..N]*T: the value's storage is the buffer or the element array + // and the int32 used length or count beside it (§2.8, §4.2, §7.2), two + // members, so the ENTRY is the handle that reaches both. A caller fills // `entry->value` and its companion, and leaves `entry->key` to the // map, which owns the sort the key carries. g.pf("// THIS VALUE'S STORAGE IS A PAIR (§2.8, §4.2, §7.2): `value` beside\n") @@ -898,10 +923,12 @@ func (g *tableGen) emitMapEntrySurface(owner *ir.Struct, f *ir.Field) { g.pf("inline const %s * TableEntryFound( const %s * entry ) { return entry; }\n", n, n) g.pf("inline %s * TableEntryValue( %s * entry ) { return entry; }\n", n, n) case mapValueIsFixedArray(f): - // a map[K][N]T: the value's storage is ONE member, an array of a fixed - // extent (§4.2), so the handle is a pointer to the ARRAY and keeps - // that extent. The alias is what lets a return type spell it. - elem, _ := g.cppFieldType(value.Type) + // a map[K][N]T or map[K][N]*T: the value's storage is ONE member, an + // array of a fixed extent (§4.2), so the handle is a pointer to the + // ARRAY and keeps that extent. A pointer element makes that element a + // `TableRef` (§2.1) and changes nothing else. The alias is what lets a + // return type spell it. + elem := g.mapValueElementType(f) alias := mapValueArrayAlias(f) g.pf("// A FIXED ARRAY VALUE IS ONE MEMBER (§2.8, §4.2): the handle points at the\n") g.pf("// ARRAY and not at its first element, so the extent survives the handoff.\n") diff --git a/test/conformance/harness/maps_test.go b/test/conformance/harness/maps_test.go index e324d4b89..1ad9b8f50 100644 --- a/test/conformance/harness/maps_test.go +++ b/test/conformance/harness/maps_test.go @@ -36,6 +36,10 @@ func TestTheToolWritesTheReferencesMapBytes(t *testing.T) { // unbounded array, a text buffer and a byte buffer, each under a map key {"map_cells", "Cells"}, {"map_runs", "Runs"}, {"map_slots", "Slots"}, {"map_spans", "Spans"}, {"map_docs", "Docs"}, {"map_chunks", "Chunks"}, + // one row per value kind whose ELEMENT is a pointer (#666): a fixed + // array, a counted array and an unbounded array of node references, + // each carrying a shared node and a slot that names none + {"map_pairs", "Pairs"}, {"map_crews", "Crews"}, {"map_trails", "Trails"}, } { name, rootName := tc.name, tc.root data, err := os.ReadFile(root + "testdata/wire/tables/" + name + ".bin") diff --git a/test/tables/maps_main.cpp b/test/tables/maps_main.cpp index 0b3796ff2..f925a3de2 100644 --- a/test/tables/maps_main.cpp +++ b/test/tables/maps_main.cpp @@ -22,6 +22,9 @@ #include "SpansTable.h" #include "DocsTable.h" #include "ChunksTable.h" +#include "PairsTable.h" +#include "CrewsTable.h" +#include "TrailsTable.h" #include "wirebuilder.h" using namespace mapdemo; @@ -2085,6 +2088,350 @@ static void test_byte_blob_values() free( chunk_json ); } +// ---- A MAP WHOSE VALUE IS AN ARRAY OF POINTERS (§2.1, §2.8, §4.2) ---- +// +// The element is a pointer, so the value's storage is the array of `TableRef` +// §4.2 gives an array of pointers at a field, and the handle follows the ARRAY +// FORM and not the element: the array for a `[N]*T`, the ENTRY for a `[..N]*T` +// because the count beside it makes two members, and the list SLOT for a +// `[]*T`. Each slot names a node of the one declaration-order walk, so two +// slots may name one node and a slot may name none. + +static void test_pointer_array_values() +{ + // [2]*Item: ONE member, TableRef value[2], so the handle is a pointer to + // the ARRAY. Key 4 spends both slots on ONE node, key 9 leaves the second + // null, so a share and a null both ride through a map value. + PairsBuilder b; + Pairs * p = b.GetRoot(); + static const uint32_t keys[2] = { 9, 4 }; // OUT OF KEY ORDER + for ( int i = 0; i < 2; i++ ) + { + PairsSlotsEntryValue * slots = PairsSlotsInsert( b.main, p->slots, keys[i] ); + CHECK( slots != NULL ); + if ( slots == NULL ) { return; } + Item * item = ItemEmplace( b.main, ( *slots )[0] ); + CHECK( item != NULL ); + if ( item == NULL ) { return; } + item->count = (int32_t) ( keys[i] * 10 ); + // key 4 names the same node twice; key 9 leaves the second slot null + if ( keys[i] == 4 ) { ( *slots )[1] = ( *slots )[0]; } + } + p->after = 5; + + PairsSlotsEntryValue * found = PairsSlotsFind( b.arena, p->slots, 4u ); + CHECK( found != NULL ); + if ( found != NULL ) + { + const Item * shared = ItemAt( b.arena, ( *found )[0] ); + CHECK( shared != NULL && shared->count == 40 ); + CHECK( ItemAt( b.arena, ( *found )[1] ) == shared ); // two slots, one node + } + + const int64_t measured = PairsMeasure( b ); + static uint8_t wire[1u << 16]; + const int64_t n = PairsSave( b, wire, sizeof( wire ) ); + CHECK_EQ( measured, n ); + pin_golden( "map_pairs", wire, n ); + + const int64_t need = PairsLoadMeasure( wire, n ); + CHECK( need > 0 ); + uint8_t * region = (uint8_t *) MEASURED_CALLOC( need, 0 ); + if ( region == NULL ) { return; } + TableReport report; + const Pairs * loaded = PairsLoad( region, need, wire, n, &report ); + CHECK( loaded != NULL ); + CHECK( !report.malformed ); + if ( loaded != NULL ) + { + const PairsSlotsEntryValue * four = loaded->slots.Find( 4u ); + CHECK( four != NULL ); + if ( four != NULL ) + { + const Item * a = ItemAt( ( *four )[0] ); + CHECK( a != NULL && a->count == 40 ); + CHECK( ItemAt( ( *four )[1] ) == a ); // ONE node in the region too + } + const PairsSlotsEntryValue * nine = loaded->slots.Find( 9u ); + CHECK( nine != NULL ); + if ( nine != NULL ) + { + const Item * only = ItemAt( ( *nine )[0] ); + CHECK( only != NULL && only->count == 90 ); + CHECK( ItemAt( ( *nine )[1] ) == NULL ); // the null slot stayed null + } + CHECK_EQ( loaded->after, 5 ); // and the parent read on past the map + static uint8_t again[1u << 16]; + CHECK_EQ( PairsSave( loaded, again, sizeof( again ) ), n ); + CHECK( memcmp( again, wire, (size_t) n ) == 0 ); + } + free( region ); + + CHECK( b.Lock() ); + const Pairs * locked = b.AsConst(); + CHECK( locked != NULL ); + if ( locked == NULL ) { return; } + const int64_t json_bytes = PairsToJsonMeasure( locked ); + CHECK( json_bytes > 0 ); + char * json = (char *) MEASURED_CALLOC( json_bytes, 1 ); + if ( json == NULL ) { return; } + CHECK_EQ( PairsToJson( locked, json, json_bytes ), json_bytes ); + // the value takes the pointer row inside a JSON array: the node's object, + // `&node` where one node is named twice, `null` where a slot names none + CHECK( strstr( json, "\"&node\": 1" ) != NULL ); + CHECK( strstr( json, "\"count\": 40" ) != NULL ); + CHECK( strstr( json, "\"count\": 90" ) != NULL ); + CHECK( strstr( json, "null" ) != NULL ); + const char * four_at = strstr( json, "\"4\"" ); + const char * nine_at = strstr( json, "\"9\"" ); + CHECK( four_at != NULL && nine_at != NULL && four_at < nine_at ); // ASCENDING + + PairsBuilder into; + TableReport text_report; + CHECK( PairsFromJson( into, json, json_bytes, &text_report ) ); + CHECK( !text_report.malformed ); + static uint8_t from_text[1u << 16]; + CHECK_EQ( PairsSave( into, from_text, sizeof( from_text ) ), n ); + CHECK( memcmp( from_text, wire, (size_t) n ) == 0 ); + free( json ); + + // a DUPLICATE key REPLACES and the value half is reset WHOLE: EVERY slot + // back to null, so the repeat names no node the first insert named + PairsBuilder dup; + Pairs * d = dup.GetRoot(); + PairsSlotsEntryValue * first = PairsSlotsInsert( dup.main, d->slots, 4u ); + CHECK( first != NULL ); + if ( first == NULL ) { return; } + CHECK( ItemEmplace( dup.main, ( *first )[0] ) != NULL ); + CHECK( ItemEmplace( dup.main, ( *first )[1] ) != NULL ); + PairsSlotsEntryValue * repeat = PairsSlotsInsert( dup.main, d->slots, 4u ); + CHECK( repeat == first ); // the same entry, key and address unchanged + CHECK( repeat != NULL && ( *repeat )[0].value == 0 && ( *repeat )[1].value == 0 ); + CHECK_EQ( d->slots.count, 1 ); +} + +static void test_counted_pointer_array_values() +{ + // [..2]*Item: a PAIR, the TableRef array beside its int32 used count, so + // the handle is the ENTRY, which reaches both. + CrewsBuilder b; + Crews * c = b.GetRoot(); + static const uint32_t keys[2] = { 9, 4 }; // OUT OF KEY ORDER + for ( int i = 0; i < 2; i++ ) + { + CrewsMembersEntry * entry = CrewsMembersInsert( b.main, c->members, keys[i] ); + CHECK( entry != NULL ); + if ( entry == NULL ) { return; } + entry->value_count = keys[i] == 4 ? 2 : 1; + for ( int j = 0; j < entry->value_count; j++ ) + { + Item * item = ItemEmplace( b.main, entry->value[j] ); + CHECK( item != NULL ); + if ( item == NULL ) { return; } + item->count = (int32_t) ( keys[i] * 10 + j ); + } + } + c->after = 2; + + const int64_t measured = CrewsMeasure( b ); + static uint8_t wire[1u << 16]; + const int64_t n = CrewsSave( b, wire, sizeof( wire ) ); + CHECK_EQ( measured, n ); + pin_golden( "map_crews", wire, n ); + + const int64_t need = CrewsLoadMeasure( wire, n ); + CHECK( need > 0 ); + uint8_t * region = (uint8_t *) MEASURED_CALLOC( need, 0 ); + if ( region == NULL ) { return; } + TableReport report; + const Crews * loaded = CrewsLoad( region, need, wire, n, &report ); + CHECK( loaded != NULL ); + CHECK( !report.malformed ); + if ( loaded != NULL ) + { + // key 4 was inserted SECOND and carries TWO slots, key 9 first and + // carries one, so a walk that numbered the nodes in insertion order + // rather than in key order would read one entry's node as another's + const CrewsMembersEntry * four = loaded->members.Find( 4u ); + CHECK( four != NULL && four->value_count == 2 ); + if ( four != NULL && four->value_count == 2 ) + { + const Item * a = ItemAt( four->value[0] ); + const Item * second = ItemAt( four->value[1] ); + CHECK( a != NULL && a->count == 40 ); + CHECK( second != NULL && second->count == 41 ); + } + const CrewsMembersEntry * nine = loaded->members.Find( 9u ); + CHECK( nine != NULL && nine->value_count == 1 ); + if ( nine != NULL && nine->value_count == 1 ) + { + const Item * only = ItemAt( nine->value[0] ); + CHECK( only != NULL && only->count == 90 ); + } + CHECK_EQ( loaded->after, 2 ); + static uint8_t again[1u << 16]; + CHECK_EQ( CrewsSave( loaded, again, sizeof( again ) ), n ); + CHECK( memcmp( again, wire, (size_t) n ) == 0 ); + } + free( region ); + + CHECK( b.Lock() ); + const Crews * locked = b.AsConst(); + CHECK( locked != NULL ); + if ( locked == NULL ) { return; } + const int64_t json_bytes = CrewsToJsonMeasure( locked ); + CHECK( json_bytes > 0 ); + char * json = (char *) MEASURED_CALLOC( json_bytes, 1 ); + if ( json == NULL ) { return; } + CHECK_EQ( CrewsToJson( locked, json, json_bytes ), json_bytes ); + // the USED COUNT decides the row's length: two nodes under 4, one under 9 + CHECK( strstr( json, "\"count\": 40" ) != NULL ); + CHECK( strstr( json, "\"count\": 41" ) != NULL ); + CHECK( strstr( json, "\"count\": 90" ) != NULL ); + const char * four_at = strstr( json, "\"4\"" ); + const char * nine_at = strstr( json, "\"9\"" ); + CHECK( four_at != NULL && nine_at != NULL && four_at < nine_at ); // ASCENDING + + CrewsBuilder into; + TableReport text_report; + CHECK( CrewsFromJson( into, json, json_bytes, &text_report ) ); + CHECK( !text_report.malformed ); + static uint8_t from_text[1u << 16]; + CHECK_EQ( CrewsSave( into, from_text, sizeof( from_text ) ), n ); + CHECK( memcmp( from_text, wire, (size_t) n ) == 0 ); + free( json ); + + // a DUPLICATE resets BOTH members of the pair, every slot AND the count + CrewsBuilder dup; + Crews * d = dup.GetRoot(); + CrewsMembersEntry * first = CrewsMembersInsert( dup.main, d->members, 4u ); + CHECK( first != NULL ); + if ( first == NULL ) { return; } + first->value_count = 2; + CHECK( ItemEmplace( dup.main, first->value[0] ) != NULL ); + CrewsMembersEntry * repeat = CrewsMembersInsert( dup.main, d->members, 4u ); + CHECK( repeat == first ); + CHECK( repeat != NULL && repeat->value_count == 0 ); + CHECK( repeat != NULL && repeat->value[0].value == 0 ); + CHECK_EQ( d->members.count, 1 ); +} + +static void test_unbounded_pointer_array_values() +{ + // []*Item: ONE member, the sixteen-byte list slot over TableRef elements, + // so the handle is the slot itself. Its slots ride in the HOLDER'S node + // extent as any list's do, and each names a node of the one walk. + TrailsBuilder b; + Trails * t = b.GetRoot(); + static const uint32_t keys[2] = { 9, 4 }; // OUT OF KEY ORDER + for ( int i = 0; i < 2; i++ ) + { + TableList * steps = TrailsStepsInsert( b.main, t->steps, keys[i] ); + CHECK( steps != NULL ); + if ( steps == NULL ) { return; } + TableRef * slot = TrailsStepsEntryValueAdd( b.main, *steps ); + CHECK( slot != NULL ); + if ( slot == NULL ) { return; } + Item * item = ItemEmplace( b.main, *slot ); + CHECK( item != NULL ); + if ( item == NULL ) { return; } + item->count = (int32_t) ( keys[i] * 10 ); + if ( keys[i] == 4 ) + { + // two slots, one node, and a slot that names none past them + TableRef * share = TrailsStepsEntryValueAdd( b.main, *steps ); + CHECK( share != NULL ); + if ( share == NULL ) { return; } + *share = *slot; + CHECK( TrailsStepsEntryValueAdd( b.main, *steps ) != NULL ); + } + } + t->after = 1; + + const int64_t measured = TrailsMeasure( b ); + static uint8_t wire[1u << 16]; + const int64_t n = TrailsSave( b, wire, sizeof( wire ) ); + CHECK_EQ( measured, n ); + pin_golden( "map_trails", wire, n ); + + // the region: the entry array first, then EACH ENTRY'S list slots, the + // pre-order placement §2.8 states + const int64_t need = TrailsLoadMeasure( wire, n ); + CHECK( need > 0 ); + uint8_t * region = (uint8_t *) MEASURED_CALLOC( need, 0 ); + if ( region == NULL ) { return; } + TableReport report; + const Trails * loaded = TrailsLoad( region, need, wire, n, &report ); + CHECK( loaded != NULL ); + CHECK( !report.malformed ); + if ( loaded != NULL ) + { + // A LIST WHOSE SLOTS WERE NOT PLACED HAS A NULL ELEMENT POINTER, so + // the count is checked and the slots are read only under it: a walk + // that never laid the extent must report on a CHECK and not on a fault + const TableList * four = loaded->steps.Find( 4u ); + CHECK( four != NULL && four->count == 3 ); + if ( four != NULL && four->count == 3 ) + { + const Item * a = ( *four )[0]; + CHECK( a != NULL && a->count == 40 ); + CHECK( ( *four )[1] == a ); // two slots, one node + CHECK( ( *four )[2] == NULL ); // and a slot that names none + } + const TableList * nine = loaded->steps.Find( 9u ); + CHECK( nine != NULL && nine->count == 1 ); + if ( nine != NULL && nine->count == 1 ) + { + const Item * only = ( *nine )[0]; + CHECK( only != NULL && only->count == 90 ); + } + CHECK_EQ( loaded->after, 1 ); + static uint8_t again[1u << 16]; + CHECK_EQ( TrailsSave( loaded, again, sizeof( again ) ), n ); + CHECK( memcmp( again, wire, (size_t) n ) == 0 ); + } + free( region ); + + CHECK( b.Lock() ); + const Trails * locked = b.AsConst(); + CHECK( locked != NULL ); + if ( locked == NULL ) { return; } + const int64_t json_bytes = TrailsToJsonMeasure( locked ); + CHECK( json_bytes > 0 ); + char * json = (char *) MEASURED_CALLOC( json_bytes, 1 ); + if ( json == NULL ) { return; } + CHECK_EQ( TrailsToJson( locked, json, json_bytes ), json_bytes ); + CHECK( strstr( json, "\"&node\": 1" ) != NULL ); + CHECK( strstr( json, "\"count\": 40" ) != NULL ); + CHECK( strstr( json, "\"count\": 90" ) != NULL ); + CHECK( strstr( json, "null" ) != NULL ); + const char * four_at = strstr( json, "\"4\"" ); + const char * nine_at = strstr( json, "\"9\"" ); + CHECK( four_at != NULL && nine_at != NULL && four_at < nine_at ); // ASCENDING + + TrailsBuilder into; + TableReport text_report; + CHECK( TrailsFromJson( into, json, json_bytes, &text_report ) ); + CHECK( !text_report.malformed ); + static uint8_t from_text[1u << 16]; + CHECK_EQ( TrailsSave( into, from_text, sizeof( from_text ) ), n ); + CHECK( memcmp( from_text, wire, (size_t) n ) == 0 ); + free( json ); + + // a DUPLICATE resets the list SLOT, so the repeat starts empty and the + // first insert's nodes are unreachable rather than inherited + TrailsBuilder dup; + Trails * d = dup.GetRoot(); + TableList * first = TrailsStepsInsert( dup.main, d->steps, 4u ); + CHECK( first != NULL ); + if ( first == NULL ) { return; } + CHECK( TrailsStepsEntryValueAdd( dup.main, *first ) != NULL ); + TableList * repeat = TrailsStepsInsert( dup.main, d->steps, 4u ); + CHECK( repeat == first ); + CHECK( repeat != NULL && repeat->count == 0 ); + CHECK_EQ( d->steps.count, 1 ); +} + // ---- the MESSAGE FORM over maps (docs/SPEC-TABLES.md §2.8, §3.3) ---- // // A map rides on the message wire as its thirty-two bit count and its entries @@ -2179,6 +2526,9 @@ int main( int argc, char ** argv ) test_unbounded_values(); test_blob_values(); test_byte_blob_values(); + test_pointer_array_values(); + test_counted_pointer_array_values(); + test_unbounded_pointer_array_values(); test_message_form(); if ( failures != 0 ) diff --git a/testdata/wire/tables/map_crews.bin b/testdata/wire/tables/map_crews.bin new file mode 100644 index 0000000000000000000000000000000000000000..26c364574de0e47f6b58da605b2a06cc77b5025f GIT binary patch literal 137 zcmZQ%Hk0%^Y5R)jg~61+PikJiD$M6+Y})@H3fMB-XIxx(KB#0-rAyzV LryJS9x)~S%9qJ;m literal 0 HcmV?d00001 diff --git a/testdata/wire/tables/map_trails.bin b/testdata/wire/tables/map_trails.bin new file mode 100644 index 0000000000000000000000000000000000000000..dad875d025b3a74f2a1718286fe79283a4b13e02 GIT binary patch literal 129 zcmZQ% Date: Mon, 7 Sep 2026 05:55:01 -0400 Subject: [PATCH 3/5] docs: an array of pointers under a map key, and its handle (#666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §2.8 gains one paragraph between the storage list and the handle rule. AN ARRAY OF POINTERS IS ITS ARRAY FORM'S ROW OVER A REFERENCE ELEMENT: the element is the eight-byte reference every pointer field has, and the array form decides the rest, so `[N]*T` is `N` references and one member, `[..N]*T` is `N` references beside its `int32` used count and two members, and `[]*T` is the sixteen-byte list slot over reference elements and one member. Each slot names a node of the walk below, so two slots may name one node and a slot may name none. `[E]*T`, `[N]*string` and `[]*bytes` are refused by name, and the refusal reaches a map's value at the entry. The handle rule gains the sentence that follows from it: THE ARRAY FORM DECIDES AN ARRAY OF POINTERS' HANDLE, NEVER THE ELEMENT, which is the one the C++ reference had backwards. §4.2's map-value paragraph names the element kind the wire fuzzer's array strategies meet at these three: kind `17`, a node index, so a `[N]*T`, a `[..N]*T` and a `[]*T` are the array strategies over node indices rather than a strategy of their own. Co-Authored-By: Claude Opus 5 --- docs/SPEC-TABLES.md | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/docs/SPEC-TABLES.md b/docs/SPEC-TABLES.md index cb45b97c4..e2a1b9578 100644 --- a/docs/SPEC-TABLES.md +++ b/docs/SPEC-TABLES.md @@ -1699,6 +1699,21 @@ about a value's storage is the map's: field has (§2.1, §2.5), one member, and the node it names takes its index where the map is reached. +**AN ARRAY OF POINTERS IS ITS ARRAY FORM'S ROW OVER A REFERENCE ELEMENT** +(§2.1, §4.2). The element is a pointer, so the ELEMENT is the eight-byte +reference and the ARRAY FORM decides the rest, exactly as the same three +spellings decide it at any other field: + +- `[N]*T` stores `N` references, one member; +- `[..N]*T` stores `N` references beside its `int32` used count, two members; +- `[]*T` stores the sixteen-byte list slot over reference elements, one member. + +Each slot names a node of the walk below, so two slots may name one node and a +slot may name none, and nothing about that is the map's either. `[E]*T`, +`[N]*string` and `[]*bytes` are refused by name (§2.4, §15), and the refusal +reaches a map's value at the entry, because the value is a field of a table +nobody wrote. + **THE HANDLE FOLLOWS THE STORAGE.** `Insert`, `Find` and `Each` hand back a pointer to the `value` member where the storage is ONE member, and the ENTRY where it is two, because two members are not one addressable slot and a caller @@ -1706,9 +1721,12 @@ that cannot set the length or the count cannot fill the value. A `[N]T` value's handle points at the ARRAY rather than at its first element, so the extent survives the handoff. A `*T`, `*string` or `*bytes` value's BUILDER handle is the SLOT, which is what an `Emplace` fills, and the const `Find` answers the -RESOLVED node, one add on the self-relative delta. Where the handle is the -entry, the caller fills `value` and its companion and leaves `key` to the map, -which owns the order the key carries. +RESOLVED node, one add on the self-relative delta. THE ARRAY FORM DECIDES AN +ARRAY OF POINTERS' HANDLE, NEVER THE ELEMENT: a `[N]*T` hands back the array of +references, a `[..N]*T` the entry, a `[]*T` the list slot, and the caller +resolves each slot as it resolves any pointer. Where the handle is the entry, +the caller fills `value` and its companion and leaves `key` to the map, which +owns the order the key carries. **And a map is a BY-VALUE EDGE of the ONE declaration-order walk** (§3.1, schema#438). The numbering, the pack measure and the pack are one walk over @@ -6158,7 +6176,9 @@ without a rule of its own.** The value is an ordinary field of the generated entry and its storage is that field's own row (§2.8), so an entry's body carries a field header, a length and a payload of the value's own kind: an array's `N` and ELEMENT KIND where the value is `[N]T`, `[..N]T`, `[E]T` or an -unbounded `[]T`; a kind `17` NODE INDEX where it is `*T`, `*string` or +unbounded `[]T`, and that element kind is `17` where the element is a pointer, +so a `[N]*T`, a `[..N]*T` and a `[]*T` are the array strategies over node +indices; a kind `17` NODE INDEX where the value is `*T`, `*string` or `*bytes`, and the blob record it names is a record of the node table like any other; and the kind `12` and kind `33` payloads the text strategies above name. The strategies are enumerated over field positions, so each lands inside an From e1060a00acc4abfa8449321e154a334f7ba68475 Mon Sep 17 00:00:00 2001 From: Rowan Date: Mon, 7 Sep 2026 05:55:17 -0400 Subject: [PATCH 4/5] tables: regenerate the maps unit's goldens over the three new roots (#666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `make update-goldens`. Every moved file is the maps unit and nothing else, and each movement is the three new roots joining that unit: - SIX NEW GOLDENS, `PairsTable.{h,cpp}`, `CrewsTable.{h,cpp}` and `TrailsTable.{h,cpp}`: the three new units' own emission. - `tables/maps/tables.baseline`: the projection gains the three roots, the three anonymous entries keyed by the holder's wire id and the map field's, and the three map fields. Each entry's value renders `kind=14 elem=17`, a kind 14 array over kind 17 NODE INDEX elements, which is §4.2's row for an array of pointers and the evidence the element is a reference and not the pointee. - THE TEN EXISTING `*Table.h` GOLDENS, on FOUR NAMED CONSTANTS and the announcement blob they describe. The unit's vocabulary is one set shared by every header of the unit (§3.3), so three roots grow it for all of them: `kTableMessageEntriesHere` 66 to 75 and `kTableAnnounceBytes` 786 to 901, the projection half accounting for six of the nine new entries, which moves `kTableNodeTableFieldSlot` 48 to 54, and the tail for three. `kTableRetainKnownIds` 67 to 76 is the same count at the retain-unknown known-id table (§6.6). No existing `*Table.cpp` moved: the text form's translation unit carries no vocabulary constant. - `FleetTable.h` moves twice more. `Item` is a POINTER TARGET of the unit now, so the `ItemAt`/`ItemEmplace` surface is emitted where `Item` is declared, which is Fleet's file, and `ShipConfig`'s node slot moves 61 to 69 with the vocabulary. `BuildVersion` moves with the projection, as it does on every projection edit. - `testdata/wire/tables/map_conn.bin` 786 to 901 bytes: it IS the announcement, so it carries the nine new entries. - `testdata/wire/tables/map_full_message.bin`, 179 bytes both ways: the message body references the vocabulary by slot, and `ShipConfig`'s slot moved 61 to 69 inside a field of unchanged width. Co-Authored-By: Claude Opus 5 --- tables/maps/tables.baseline | 27 + testdata/golden/tables/maps/CellsTable.h | 187 +- testdata/golden/tables/maps/ChunksTable.h | 175 +- testdata/golden/tables/maps/CrewsTable.cpp | 3448 +++++++ testdata/golden/tables/maps/CrewsTable.h | 9466 ++++++++++++++++++ testdata/golden/tables/maps/DepthTable.h | 172 +- testdata/golden/tables/maps/DocsTable.h | 187 +- testdata/golden/tables/maps/FleetTable.h | 298 +- testdata/golden/tables/maps/PairsTable.cpp | 3448 +++++++ testdata/golden/tables/maps/PairsTable.h | 9471 ++++++++++++++++++ testdata/golden/tables/maps/RowsTable.h | 170 +- testdata/golden/tables/maps/RunsTable.h | 175 +- testdata/golden/tables/maps/SlotsTable.h | 179 +- testdata/golden/tables/maps/SpansTable.h | 171 +- testdata/golden/tables/maps/TextTable.h | 170 +- testdata/golden/tables/maps/TrailsTable.cpp | 3448 +++++++ testdata/golden/tables/maps/TrailsTable.h | 9623 +++++++++++++++++++ testdata/wire/tables/map_conn.bin | Bin 786 -> 901 bytes testdata/wire/tables/map_full_message.bin | Bin 179 -> 179 bytes 19 files changed, 40039 insertions(+), 776 deletions(-) create mode 100644 testdata/golden/tables/maps/CrewsTable.cpp create mode 100644 testdata/golden/tables/maps/CrewsTable.h create mode 100644 testdata/golden/tables/maps/PairsTable.cpp create mode 100644 testdata/golden/tables/maps/PairsTable.h create mode 100644 testdata/golden/tables/maps/TrailsTable.cpp create mode 100644 testdata/golden/tables/maps/TrailsTable.h diff --git a/tables/maps/tables.baseline b/tables/maps/tables.baseline index 5a7c732e8..a74c2dece 100644 --- a/tables/maps/tables.baseline +++ b/tables/maps/tables.baseline @@ -29,6 +29,10 @@ table 0a53e00afba279af.7b024c46e98d3404 field key id=0x3dc94a19365b10ec kind=8 field value id=0x7ce4fd9430e80cea kind=17 type=ShipConfig +table 1404200dab337086.e68c2e6bb1ee5646 + field key id=0x3dc94a19365b10ec kind=8 + field value id=0x7ce4fd9430e80cea kind=14 elem=17 type=Item array=fixed bound=2 + table 1f781dc01a2b5152.a3a7061ff10a8138 field key id=0x3dc94a19365b10ec kind=12 size=8 field value id=0x7ce4fd9430e80cea kind=14 elem=4 array=fixed bound=3 @@ -57,6 +61,10 @@ table Chunks field blobs id=0x14e2eaab9cde925b kind=14 elem=13 array=map keykind=4 field after id=0xbf82010f6f71eae9 kind=4 +table Crews + field members id=0x79d594675e391090 kind=14 elem=13 array=map keykind=8 + field after id=0xbf82010f6f71eae9 kind=4 + table Depth field one id=0x1a08aa1921ca5caf kind=13 type=Squad field many id=0x1f6459a2cea1fc02 kind=14 elem=13 type=Squad array=bounded bound=3 @@ -83,6 +91,10 @@ table Fleet table Item field count id=0xb1e5e28e4479a274 kind=4 +table Pairs + field slots id=0xe68c2e6bb1ee5646 kind=14 elem=13 array=map keykind=8 + field after id=0xbf82010f6f71eae9 kind=4 + table Row field entries id=0xc5b2a72c0845a253 kind=14 elem=13 array=map keykind=12 keybound=8 field after id=0xbf82010f6f71eae9 kind=4 @@ -112,6 +124,10 @@ table Text field blobs id=0x14e2eaab9cde925b kind=14 elem=13 array=map keykind=4 field after id=0xbf82010f6f71eae9 kind=4 +table Trails + field steps id=0x124250ad5a5b6d14 kind=14 elem=13 array=map keykind=8 + field after id=0xbf82010f6f71eae9 kind=4 + table WideRow field entries id=0xc5b2a72c0845a253 kind=14 elem=13 array=map keykind=8 field after id=0xbf82010f6f71eae9 kind=4 @@ -124,6 +140,14 @@ table b413964e3571a316.c5b2a72c0845a253 field key id=0x3dc94a19365b10ec kind=8 field value id=0x7ce4fd9430e80cea kind=13 type=Item +table b4578774a78fb150.124250ad5a5b6d14 + field key id=0x3dc94a19365b10ec kind=8 + field value id=0x7ce4fd9430e80cea kind=14 elem=17 type=Item array=unbounded + +table c85b940060088651.79d594675e391090 + field key id=0x3dc94a19365b10ec kind=8 + field value id=0x7ce4fd9430e80cea kind=14 elem=17 type=Item array=bounded bound=2 + table e8130af045a036f8.2b7dea192bb7be29 field key id=0x3dc94a19365b10ec kind=9 field value id=0x7ce4fd9430e80cea kind=13 type=Item @@ -170,3 +194,6 @@ union Force ### 2026-09-07 (UTC) — Cells, Runs, Slots, Spans, Docs and Chunks: one map per value kind that carries an extent or names a buffer node (#628) - no compatibility-affecting edits; the wire absorbs the rest + +### 2026-09-07 (UTC) — Pairs, Crews and Trails: one map per value kind whose element is a pointer (#666) +- no compatibility-affecting edits; the wire absorbs the rest diff --git a/testdata/golden/tables/maps/CellsTable.h b/testdata/golden/tables/maps/CellsTable.h index 907c00422..161361a45 100644 --- a/testdata/golden/tables/maps/CellsTable.h +++ b/testdata/golden/tables/maps/CellsTable.h @@ -376,7 +376,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -878,13 +878,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1406,7 +1406,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1424,10 +1424,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1436,6 +1436,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1444,54 +1445,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3429,25 +3439,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -3752,6 +3764,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4128,6 +4150,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -5676,7 +5703,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6434,7 +6461,7 @@ inline bool CellsRowsEntrySaveMessageBody( TableBitWriter & w, const CellsRowsEn for ( int32_t i = 0; i < 3; i++ ) { if ( value.value[i] != 0 ) { rides_value = true; break; } } if ( rides_value ) { - w.put( 10, kTableMessageRefBitsHere ); + w.put( 11, kTableMessageRefBitsHere ); for ( int32_t i = 0; i < 3; i++ ) { w.put( (uint64_t) ( value.value[i] ), 32 ); @@ -6480,15 +6507,13 @@ inline bool CellsRowsEntryLoadMessageBody( TableBitReader & r, const TableVocabu { uint64_t n = 0; if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } - int32_t kept = 0; - if ( n > (uint64_t) 8 ) { kept = 8; report->clamped++; } else { kept = (int32_t) n; } - for ( uint64_t i = 0; i < n; i++ ) - { - uint64_t by = 0; - if ( !r.get( by, 8 ) ) { report->malformed = true; return false; } - if ( (int32_t) i < kept ) { value.key[i] = (char) by; } - } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); value.key[kept] = 0; + r.offset += (int64_t) n * 8; value.key_length = kept; } break; @@ -6869,7 +6894,7 @@ inline bool CellsSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_rows.ok ) { return false; } // the sort could not run if ( order_rows.count > 0 ) { - w.put( 16, kTableMessageRefBitsHere ); + w.put( 17, kTableMessageRefBitsHere ); w.put( (uint64_t) order_rows.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_rows.count; i++ ) { @@ -6880,7 +6905,7 @@ inline bool CellsSaveMessageBody( const Ctx & ctx, const TableNumbering & number } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/ChunksTable.h b/testdata/golden/tables/maps/ChunksTable.h index 6a1d6b153..b6de29938 100644 --- a/testdata/golden/tables/maps/ChunksTable.h +++ b/testdata/golden/tables/maps/ChunksTable.h @@ -376,7 +376,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -878,13 +878,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1406,7 +1406,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1424,10 +1424,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1436,6 +1436,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1444,54 +1445,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3429,25 +3439,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -3752,6 +3764,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4128,6 +4150,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -5676,7 +5703,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6392,7 +6419,7 @@ inline bool ChunksBlobsEntrySaveMessageBody( const Ctx & ctx, const TableNumberi (void) ctx; (void) numbering; (void) index_bits; if ( value.key != 0 ) { - w.put( 11, kTableMessageRefBitsHere ); + w.put( 12, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.key ), 32 ); } { @@ -6776,7 +6803,7 @@ inline bool ChunksSaveMessageBody( const Ctx & ctx, const TableNumbering & numbe if ( !order_blobs.ok ) { return false; } // the sort could not run if ( order_blobs.count > 0 ) { - w.put( 18, kTableMessageRefBitsHere ); + w.put( 19, kTableMessageRefBitsHere ); w.put( (uint64_t) order_blobs.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_blobs.count; i++ ) { @@ -6787,7 +6814,7 @@ inline bool ChunksSaveMessageBody( const Ctx & ctx, const TableNumbering & numbe } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body @@ -7200,7 +7227,7 @@ inline bool ChunksBlobsEntryNumber( const Ctx & ctx, TableNumbering & numbering, TableNodeEntry node; node.node = (const void *) blob; node.type_id = kTableBytesTypeId; - node.type_slot = 49; // its slot in the unit's vocabulary (§3.3) + node.type_slot = 55; // its slot in the unit's vocabulary (§3.3) node.measure = &TableBlobMeasureThunk; node.save = &TableBlobSaveThunk; node.message_measure = &TableBlobMessageMeasureThunk; diff --git a/testdata/golden/tables/maps/CrewsTable.cpp b/testdata/golden/tables/maps/CrewsTable.cpp new file mode 100644 index 000000000..d42cb3cd5 --- /dev/null +++ b/testdata/golden/tables/maps/CrewsTable.cpp @@ -0,0 +1,3448 @@ +// Code generated by the schema compiler from Crews.schema. DO NOT EDIT. +// SPDX-License-Identifier: NONE — this generated output is yours, under terms of +// your choice. See the LICENSE exception in the schema compiler; the compiler is +// AGPL-3.0, its output is not. +// package mapdemo — the TABLE wire's text form (docs/SPEC-TABLES.md §16). +// Compile this file to use FromJson / ToJson; a project that +// never reads or writes a text does not compile it and pays nothing. + +#include "CrewsTable.h" + +#include // the text form: number formatting +#include // the text form: exact number conversion +#include // the text form: the runtime's decimal point + +// The guard is not vestigial. Several mapdemo Table.cpp files may be +// concatenated into ONE translation unit — a unity build — and without it +// each would redefine the walk. It is also why the walk's functions may be +// weak (vague linkage) across separate objects: ODR requires their +// definitions to be token-identical, and the generic-walk gate is what +// proves that, byte for byte, across every generated .cpp. +#ifndef MAPDEMO_SCHEMA_TABLE_JSON +#define MAPDEMO_SCHEMA_TABLE_JSON + +namespace mapdemo { + +// ---- the pointer adapters (docs/SPEC-TABLES.md §16.7) ---- +// +// The walk below is ONE walk, byte-identical in every generated .cpp, and a +// pointer is the one kind it cannot walk alone: reading one needs the +// builder's arena and writing one needs a region's deref, and neither exists +// in a unit that declares no pointer. So the walk calls these three and does +// not define them. A unit with no pointer defines them as stubs no field ever +// reaches; a pointered unit defines them in the graph half that follows the +// walk. + +struct TableJsonIn; +struct TableJsonOut; + +// a pointer field's object, or the `&node` reference standing in for it, into +// the slot; the cursor is on the opening brace +inline bool TableJsonReadPointer( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); +// the node a pointer slot names, in place — or as `&node` when it is shared +inline bool TableJsonWritePointer( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// the FIRST key of an object the walk is skipping begins with `&`: the cursor is +// on its value. A dropped definition still takes its label (§16.7); a fixed reader +// skips the value whole, as it skips everything else it does not place. +inline bool TableJsonSkippedAmpersand( TableJsonIn & in, const char * key, int32_t depth ); + +// ---- the map and list adapters (docs/SPEC-TABLES.md §2.8, §2.9, §16) ---- +// +// A MAP and an UNBOUNDED ARRAY are the other constructs the walk cannot walk +// alone: their arrays live behind a TableMap or a TableList this +// walk has no name for, reading one needs the builder's arena, and neither +// exists in a unit that declares neither construct. Same shape as the +// pointer's three: declared here, defined after the walk by whichever half +// the unit carries. Both are OUT-OF-LINE ARRAYS to the descriptors (§8.1): +// array_bound = 0 is the tell, and the type name says which of the two. + +// a map field: an out-of-line array whose type name spells the map +inline bool TableJsonIsMap( const TableFieldInfo * f ); +// the map as a plain JSON object keyed by the KEY, in ASCENDING key order +inline bool TableJsonWriteMap( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// that object back into the slot, in whatever order the text gives it +inline bool TableJsonReadMap( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); +// an unbounded array: the other out-of-line array +inline bool TableJsonIsList( const TableFieldInfo * f ); +// the list as a JSON array, in INDEX order +inline bool TableJsonWriteList( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// that array back into the slot, every element the text carries +inline bool TableJsonReadList( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); + +// ---- json walk: begin ---- +// +// The TEXT form (docs/SPEC-TABLES.md §16): one table, one text, one walk over the +// reflection descriptors (§8). Reading fills ONE caller-owned instance and +// allocates nothing beyond it; writing targets a caller buffer with the +// wire's measure/write symmetry. Everything AROUND this — which file goes +// with which instance, what key an instance is filed under, how instances +// link into a root table's collections — is a packer's opinion and stays +// with the tool that holds it. +// +// The dialect: trailing commas are accepted on read (the authoring files +// this exists for carry them) and never written; comments are not JSON and +// are refused; unknown keys are skipped and counted; a duplicate key is +// last-wins and counted; a key present with the wrong JSON type is skipped +// and counted, never coerced. + +static const int32_t kTableJsonMaxDepth = 128; + +// A key longer than this cannot name a field, so it is skipped as unknown. +static const int32_t kTableJsonMaxKey = 256; + +// The longest numeric token the walk will convert. Anything longer is a +// value no field can hold and counts as a kind mismatch. +static const int32_t kTableJsonMaxNumber = 512; + +// The decimal point the C runtime is CURRENTLY using. Number conversion is +// the one locale-sensitive corner of the grammar — JSON's point is always +// '.', the runtime's is whatever the program set — so every number crosses +// this one character on the way out and on the way back in. Nothing else in +// the walk consults the locale. +inline char TableJsonDecimalPoint() +{ + const struct lconv * conv = localeconv(); + if ( conv != NULL && conv->decimal_point != NULL && conv->decimal_point[0] != 0 ) + { + return conv->decimal_point[0]; + } + return '.'; +} + +// ---- storage access: the descriptors give an offset and a width, and the +// ---- storage is the HOST's, so every load and store goes through a width +// ---- switch rather than a memcpy into the low bytes of a wider word + +// finite: not a NaN, not an infinity. Written without — the walk's +// runtime surface stays the handful of functions it already names. +// A vocabulary entry the descriptor could not spell. The generated name +// functions answer "???" for a value outside the declared set, and that is +// not a name — writing it would put a spelling in the text that the reader +// then counts as unknown, turning a refusal into a silent loss. +inline bool TableJsonNamed( const char * name ) +{ + return name != NULL && strcmp( name, "???" ) != 0; +} + +inline bool TableJsonFinite( double v ) +{ + return v == v && v <= 1.7976931348623157e308 && v >= -1.7976931348623157e308; +} + +inline uint64_t TableJsonGetRaw( const void * storage, uint32_t width ) +{ + switch ( width ) + { + case 1: { uint8_t v = 0; memcpy( &v, storage, 1 ); return v; } + case 2: { uint16_t v = 0; memcpy( &v, storage, 2 ); return v; } + case 4: { uint32_t v = 0; memcpy( &v, storage, 4 ); return v; } + case 8: { uint64_t v = 0; memcpy( &v, storage, 8 ); return v; } + } + return 0; +} + +inline void TableJsonSetRaw( void * storage, uint32_t width, uint64_t value ) +{ + switch ( width ) + { + case 1: { uint8_t v = (uint8_t) value; memcpy( storage, &v, 1 ); break; } + case 2: { uint16_t v = (uint16_t) value; memcpy( storage, &v, 2 ); break; } + case 4: { uint32_t v = (uint32_t) value; memcpy( storage, &v, 4 ); break; } + case 8: { uint64_t v = value; memcpy( storage, &v, 8 ); break; } + } +} + +inline int64_t TableJsonGetSigned( const void * storage, uint32_t width ) +{ + uint64_t raw = TableJsonGetRaw( storage, width ); + if ( width < 8 ) + { + uint64_t sign = uint64_t( 1 ) << ( width * 8 - 1 ); + if ( ( raw & sign ) != 0 ) + { + raw |= ~( ( sign << 1 ) - 1 ); + } + } + return (int64_t) raw; +} + +// ---- the WIDE kinds (docs/SPEC-TABLES.md §3, §16.2) ---- +// +// The 128-bit integers and the fixed-point family convert EXACTLY, over two +// 64-bit lanes: a 128-bit integer is a decimal integer, a fixed value a +// decimal in WHOLE UNITS (1.0, -0.25, 3.0000152587890625) and nothing +// on either path passes through a double. Nothing here needs a 128-bit type +// either, which is what keeps this walk one text for every unit. +struct TableJsonWide +{ + uint64_t lo; + uint64_t hi; +}; + +inline bool TableJsonKindWide( uint8_t kind ) { return kind >= 18 && kind <= 29; } +inline bool TableJsonKindWideSigned( uint8_t kind ) { return kind == 18 || ( kind >= 20 && kind <= 24 ); } +inline bool TableJsonKindFixed( uint8_t kind ) { return kind >= 20 && kind <= 29; } + +inline bool TableJsonWideZero( TableJsonWide v ) { return v.lo == 0 && v.hi == 0; } +inline bool TableJsonWideNegative( TableJsonWide v ) { return ( v.hi >> 63 ) != 0; } + +inline int TableJsonWideCompare( TableJsonWide a, TableJsonWide b, bool is_signed ) +{ + if ( is_signed && TableJsonWideNegative( a ) != TableJsonWideNegative( b ) ) { return TableJsonWideNegative( a ) ? -1 : 1; } + if ( a.hi != b.hi ) { return a.hi < b.hi ? -1 : 1; } + if ( a.lo != b.lo ) { return a.lo < b.lo ? -1 : 1; } + return 0; +} + +inline TableJsonWide TableJsonWideShl( TableJsonWide v, int n ) +{ + TableJsonWide r = { 0, 0 }; + if ( n <= 0 ) { return v; } + if ( n >= 128 ) { return r; } + if ( n >= 64 ) { r.hi = v.lo << ( n - 64 ); return r; } + r.hi = ( v.hi << n ) | ( v.lo >> ( 64 - n ) ); + r.lo = v.lo << n; + return r; +} + +inline TableJsonWide TableJsonWideShr( TableJsonWide v, int n ) +{ + TableJsonWide r = { 0, 0 }; + if ( n <= 0 ) { return v; } + if ( n >= 128 ) { return r; } + if ( n >= 64 ) { r.lo = v.hi >> ( n - 64 ); return r; } + r.lo = ( v.lo >> n ) | ( v.hi << ( 64 - n ) ); + r.hi = v.hi >> n; + return r; +} + +inline TableJsonWide TableJsonWideNeg( TableJsonWide v ) +{ + TableJsonWide r; + r.lo = ~v.lo + 1; + r.hi = ~v.hi + ( r.lo == 0 ? 1 : 0 ); + return r; +} + +// v = v * m + a; the return is the carry out of 128 bits +inline uint32_t TableJsonWideMulAdd( TableJsonWide * v, uint32_t m, uint32_t a ) +{ + uint64_t limb[4] = { v->lo & 0xffffffffull, v->lo >> 32, v->hi & 0xffffffffull, v->hi >> 32 }; + uint64_t carry = a; + for ( int i = 0; i < 4; i++ ) + { + uint64_t p = limb[i] * m + carry; + limb[i] = p & 0xffffffffull; + carry = p >> 32; + } + v->lo = limb[0] | ( limb[1] << 32 ); + v->hi = limb[2] | ( limb[3] << 32 ); + return (uint32_t) carry; +} + +// v = v / d; the return is the remainder +inline uint32_t TableJsonWideDiv( TableJsonWide * v, uint32_t d ) +{ + uint64_t limb[4] = { v->lo & 0xffffffffull, v->lo >> 32, v->hi & 0xffffffffull, v->hi >> 32 }; + uint64_t rem = 0; + for ( int i = 3; i >= 0; i-- ) + { + uint64_t cur = ( rem << 32 ) | limb[i]; + limb[i] = cur / d; + rem = cur % d; + } + v->lo = limb[0] | ( limb[1] << 32 ); + v->hi = limb[2] | ( limb[3] << 32 ); + return (uint32_t) rem; +} + +// The storage of a wide kind, as lanes. A sixteen-byte storage is serialize's +// pair — native __int128 in the host's byte order, or the emulated struct with +// its low lane first — so the lanes are read in the host's order; a narrower +// storage is one lane, sign-extended for a signed kind. +inline TableJsonWide TableJsonWideLoad( const void * storage, uint32_t width, bool is_signed ) +{ + TableJsonWide v = { 0, 0 }; + if ( width == 16 ) + { + uint64_t half[2]; + memcpy( half, storage, 16 ); + uint16_t probe = 1; + bool little = *(const uint8_t *) &probe == 1; + v.lo = little ? half[0] : half[1]; + v.hi = little ? half[1] : half[0]; + return v; + } + v.lo = is_signed ? (uint64_t) TableJsonGetSigned( storage, width ) : TableJsonGetRaw( storage, width ); + v.hi = ( is_signed && ( v.lo >> 63 ) != 0 ) ? ~uint64_t( 0 ) : 0; + return v; +} + +inline void TableJsonWideStore( void * storage, uint32_t width, TableJsonWide v ) +{ + if ( width == 16 ) + { + uint16_t probe = 1; + bool little = *(const uint8_t *) &probe == 1; + uint64_t half[2]; + half[0] = little ? v.lo : v.hi; + half[1] = little ? v.hi : v.lo; + memcpy( storage, half, 16 ); + return; + } + TableJsonSetRaw( storage, width, v.lo ); +} + +// a counted field's companion: a string's length, a bytes' length, a counted +// array's count. Bounded by the declared extent on the way out, so a storage +// invariant a caller broke cannot walk off the end of the array. +inline int32_t TableJsonCount( const void * base, const TableFieldInfo * f ) +{ + if ( !f->counted ) + { + return f->array_bound; + } + int32_t count = 0; + memcpy( &count, (const uint8_t *) base + f->count_offset, sizeof( count ) ); + if ( count < 0 ) { count = 0; } + if ( count > f->array_bound ) { count = f->array_bound; } + return count; +} + +inline void TableJsonSetCount( void * base, const TableFieldInfo * f, int32_t count ) +{ + if ( f->counted ) + { + memcpy( (uint8_t *) base + f->count_offset, &count, sizeof( count ) ); + } +} + +// ---- what a field's kind expects to see in the text ---- +// +// One classifier, consulted by both directions, so a reader and a writer can +// never disagree about a kind's JSON form. 'o' object, 'a' array, 's' +// string, 'n' number, 'b' boolean. +// +// A vocabulary field is spelled by NAME: an enum is one name, a flags mask +// is the array of the names of its set bits. The two are told apart by the +// id column — an enum variant rides under a wire id, a flags BIT never does +// (docs/SPEC-TABLES.md §4), so a name function with no id function is flags. +// +// bytes(N) is the one kind whose element kind does not decide its form: it +// shares u8 with a plain array of u8, and rides as base64. The schema type +// name settles it, and "bytes" is a keyword no declaration can claim. +inline bool TableJsonIsBytes( const TableFieldInfo * f ) +{ + return f->is_array && f->kind == 6 && strcmp( f->type_name, "bytes" ) == 0; +} + +// An ENUM-KEYED array (docs/SPEC-TABLES.md §2.4): its JSON form is an OBJECT +// keyed by variant name, not a positional array, because that is what the +// storage is — one slot per variant, addressed by the variant. +inline bool TableJsonIsKeyed( const TableFieldInfo * f ) +{ + return f->key_name != NULL; +} + +// THE KEY A STORAGE SLOT HOLDS (§2.4, §8): the storage shifts left, so slot i +// holds the key i + 1 and nothing is stored for None. This is the ONE place +// the walker spells the shift. +inline uint64_t TableJsonKeyedSlotKey( int64_t slot ) +{ + return (uint64_t) ( slot + 1 ); +} + +// A slot whose key names a variant of the keying enum. Every slot in +// [0, array_bound) does, unless the enum carries max-headroom variants outside +// a table closure, where a reserved value names nothing and its key id is 0 — +// the reserved id no declared name can fold to (§5). +inline bool TableJsonKeyedSlotValid( const TableFieldInfo * f, int64_t slot ) +{ + return f->key_id( TableJsonKeyedSlotKey( slot ) ) != 0; +} + +inline bool TableJsonIsFlags( const TableFieldInfo * f ) +{ + return f->enum_name != NULL && f->variant_id == NULL; +} + +inline bool TableJsonIsEnum( const TableFieldInfo * f ) +{ + return f->variant_id != NULL && f->arms == NULL; +} + +inline char TableJsonShape( const TableFieldInfo * f ) +{ + if ( TableJsonIsMap( f ) ) return 'o'; // a MAP: an object keyed by the KEY (§2.8) + if ( f->kind == 12 ) return 's'; // string + if ( f->kind == 33 ) return 's'; // wstring: the same text, transcoded (§16.2) + if ( TableJsonIsBytes( f ) ) return 's'; // bytes: base64 + if ( TableJsonIsKeyed( f ) ) return 'o'; // an object keyed by variant NAME + if ( f->is_array ) return 'a'; + if ( f->arms != NULL ) return 'o'; // union: an object with ONE key + if ( f->kind == 13 ) return 'o'; // nested table or type + if ( f->kind == 17 ) return f->table != NULL ? 'o' : 's'; // a pointer: the pointee's object in place, or null (§16.7); a byte buffer's string (§2.5) + if ( TableJsonIsEnum( f ) ) return 's'; + if ( TableJsonIsFlags( f ) ) return 'a'; + if ( f->kind == 1 ) return 'b'; + return 'n'; +} + +// the ELEMENT shape of an array field — the same classifier one level down +inline char TableJsonElementShape( const TableFieldInfo * f ) +{ + if ( f->arms != NULL ) return 'o'; // an element of an array of unions: one key, the arm (§2.6) + if ( f->kind == 13 ) return 'o'; + if ( TableJsonIsEnum( f ) ) return 's'; + if ( TableJsonIsFlags( f ) ) return 'a'; + if ( f->kind == 1 ) return 'b'; + return 'n'; +} + +// A guarded group rides only when its guard reads true — the wire's own +// elision (§4), carried into the text so a text and a wire written from one +// instance say the same thing. The guard is spelled as its branch condition +// over bool fields of the SAME type ("at_rest", "!at_rest", +// "active && has_target"), so evaluating it is a walk of the same +// descriptor. Nothing is inferred in the other direction: reading places +// every key it can name, and the guard is a plain bool key (§16.2). +inline bool TableJsonGuardHolds( const void * base, const TableTypeInfo * info, const char * guard ) +{ + const char * p = guard; + for ( ;; ) + { + while ( *p == ' ' || *p == '&' ) { p++; } + if ( *p == 0 ) { return true; } + bool want = true; + if ( *p == '!' ) { want = false; p++; } + const char * start = p; + while ( *p != 0 && *p != ' ' && *p != '&' ) { p++; } + size_t length = (size_t) ( p - start ); + bool value = false; + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + const TableFieldInfo * f = &info->fields[i]; + if ( strlen( f->name ) == length && strncmp( f->name, start, length ) == 0 ) + { + value = TableJsonGetRaw( (const uint8_t *) base + f->offset, f->elem_size ) != 0; + break; + } + } + if ( value != want ) { return false; } + } +} + +// ---- writing ---- + +// The writer sink MEASURES when the buffer is NULL and WRITES when it is +// not, over one code path — so measure and write agree byte for byte, the +// wire's invariant (§9) carried across. +struct TableJsonOut +{ + char * buffer; + int64_t capacity; + int64_t offset; + bool overflow; + void * graph; // the pointered write's identity map (§16.7); NULL for a fixed table + + void raw( const char * data, int64_t count ) + { + if ( buffer != NULL ) + { + if ( offset + count > capacity ) { overflow = true; return; } + memcpy( buffer + offset, data, (size_t) count ); + } + offset += count; + } + void put( char c ) { raw( &c, 1 ); } + void text( const char * s ) { raw( s, (int64_t) strlen( s ) ); } + void line( int32_t depth ) + { + put( '\n' ); + for ( int32_t i = 0; i < depth; i++ ) { raw( " ", 2 ); } + } +}; + +inline const char * TableJsonBase64Alphabet() +{ + return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +} + +inline void TableJsonWriteBase64( TableJsonOut & out, const uint8_t * data, int32_t length ) +{ + const char * alphabet = TableJsonBase64Alphabet(); + out.put( '"' ); + int32_t i = 0; + for ( ; i + 3 <= length; i += 3 ) + { + uint32_t triple = ( uint32_t( data[i] ) << 16 ) | ( uint32_t( data[i+1] ) << 8 ) | uint32_t( data[i+2] ); + char quad[4] = { alphabet[ ( triple >> 18 ) & 0x3f ], alphabet[ ( triple >> 12 ) & 0x3f ], + alphabet[ ( triple >> 6 ) & 0x3f ], alphabet[ triple & 0x3f ] }; + out.raw( quad, 4 ); + } + if ( i < length ) + { + int32_t left = length - i; + uint32_t triple = uint32_t( data[i] ) << 16; + if ( left == 2 ) { triple |= uint32_t( data[i+1] ) << 8; } + char quad[4] = { alphabet[ ( triple >> 18 ) & 0x3f ], alphabet[ ( triple >> 12 ) & 0x3f ], '=', '=' }; + if ( left == 2 ) { quad[2] = alphabet[ ( triple >> 6 ) & 0x3f ]; } + out.raw( quad, 4 ); + } + out.put( '"' ); +} + +// One UTF-8 sequence at s, or -1 when the bytes there are not one. Rejects +// the lot: a stray continuation, an overlong form, a surrogate half, and +// anything past U+10FFFF. +inline int32_t TableJsonUtf8( const char * s, int32_t remaining, int32_t * width ) +{ + unsigned char lead = (unsigned char) s[0]; + int32_t want = 0; + int32_t code = 0; + if ( lead < 0x80 ) { *width = 1; return lead; } + else if ( lead >= 0xc2 && lead <= 0xdf ) { want = 2; code = lead & 0x1f; } + else if ( lead >= 0xe0 && lead <= 0xef ) { want = 3; code = lead & 0x0f; } + else if ( lead >= 0xf0 && lead <= 0xf4 ) { want = 4; code = lead & 0x07; } + else { return -1; } + if ( remaining < want ) { return -1; } + for ( int32_t i = 1; i < want; i++ ) + { + unsigned char next = (unsigned char) s[i]; + if ( ( next & 0xc0 ) != 0x80 ) { return -1; } + code = ( code << 6 ) | ( next & 0x3f ); + } + if ( want == 3 && code < 0x800 ) { return -1; } // overlong + if ( want == 4 && code < 0x10000 ) { return -1; } // overlong + if ( code >= 0xd800 && code <= 0xdfff ) { return -1; } // a surrogate half + if ( code > 0x10ffff ) { return -1; } + *width = want; + return code; +} + +// The inverse: one code point encoded as UTF-8 into unit, its length +// answered. Both text kinds' writers reach it, and so does the escape +// grammar's U+FFFD replacement. +inline int32_t TableJsonEncodeUtf8( uint32_t code, char * unit ) +{ + if ( code < 0x80 ) { unit[0] = (char) code; return 1; } + if ( code < 0x800 ) + { + unit[0] = (char) ( 0xc0 | ( code >> 6 ) ); + unit[1] = (char) ( 0x80 | ( code & 0x3f ) ); + return 2; + } + if ( code < 0x10000 ) + { + unit[0] = (char) ( 0xe0 | ( code >> 12 ) ); + unit[1] = (char) ( 0x80 | ( ( code >> 6 ) & 0x3f ) ); + unit[2] = (char) ( 0x80 | ( code & 0x3f ) ); + return 3; + } + unit[0] = (char) ( 0xf0 | ( code >> 18 ) ); + unit[1] = (char) ( 0x80 | ( ( code >> 12 ) & 0x3f ) ); + unit[2] = (char) ( 0x80 | ( ( code >> 6 ) & 0x3f ) ); + unit[3] = (char) ( 0x80 | ( code & 0x3f ) ); + return 4; +} + +// A JSON text MUST be valid UTF-8 (RFC 8259 §8.1). The read path is +// byte-transparent — the wire imposes no encoding (§3) and a string may hold +// anything — so the WRITER is where that obligation is met: a byte that is +// not part of a well-formed sequence is written as U+FFFD, one per bad byte, +// and never raw. A text this walk writes is therefore readable by any +// conforming parser, which a raw byte would not be. The cost is stated +// plainly: for a string holding invalid UTF-8, the round trip is NOT +// byte-identical, because the alternative is emitting a text that is not +// JSON. +inline void TableJsonWriteString( TableJsonOut & out, const char * s, int32_t length ) +{ + static const char hex[] = "0123456789abcdef"; + out.put( '"' ); + for ( int32_t i = 0; i < length; i++ ) + { + unsigned char c = (unsigned char) s[i]; + switch ( c ) + { + case '"': out.raw( "\\\"", 2 ); break; + case '\\': out.raw( "\\\\", 2 ); break; + case '\b': out.raw( "\\b", 2 ); break; + case '\f': out.raw( "\\f", 2 ); break; + case '\n': out.raw( "\\n", 2 ); break; + case '\r': out.raw( "\\r", 2 ); break; + case '\t': out.raw( "\\t", 2 ); break; + default: + if ( c < 0x20 ) + { + char escape[6] = { '\\', 'u', '0', '0', hex[ c >> 4 ], hex[ c & 0xf ] }; + out.raw( escape, 6 ); + } + else if ( c < 0x80 ) + { + out.put( (char) c ); + } + else + { + int32_t width = 0; + if ( TableJsonUtf8( s + i, length - i, &width ) < 0 ) + { + out.raw( "\xef\xbf\xbd", 3 ); // U+FFFD, one per bad byte + } + else + { + out.raw( s + i, width ); + i += width - 1; + } + } + break; + } + } + out.put( '"' ); +} + +// A WIDE field's text: the code units transcoded back to UTF-8 (§16.2). A +// SURROGATE PAIR is one code point; an UNPAIRED SURROGATE is not a code point +// at all, encodes to nothing, and writes one U+FFFD per ill-formed unit; a +// ZERO UNIT is U+0000, which JSON has an escape for, and writes \u0000 +// (§16.3). No wire can put either into storage (§3), so both answer for +// storage a PROGRAM built. +inline void TableJsonWriteWString( TableJsonOut & out, const char16_t * s, int32_t length ) +{ + static const char hex[] = "0123456789abcdef"; + out.put( '"' ); + for ( int32_t i = 0; i < length; i++ ) + { + uint32_t code = (uint32_t) (uint16_t) s[i]; + if ( code >= 0xd800 && code <= 0xdbff && i + 1 < length ) + { + const uint32_t low = (uint32_t) (uint16_t) s[i + 1]; + if ( low >= 0xdc00 && low <= 0xdfff ) + { + code = 0x10000 + ( ( code - 0xd800 ) << 10 ) + ( low - 0xdc00 ); + i++; + } + } + if ( code >= 0xd800 && code <= 0xdfff ) { code = 0xfffd; } // an unpaired surrogate + switch ( code ) + { + case '"': out.raw( "\\\"", 2 ); continue; + case '\\': out.raw( "\\\\", 2 ); continue; + case '\b': out.raw( "\\b", 2 ); continue; + case '\f': out.raw( "\\f", 2 ); continue; + case '\n': out.raw( "\\n", 2 ); continue; + case '\r': out.raw( "\\r", 2 ); continue; + case '\t': out.raw( "\\t", 2 ); continue; + default: break; + } + if ( code < 0x20 ) + { + char escape[6] = { '\\', 'u', '0', '0', hex[ code >> 4 ], hex[ code & 0xf ] }; + out.raw( escape, 6 ); + continue; + } + char encoded[4]; + const int32_t encoded_length = TableJsonEncodeUtf8( code, encoded ); + out.raw( encoded, encoded_length ); + } + out.put( '"' ); +} + +inline void TableJsonWriteUnsigned( TableJsonOut & out, uint64_t value ) +{ + char digits[24]; + int32_t n = 0; + do + { + digits[n++] = (char) ( '0' + (int) ( value % 10 ) ); + value /= 10; + } while ( value != 0 ); + char text[24]; + for ( int32_t i = 0; i < n; i++ ) { text[i] = digits[n - 1 - i]; } + out.raw( text, n ); +} + +inline void TableJsonWriteSigned( TableJsonOut & out, int64_t value ) +{ + if ( value < 0 ) + { + out.put( '-' ); + TableJsonWriteUnsigned( out, uint64_t( 0 ) - (uint64_t) value ); + return; + } + TableJsonWriteUnsigned( out, (uint64_t) value ); +} + +// A wide kind writes its raw storage as §16.2's text: a 128-bit integer as a +// decimal integer; a fixed value in WHOLE UNITS as the shortest exact decimal +// with at least one fractional digit (1.0, -0.25), the spelling the schema text +// gives a fixed default. The fraction terminates because a dyadic fraction has +// a finite decimal expansion — at most F digits. +inline void TableJsonWriteWide( TableJsonOut & out, const void * storage, const TableFieldInfo * f ) +{ + bool is_signed = TableJsonKindWideSigned( f->kind ); + TableJsonWide v = TableJsonWideLoad( storage, f->elem_size, is_signed ); + if ( is_signed && TableJsonWideNegative( v ) ) + { + out.put( '-' ); + v = TableJsonWideNeg( v ); + } + int frac = f->frac_bits; + TableJsonWide whole = TableJsonWideShr( v, frac ); + char digits[40]; + int32_t n = 0; + do + { + digits[n++] = (char) ( '0' + (int) TableJsonWideDiv( &whole, 10 ) ); + } while ( !TableJsonWideZero( whole ) ); + char text[40]; + for ( int32_t i = 0; i < n; i++ ) { text[i] = digits[n - 1 - i]; } + out.raw( text, n ); + if ( !TableJsonKindFixed( f->kind ) ) { return; } + out.put( '.' ); + // the fraction bits alone: v with everything at and above bit F cleared + TableJsonWide fraction = v; + if ( frac < 64 ) { fraction.hi = 0; fraction.lo &= ( uint64_t( 1 ) << frac ) - 1; } + else { fraction.hi &= ( uint64_t( 1 ) << ( frac - 64 ) ) - 1; } + if ( frac == 0 ) { fraction.lo = 0; } + if ( TableJsonWideZero( fraction ) ) + { + out.put( '0' ); + return; + } + while ( !TableJsonWideZero( fraction ) ) + { + // ×10: the digit is what lands at and above bit F, including the + // carry out of 128 bits when F leaves no room for it below + uint32_t carry = TableJsonWideMulAdd( &fraction, 10, 0 ); + uint64_t digit = TableJsonWideShr( fraction, frac ).lo; + if ( frac > 64 ) { digit |= uint64_t( carry ) << ( 128 - frac ); } + out.put( (char) ( '0' + (int) digit ) ); + if ( frac < 64 ) { fraction.hi = 0; fraction.lo &= ( uint64_t( 1 ) << frac ) - 1; } + else { fraction.hi &= ( uint64_t( 1 ) << ( frac - 64 ) ) - 1; } + } +} + +// A float writes at the SHORTEST precision that reads back as the same value +// at the field's own width, so a round trip is exact and a text stays +// readable. Non-finite values have no JSON spelling at all, and the writer +// REFUSES rather than losing one silently — the same rule measure and save +// already apply to an enum value no variant names (§5). +inline bool TableJsonWriteFloat( TableJsonOut & out, double value, bool single ) +{ + if ( !TableJsonFinite( value ) ) { return false; } + char text[64]; + int low = single ? 6 : 15; + int high = single ? 9 : 17; + int length = 0; + for ( int digits = low; ; digits++ ) + { + length = snprintf( text, sizeof( text ), "%.*g", digits, value ); + if ( length <= 0 || length >= (int) sizeof( text ) ) { return false; } + if ( digits >= high ) { break; } + // the round-trip check runs BEFORE the decimal point is normalised: + // the token still carries whatever point snprintf just produced + if ( single ) + { + if ( (double) strtof( text, NULL ) == value ) { break; } + } + else + { + if ( strtod( text, NULL ) == value ) { break; } + } + } + char point = TableJsonDecimalPoint(); + if ( point != '.' ) + { + for ( int i = 0; i < length; i++ ) + { + if ( text[i] == point ) { text[i] = '.'; } + } + } + out.raw( text, length ); + return true; +} + +inline bool TableJsonWriteValue( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth ); +// a UNION ARM that names no declaration writes through the field walk one key +// down (docs/SPEC-TABLES.md §2.6, §16.2), which is defined below +inline bool TableJsonWriteField( TableJsonOut & out, const void * base, const TableFieldInfo * f, int32_t depth ); + +// one scalar, at one storage address: a nested object, a union, a +// vocabulary, or a number +inline bool TableJsonWriteScalar( TableJsonOut & out, const void * storage, const TableFieldInfo * f, int32_t depth ) +{ + if ( f->arms != NULL ) + { + // a union is an object with ONE key, the arm's name; None is {} + const TableUnionInfo * arms = f->arms(); + uint64_t tag = TableJsonGetRaw( (const uint8_t *) storage + arms->tag_offset, arms->tag_size ); + if ( tag == 0 ) + { + out.raw( "{}", 2 ); + return true; + } + if ( (int64_t) tag > f->enum_max ) + { + return false; // a tag no arm names, exactly as measure refuses it + } + const char * arm = f->enum_name( tag ); + // and refuse on the NAME, not merely on the bound: §16.2 says a value + // no variant NAMES is refused, so the check is the name. Writing + // whatever came back would emit "???", a spelling the reader counts + // as unknown — a silent round-trip loss in place of a refusal. + if ( !TableJsonNamed( arm ) ) { return false; } + out.put( '{' ); + out.line( depth + 1 ); + TableJsonWriteString( out, arm, (int32_t) strlen( arm ) ); + out.raw( ": ", 2 ); + // THE ARM'S VALUE TAKES THE ARM'S OWN ROW (§16.2): an arm that names + // no declaration carries the FIELD descriptor a field of its type + // would carry, offsets taken inside the union storage (§2.6), so the + // value walks through the field writer one key down. + if ( arms->arms[tag].field != NULL ) + { + if ( !TableJsonWriteField( out, storage, arms->arms[tag].field, depth + 1 ) ) + { + return false; + } + } + else if ( arms->arms[tag].table == NULL ) + { + out.raw( "null", 4 ); // a payload-free arm: the name selects it (§2.6) + } + else if ( !TableJsonWriteValue( out, (const uint8_t *) storage + arms->arms[tag].offset, arms->arms[tag].table, depth + 1 ) ) + { + return false; + } + out.line( depth ); + out.put( '}' ); + return true; + } + if ( f->kind == 13 ) + { + return TableJsonWriteValue( out, storage, f->table, depth ); + } + if ( TableJsonIsEnum( f ) ) + { + uint64_t value = TableJsonGetRaw( storage, f->elem_size ); + // a value no variant names has no text spelling, exactly as it has no + // wire identity: the writer REFUSES rather than writing None over it, + // the rule measure and save already apply (docs/SPEC-TABLES.md §5) + if ( (int64_t) value > f->enum_max ) { return false; } + if ( value != 0 && f->variant_id( value ) == 0 ) { return false; } + const char * name = f->enum_name( value ); + if ( !TableJsonNamed( name ) ) { return false; } + TableJsonWriteString( out, name, (int32_t) strlen( name ) ); + return true; + } + if ( TableJsonIsFlags( f ) ) + { + uint64_t bits = TableJsonGetRaw( storage, f->elem_size ); + if ( bits == 0 ) + { + out.raw( "[]", 2 ); + return true; + } + out.put( '[' ); + bool first = true; + for ( int64_t bit = 0; bit < 64; bit++ ) + { + if ( ( bits & ( uint64_t( 1 ) << bit ) ) == 0 ) { continue; } + if ( bit > f->enum_max ) + { + return false; // a bit no variant names has no text spelling + } + const char * name = f->enum_name( (uint64_t) bit ); + if ( !TableJsonNamed( name ) ) { return false; } + if ( !first ) { out.put( ',' ); } + first = false; + out.line( depth + 1 ); + TableJsonWriteString( out, name, (int32_t) strlen( name ) ); + } + out.line( depth ); + out.put( ']' ); + return true; + } + switch ( f->kind ) + { + case 1: + out.text( TableJsonGetRaw( storage, f->elem_size ) != 0 ? "true" : "false" ); + return true; + case 10: + { + float v = 0.0f; + memcpy( &v, storage, sizeof( v ) ); + return TableJsonWriteFloat( out, (double) v, true ); + } + case 11: + { + double v = 0.0; + memcpy( &v, storage, sizeof( v ) ); + return TableJsonWriteFloat( out, v, false ); + } + case 2: case 3: case 4: case 5: + TableJsonWriteSigned( out, TableJsonGetSigned( storage, f->elem_size ) ); + return true; + default: + if ( TableJsonKindWide( f->kind ) ) + { + TableJsonWriteWide( out, storage, f ); + return true; + } + TableJsonWriteUnsigned( out, TableJsonGetRaw( storage, f->elem_size ) ); + return true; + } +} + +inline bool TableJsonWriteField( TableJsonOut & out, const void * base, const TableFieldInfo * f, int32_t depth ) +{ + const uint8_t * storage = (const uint8_t *) base + f->offset; + if ( TableJsonIsMap( f ) ) + { + return TableJsonWriteMap( out, (const void *) storage, f, depth ); + } + if ( TableJsonIsList( f ) ) + { + return TableJsonWriteList( out, (const void *) storage, f, depth ); + } + if ( f->kind == 17 && !f->is_array ) + { + return TableJsonWritePointer( out, storage, f, depth ); + } + if ( f->kind == 17 ) + { + // an ARRAY OF POINTERS (§2.1): the pointer row per element — the + // pointee's object in place, null, or `&node` for a shared one (§16.7) + int32_t count = TableJsonCount( base, f ); + if ( count == 0 ) { out.raw( "[]", 2 ); return true; } + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + if ( !TableJsonWritePointer( out, storage + (int64_t) i * f->elem_size, f, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( ']' ); + return true; + } + if ( f->kind == 12 ) + { + TableJsonWriteString( out, (const char *) storage, TableJsonCount( base, f ) ); + return true; + } + if ( f->kind == 33 ) + { + TableJsonWriteWString( out, (const char16_t *) (const void *) storage, TableJsonCount( base, f ) ); + return true; + } + if ( TableJsonIsBytes( f ) ) + { + TableJsonWriteBase64( out, storage, TableJsonCount( base, f ) ); + return true; + } + if ( TableJsonIsKeyed( f ) ) + { + // one entry per SLOT, keyed by the variant that owns it, so inserting + // a variant next season moves nothing in the text either. Slot i holds + // the key i + 1: nothing is stored for None, so nothing is written for it. + out.put( '{' ); + bool first = true; + for ( int64_t slot = 0; slot < f->array_bound; slot++ ) + { + if ( !TableJsonKeyedSlotValid( f, slot ) ) { continue; } + if ( !first ) { out.put( ',' ); } + first = false; + out.line( depth + 1 ); + const char * key = f->key_name( TableJsonKeyedSlotKey( slot ) ); + TableJsonWriteString( out, key, (int32_t) strlen( key ) ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteScalar( out, storage + slot * f->elem_size, f, depth + 1 ) ) + { + return false; + } + } + if ( first ) { out.raw( "}", 1 ); return true; } + out.line( depth ); + out.put( '}' ); + return true; + } + if ( f->is_array ) + { + int32_t count = TableJsonCount( base, f ); + if ( count == 0 ) + { + out.raw( "[]", 2 ); + return true; + } + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + if ( !TableJsonWriteScalar( out, storage + (int64_t) i * f->elem_size, f, depth + 1 ) ) + { + return false; + } + } + out.line( depth ); + out.put( ']' ); + return true; + } + return TableJsonWriteScalar( out, storage, f, depth ); +} + +// One instance's fields, in DECLARATION ORDER, defaults included — a text is +// for people and tools, and a text that elides is a text a reader has to know +// the schema to complete. `any` says whether the object is already open on +// entry — a shared node's `&node` opens it before the fields (§16.7) — and +// whether it is open on return. +inline bool TableJsonWriteFields( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth, bool & any ) +{ + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + const TableFieldInfo * f = &info->fields[i]; + if ( f->guard[0] != 0 && !TableJsonGuardHolds( base, info, f->guard ) ) { continue; } + // an ABSENT optional writes no key: presence of the key IS the + // presence (§16.2), so an absent field is an absent key and nothing + // else would read back as absent + if ( f->optional && + TableJsonGetRaw( (const uint8_t *) base + f->present_offset, 1 ) == 0 ) + { + continue; + } + if ( !any ) { out.put( '{' ); } + else { out.put( ',' ); } + any = true; + out.line( depth + 1 ); + TableJsonWriteString( out, f->json, (int32_t) strlen( f->json ) ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteField( out, base, f, depth + 1 ) ) { return false; } + } + return true; +} + +// One instance as one object. The writer carries the reader's depth cap +// (§16.2): a pointer chain nests as deep as it is long (§16.7), and a text the +// writer produced past the cap would be a text the reader refuses. +inline bool TableJsonWriteValue( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { return false; } + bool any = false; + if ( !TableJsonWriteFields( out, base, info, depth, any ) ) { return false; } + if ( !any ) + { + out.raw( "{}", 2 ); + return true; + } + out.line( depth ); + out.put( '}' ); + return true; +} + +// ---- reading ---- + +struct TableJsonIn +{ + const char * text; + int64_t size; + int64_t pos; + TableReport * report; + bool bad; // the text is not JSON: the walk stops and keeps what it placed + void * graph; // the pointered read's builder and label map (§16.7); NULL for a fixed table +}; + +inline void TableJsonSpace( TableJsonIn & in ) +{ + while ( in.pos < in.size ) + { + char c = in.text[in.pos]; + if ( c == ' ' || c == '\t' || c == '\n' || c == '\r' ) { in.pos++; continue; } + // COMMENTS ARE ACCEPTED ON READ AND NEVER WRITTEN (docs/SPEC-TABLES.md + // §16.2): a line comment runs to the end of the line or of the input, + // a block comment to its closing delimiter, which does not nest, and + // an UNCLOSED block comment is malformed on the terms an unclosed + // string is. Both are legal wherever whitespace is; a lone slash is not JSON. + if ( c == '/' && in.pos + 1 < in.size && in.text[in.pos + 1] == '/' ) + { + in.pos += 2; + while ( in.pos < in.size && in.text[in.pos] != '\n' ) { in.pos++; } + continue; + } + if ( c == '/' && in.pos + 1 < in.size && in.text[in.pos + 1] == '*' ) + { + int64_t at = in.pos + 2; + while ( at + 1 < in.size && !( in.text[at] == '*' && in.text[at + 1] == '/' ) ) { at++; } + if ( at + 1 >= in.size ) { in.bad = true; in.pos = in.size; return; } + in.pos = at + 2; + continue; + } + if ( c == '/' ) { in.bad = true; } + return; + } +} + +inline char TableJsonPeek( TableJsonIn & in ) +{ + TableJsonSpace( in ); + return in.pos < in.size ? in.text[in.pos] : 0; +} + +// the shape of the value sitting at the cursor, without consuming it +inline char TableJsonValueShape( TableJsonIn & in ) +{ + char c = TableJsonPeek( in ); + switch ( c ) + { + case '{': return 'o'; + case '[': return 'a'; + case '"': return 's'; + case 't': case 'f': return 'b'; + case 'n': return 'z'; + case 0: return 0; + default: return 'n'; + } +} + +inline bool TableJsonLiteral( TableJsonIn & in, const char * word ) +{ + int64_t length = (int64_t) strlen( word ); + if ( in.pos + length > in.size || memcmp( in.text + in.pos, word, (size_t) length ) != 0 ) + { + in.bad = true; + return false; + } + in.pos += length; + return true; +} + +// one \uXXXX escape body; -1 when the four hex digits are not there +inline int TableJsonHex4( TableJsonIn & in ) +{ + if ( in.pos + 4 > in.size ) { return -1; } + int value = 0; + for ( int i = 0; i < 4; i++ ) + { + char c = in.text[in.pos + i]; + int digit; + if ( c >= '0' && c <= '9' ) { digit = c - '0'; } + else if ( c >= 'a' && c <= 'f' ) { digit = c - 'a' + 10; } + else if ( c >= 'A' && c <= 'F' ) { digit = c - 'A' + 10; } + else { return -1; } + value = ( value << 4 ) | digit; + } + in.pos += 4; + return value; +} + + +// One STRING BODY CHARACTER at the cursor, encoded into unit as UTF-8 and +// its length answered: an escape's code point, or a UTF-8 sequence read +// whole. It is ONE grammar serving both text kinds — the narrow scan places +// these bytes and the wide scan converts them back to code units — so the +// escape table, the lone-surrogate rule and the U+FFFD replacement are stated +// once. false means the text is not JSON and in.bad says so; a returned +// length of 0 means the closing quote was consumed and the string is done. +inline bool TableJsonScanUnit( TableJsonIn & in, char * unit, int32_t * unit_length_out ) +{ + int32_t unit_length = 0; + *unit_length_out = 0; + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos]; + if ( c == '"' ) { in.pos++; return true; } + if ( c == '\\' ) + { + in.pos++; + if ( in.pos >= in.size ) { in.bad = true; return false; } + char escape = in.text[in.pos++]; + switch ( escape ) + { + case '"': unit[0] = '"'; unit_length = 1; break; + case '\\': unit[0] = '\\'; unit_length = 1; break; + case '/': unit[0] = '/'; unit_length = 1; break; + case 'b': unit[0] = '\b'; unit_length = 1; break; + case 'f': unit[0] = '\f'; unit_length = 1; break; + case 'n': unit[0] = '\n'; unit_length = 1; break; + case 'r': unit[0] = '\r'; unit_length = 1; break; + case 't': unit[0] = '\t'; unit_length = 1; break; + case 'u': + { + int high = TableJsonHex4( in ); + if ( high < 0 ) { in.bad = true; return false; } + uint32_t code = (uint32_t) high; + if ( high >= 0xd800 && high <= 0xdbff && in.pos + 2 <= in.size && + in.text[in.pos] == '\\' && in.text[in.pos + 1] == 'u' ) + { + int64_t mark = in.pos; + in.pos += 2; + int low = TableJsonHex4( in ); + if ( low >= 0xdc00 && low <= 0xdfff ) + { + code = 0x10000 + ( ( (uint32_t) high - 0xd800 ) << 10 ) + ( (uint32_t) low - 0xdc00 ); + } + else + { + in.pos = mark; // a lone lead surrogate rides as itself + } + } + // a surrogate half that never found its partner has no + // UTF-8 encoding: encoding it anyway would manufacture + // CESU-8 — invalid UTF-8 — out of input that was valid + // JSON, so it reads as the replacement character + if ( code >= 0xd800 && code <= 0xdfff ) { code = 0xfffd; } + unit_length = TableJsonEncodeUtf8( code, unit ); + break; + } + default: in.bad = true; return false; + } + } + else if ( (unsigned char) c < 0x20 ) + { + in.bad = true; // a raw control character is not a JSON string body + return false; + } + else + { + // a UTF-8 sequence read WHOLE, so the clamp below can only land + // between code points. Only bytes that ACTUALLY look like + // continuations are taken: the wire imposes no encoding (§3), so + // a string may legitimately hold a stray lead byte, and one at + // the end of a text must not swallow the closing quote. + unsigned char lead = (unsigned char) c; + int32_t want = 1; + if ( ( lead & 0xe0 ) == 0xc0 ) { want = 2; } + else if ( ( lead & 0xf0 ) == 0xe0 ) { want = 3; } + else if ( ( lead & 0xf8 ) == 0xf0 ) { want = 4; } + unit[0] = c; + in.pos++; + unit_length = 1; + while ( unit_length < want && in.pos < in.size && + ( (unsigned char) in.text[in.pos] & 0xc0 ) == 0x80 ) + { + unit[unit_length++] = in.text[in.pos++]; + } + // A SEQUENCE THAT IS NOT A CODE POINT READS AS U+FFFD, which is + // §16.3's rule at the point the defect ENTERS rather than at the + // point it leaves: a lone surrogate escape already reads that way + // above, RFC 8259 requires a JSON text to be valid UTF-8, and a + // kind 12 payload is well-formed UTF-8 (§3), so storage the text + // form built has to be storage the wire can carry (§5). + if ( !TableUtf8Valid( (const uint8_t *) unit, unit_length ) ) + { + unit_length = TableJsonEncodeUtf8( 0xfffd, unit ); + } + } + } + *unit_length_out = unit_length; + return true; +} + +// Scan one JSON string into a caller buffer. Bytes are appended ONE CODE +// POINT AT A TIME — an escape's encoding, or a UTF-8 sequence read whole — +// so a string longer than the field is clamped AT A CODE POINT BOUNDARY and +// never cut through a multi-byte character. Clamping is counted, never +// fatal, exactly as it is on the wire (§4). A NULL destination scans past a +// string without keeping it. +// +// A CALLER THAT TAKES clamped_out OWNS THE COUNTER. The value paths leave it +// NULL, and the clamp is a value's clamp, counted here. A MAP KEY takes it, +// because a key never clamps: a key this buffer could not hold whole is not a +// shorter key, and its entry drops instead (§2.8). +inline bool TableJsonScanString( TableJsonIn & in, char * out, int32_t capacity, int32_t * length, + bool * clamped_out = NULL ) +{ + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + int32_t placed = 0; + bool clamped = false; + for ( ;; ) + { + char unit[4]; + int32_t unit_length = 0; + if ( !TableJsonScanUnit( in, unit, &unit_length ) ) { return false; } + if ( unit_length == 0 ) { break; } + if ( out == NULL ) + { + placed += unit_length; // measured and not kept: a byte buffer's read sizes its node this way (§2.5) + } + else if ( !clamped && placed + unit_length <= capacity ) + { + memcpy( out + placed, unit, (size_t) unit_length ); + placed += unit_length; + } + else + { + // A CLAMP IS A PREFIX. Once one code point does not fit, the scan + // stops placing: a later SHORTER code point sliding into the room + // the long one left would store a string the input never spelled, + // and one clamped count cannot tell the two apart. + clamped = true; + } + } + if ( clamped_out != NULL ) { *clamped_out = clamped; } + else if ( clamped ) { in.report->clamped++; } + if ( length != NULL ) { *length = placed; } + return true; +} + +// One UTF-8 sequence back to its CODE POINT, over bytes TableJsonScanUnit +// produced and therefore already well formed. It is the inverse of +// TableJsonEncodeUtf8 and nothing more. +inline uint32_t TableJsonDecodeUtf8( const char * unit, int32_t unit_length ) +{ + const unsigned char lead = (unsigned char) unit[0]; + if ( unit_length == 1 ) { return lead; } + uint32_t code = lead & ( unit_length == 2 ? 0x1fu : ( unit_length == 3 ? 0x0fu : 0x07u ) ); + for ( int32_t i = 1; i < unit_length; i++ ) + { + code = ( code << 6 ) | (uint32_t) ( (unsigned char) unit[i] & 0x3f ); + } + return code; +} + +// Scan one JSON string into a caller buffer of UTF-16 CODE UNITS: the wstring +// row of §16.2, the text TRANSCODED at the boundary. It shares +// TableJsonScanUnit with the narrow scan, so the escape grammar and the +// lone-surrogate rule are one grammar, and appends ONE CODE POINT AT A TIME — +// one unit below U+10000 and a surrogate PAIR above it. A string longer than +// the field is therefore clamped at N code units with a pair never split, and +// a high surrogate left without its low half is dropped with it, which is the +// same sentence the wire's clamp takes (§3). Clamping is counted, never fatal. +inline bool TableJsonScanWString( TableJsonIn & in, char16_t * out, int32_t capacity, int32_t * length ) +{ + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + int32_t placed = 0; + bool clamped = false; + for ( ;; ) + { + char unit[4]; + int32_t unit_length = 0; + if ( !TableJsonScanUnit( in, unit, &unit_length ) ) { return false; } + if ( unit_length == 0 ) { break; } + const uint32_t code = TableJsonDecodeUtf8( unit, unit_length ); + char16_t units[2]; + int32_t units_length = 1; + if ( code < 0x10000 ) + { + units[0] = (char16_t) code; + } + else + { + const uint32_t rest = code - 0x10000; + units[0] = (char16_t) ( 0xd800 + ( rest >> 10 ) ); + units[1] = (char16_t) ( 0xdc00 + ( rest & 0x3ff ) ); + units_length = 2; + } + if ( out == NULL ) + { + placed += units_length; + } + else if ( !clamped && placed + units_length <= capacity ) + { + for ( int32_t i = 0; i < units_length; i++ ) { out[placed + i] = units[i]; } + placed += units_length; + } + else + { + // A CLAMP IS A PREFIX, the narrow scan's own rule: once one code + // point does not fit, the scan stops placing. A pair is placed + // whole or not at all, so no clamp can leave an unpaired + // surrogate in storage. + clamped = true; + } + } + if ( clamped ) { in.report->clamped++; } + if ( length != NULL ) { *length = placed; } + return true; +} + + +// the numeric token at the cursor, copied out whole; false = not a number +// Scan one number, to JSON's OWN grammar (RFC 8259 §6) and not to a run of +// number-ish characters: +// +// number = [ "-" ] int [ frac ] [ exp ] +// int = "0" / ( digit1-9 *digit ) +// frac = "." 1*digit +// exp = ( "e" / "E" ) [ "-" / "+" ] 1*digit +// +// Scanning the production is what makes a typo in an authoring file a +// DIAGNOSTIC rather than a value: "1-2" scans as 1 and leaves "-2" where the +// object expects a comma, so the text is malformed — which is what §16.2 +// already promises. A permissive scan would hand "1-2" to a digit loop and +// report a clamp, and a config pipeline would never hear about it. Leading +// "+", leading zeros, ".5" and "3." are not JSON either. +inline bool TableJsonWalkNumber( TableJsonIn & in, bool * integral ) +{ + TableJsonSpace( in ); + bool whole = true; + if ( in.pos < in.size && in.text[in.pos] == '-' ) { in.pos++; } + // int: a lone zero, or a non-zero digit and any digits after it + if ( in.pos >= in.size ) { return false; } + if ( in.text[in.pos] == '0' ) + { + in.pos++; + } + else if ( in.text[in.pos] >= '1' && in.text[in.pos] <= '9' ) + { + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + } + else + { + return false; + } + // frac + if ( in.pos < in.size && in.text[in.pos] == '.' ) + { + in.pos++; + if ( in.pos >= in.size || in.text[in.pos] < '0' || in.text[in.pos] > '9' ) { return false; } + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + whole = false; + } + // exp + if ( in.pos < in.size && ( in.text[in.pos] == 'e' || in.text[in.pos] == 'E' ) ) + { + in.pos++; + if ( in.pos < in.size && ( in.text[in.pos] == '-' || in.text[in.pos] == '+' ) ) { in.pos++; } + if ( in.pos >= in.size || in.text[in.pos] < '0' || in.text[in.pos] > '9' ) { return false; } + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + whole = false; + } + *integral = whole; + return true; +} + +// the same production, with the token kept for conversion +inline bool TableJsonScanNumber( TableJsonIn & in, char * token, int32_t capacity, int32_t * length, bool * integral ) +{ + TableJsonSpace( in ); + int64_t start = in.pos; + if ( !TableJsonWalkNumber( in, integral ) ) { return false; } + int64_t count = in.pos - start; + if ( count <= 0 || count >= capacity ) { return false; } + memcpy( token, in.text + start, (size_t) count ); + token[count] = 0; + *length = (int32_t) count; + return true; +} + +// the token's exact double, through the runtime's own converter — which +// speaks the LOCALE's decimal point, so the token crosses back over that +// character on its way in +inline double TableJsonTokenDouble( const char * token, int32_t length, bool single ) +{ + char work[kTableJsonMaxNumber]; + memcpy( work, token, (size_t) length ); + work[length] = 0; + char point = TableJsonDecimalPoint(); + if ( point != '.' ) + { + for ( int32_t i = 0; i < length; i++ ) + { + if ( work[i] == '.' ) { work[i] = point; } + } + } + if ( single ) { return (double) strtof( work, NULL ); } + return strtod( work, NULL ); +} + +// ---- ONE CHECKED NUMERIC INTERPRETATION (docs/SPEC-TABLES.md §16.2) ---- +// +// JSON HAS ONE NUMBER TYPE, so every integer target reads a token the same way +// and the VALUE decides rather than the spelling: 2, 2.0 and 1e3 are the +// integers 2, 2 and 1000. What comes out of a token is a SIGN, a MAGNITUDE and +// a STATUS, and nothing on the way is cast through a type that cannot hold what +// it is handed. A uint64 magnitude past INT64_MAX is a magnitude and never a +// negative, and a double is consulted only for a spelling the digit path cannot +// read exactly. +// +// TWO POLICIES SIT ON TOP OF THE ONE VALUE and neither reinterprets the token: +// an ordinary FIELD clamps to its domain and counts, and a MAP KEY rejects the +// whole entry, because a key is an identity and a clamped one is two entries +// merged. That difference is the only difference between them. +// +// THE KEY READS ITS TOKEN EXACTLY AND A FIELD READS IT THROUGH THE DOUBLE, and +// that is the one place the two interpretations part. A key is an IDENTITY, so +// two spellings a 53-bit mantissa cannot tell apart are two keys and the key +// path carries TableJsonInterpretExact below. A field's value is a quantity +// under a clamp, and its interpretation is the one the C, Go and Rust ports +// read the same texts with, so it lives here and reads as they read. +struct TableJsonInteger +{ + uint64_t magnitude; // |value|, exact for every integral token 64 bits hold + bool negative; + bool fractional; // a genuinely fractional VALUE: the wrong shape for an integer + bool saturated; // a magnitude past what 64 bits hold, held at that edge + bool finite; // false: no integer target holds it at all +}; + +// THE FIELD'S INTERPRETATION: the token, parsed digit by digit so no width and +// no locale can move it, and through the runtime's converter only where the +// spelling carries a fraction or an exponent +inline TableJsonInteger TableJsonInterpret( const char * token, int32_t length, bool integral ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = false; + out.fractional = false; + out.saturated = false; + out.finite = true; + if ( integral ) + { + int32_t i = 0; + if ( i < length && token[i] == '-' ) // WalkNumber refuses a leading plus + { + out.negative = true; + i++; + } + for ( ; i < length; i++ ) + { + const uint64_t digit = (uint64_t) ( token[i] - '0' ); + if ( out.magnitude > ( UINT64_MAX - digit ) / 10 ) + { + out.magnitude = UINT64_MAX; + out.saturated = true; + break; + } + out.magnitude = out.magnitude * 10 + digit; + } + if ( out.magnitude == 0 ) { out.negative = false; } // -0 IS zero + return out; + } + const double d = TableJsonTokenDouble( token, length, false ); + if ( !TableJsonFinite( d ) ) { out.finite = false; return out; } + out.negative = d < 0; + const double whole = out.negative ? -d : d; + // THE DOMAIN IS ESTABLISHED BEFORE THE CAST: a magnitude past what sixty-four + // bits hold is answered here, so no value ever reaches a conversion that is + // undefined for it + if ( whole >= 18446744073709551616.0 ) + { + out.magnitude = UINT64_MAX; + out.saturated = true; + return out; + } + const uint64_t truncated = (uint64_t) whole; + if ( (double) truncated != whole ) { out.fractional = true; return out; } + out.magnitude = truncated; + if ( out.magnitude == 0 ) { out.negative = false; } + return out; +} + +// THE DECIMAL BAND a token is answered in without arithmetic: 10^20 is above +// UINT64_MAX whatever the digits are, so a point past it saturates and a token +// spelling 1e999999999 costs nothing to refuse. +const int64_t kTableJsonDecimalBand = 20; + +// THE MAP KEY'S INTERPRETATION, and no other path's: the token's own digits, +// read where they stand, so no 53-bit mantissa decides the identity of a 64-bit +// key. The int and frac runs are one digit string with the point after "point" +// of them, and the exponent moves the point rather than the digits, which is +// the normalization the wide kinds already use over one 64-bit lane. A zero +// fraction is the integer the token spells at every magnitude the kind holds, +// so 9007199254740993.0 is that key rather than the one a double rounds it to. +// No exact reader has an infinity, so finite is true here always. +inline TableJsonInteger TableJsonInterpretExact( const char * token, int32_t length ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = false; + out.fractional = false; + out.saturated = false; + out.finite = true; + int32_t i = 0; + if ( i < length && token[i] == '-' ) { out.negative = true; i++; } // WalkNumber refuses a leading plus + const char * int_digits = token + i; + int32_t int_len = 0; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { int_len++; i++; } + const char * frac_digits = token + i; + int32_t frac_len = 0; + if ( i < length && token[i] == '.' ) + { + i++; + frac_digits = token + i; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { frac_len++; i++; } + } + int64_t exp = 0; + if ( i < length && ( token[i] == 'e' || token[i] == 'E' ) ) + { + i++; + bool exp_negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { exp_negative = token[i] == '-'; i++; } + while ( i < length && token[i] >= '0' && token[i] <= '9' ) + { + if ( exp < 100000 ) { exp = exp * 10 + ( token[i] - '0' ); } + i++; + } + if ( exp_negative ) { exp = -exp; } + } + // leading and trailing zeros stripped, so the last digit kept is significant + int32_t start = 0, end = int_len + frac_len; + int64_t point = (int64_t) int_len + exp; + while ( start < end && ( start < int_len ? int_digits[start] : frac_digits[start - int_len] ) == '0' ) { start++; point--; } + while ( end > start && ( end - 1 < int_len ? int_digits[end - 1] : frac_digits[end - 1 - int_len] ) == '0' ) { end--; } + const int64_t digits = end - start; + if ( digits == 0 ) { out.negative = false; return out; } // the value is zero, and -0 IS zero + if ( point < digits ) { out.fractional = true; return out; } // a significant digit below the point + if ( point > kTableJsonDecimalBand ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + for ( int32_t k = start; k < end; k++ ) + { + const uint64_t digit = (uint64_t) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ); + if ( out.magnitude > ( UINT64_MAX - digit ) / 10 ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + out.magnitude = out.magnitude * 10 + digit; + } + for ( int64_t k = digits; k < point; k++ ) // the point's own zeros, which no digit spells + { + if ( out.magnitude > UINT64_MAX / 10 ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + out.magnitude *= 10; + } + return out; +} + +// a declared range bound as the same value. A bound is inside the field's own +// domain by construction, so nothing here saturates. +inline TableJsonInteger TableJsonIntegerOf( double bound ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = bound < 0; + out.fractional = false; + out.saturated = false; + out.finite = true; + const double whole = out.negative ? -bound : bound; + out.magnitude = whole >= 18446744073709551616.0 ? UINT64_MAX : (uint64_t) whole; + if ( out.magnitude == 0 ) { out.negative = false; } + return out; +} + +// THE TARGET DOMAIN, established before the value reaches storage: the bytes of +// storage, signed or not. Answers what the target holds and whether the domain +// MOVED it. An unsigned magnitude above INT64_MAX rides out as its bit pattern, +// which is the storage's own image of it and not a negative number. +inline int64_t TableJsonIntegerInDomain( const TableJsonInteger & number, bool is_signed, int32_t bytes, bool & moved ) +{ + moved = false; + uint64_t magnitude = number.magnitude; + if ( is_signed ) + { + const uint64_t high = bytes >= 8 ? (uint64_t) INT64_MAX : ( ( uint64_t( 1 ) << ( bytes * 8 - 1 ) ) - 1 ); + if ( number.negative ) + { + const uint64_t low = high + 1; // the floor's magnitude + if ( magnitude > low ) { magnitude = low; moved = true; } + return (int64_t) ( ~magnitude + 1 ); // two's complement, INT64_MIN included + } + if ( magnitude > high ) { magnitude = high; moved = true; } + return (int64_t) magnitude; + } + // A NEGATIVE TOKEN IN AN UNSIGNED FIELD CLAMPS TO ZERO, and -0 is zero, + // which is why the sign is dropped at a zero magnitude above + if ( number.negative ) { moved = true; return 0; } + const uint64_t high = bytes >= 8 ? UINT64_MAX : ( ( uint64_t( 1 ) << ( bytes * 8 ) ) - 1 ); + if ( magnitude > high ) { magnitude = high; moved = true; } + return (int64_t) magnitude; +} + +// A number token into a wide kind's raw storage (docs/SPEC-TABLES.md §16.2). A +// 128-bit integer takes any token whose VALUE is integral; a fixed field any +// token whose value is EXACTLY representable in its Q I.F — a finer fraction +// is the wrong shape for the field, counted as a kind mismatch and never +// rounded, the rule SPEC.md §4.6 gives a fixed default. A magnitude past 128 +// bits saturates and counts as a clamp, as an int64 field saturates at +// INT64_MAX; the declared range clamps after it, on the RAW scale, as it does +// for every bounded scalar. +// +// The token is normalized to its digits with the decimal point after "point" +// of them. An integer part past 40 digits is above 2^128 whatever the digits +// are, and a value below 10^-40 is finer than 2^-127, the finest fraction any +// F can spell — so outside that band the answer is known without the +// arithmetic, and a token spelling 1e999999999 costs nothing to refuse. +inline bool TableJsonReadWide( TableJsonIn & in, const char * token, int32_t length, void * storage, const TableFieldInfo * f ) +{ + bool is_signed = TableJsonKindWideSigned( f->kind ); + int frac = f->frac_bits; + int32_t i = 0; + bool negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { negative = token[i] == '-'; i++; } + const char * int_digits = token + i; + int32_t int_len = 0; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { int_len++; i++; } + const char * frac_digits = token + i; + int32_t frac_len = 0; + if ( i < length && token[i] == '.' ) + { + i++; + frac_digits = token + i; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { frac_len++; i++; } + } + int64_t exp = 0; + if ( i < length && ( token[i] == 'e' || token[i] == 'E' ) ) + { + i++; + bool exp_negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { exp_negative = token[i] == '-'; i++; } + while ( i < length && token[i] >= '0' && token[i] <= '9' ) + { + if ( exp < 100000 ) { exp = exp * 10 + ( token[i] - '0' ); } + i++; + } + if ( exp_negative ) { exp = -exp; } + } + // the digits, with the point after "point" of them; leading and trailing + // zeros stripped. digit( k ) reads the k-th of the int and frac runs. + int32_t start = 0, end = int_len + frac_len; + int64_t point = int_len + exp; + while ( start < end && ( start < int_len ? int_digits[start] : frac_digits[start - int_len] ) == '0' ) { start++; point--; } + while ( end > start && ( end - 1 < int_len ? int_digits[end - 1] : frac_digits[end - 1 - int_len] ) == '0' ) { end--; } + + TableJsonWide raw = { 0, 0 }; + bool saturated = false; + TableJsonWide signed_max = { ~uint64_t( 0 ), ~uint64_t( 0 ) >> 1 }; + TableJsonWide signed_min = { 0, uint64_t( 1 ) << 63 }; + TableJsonWide unsigned_max = { ~uint64_t( 0 ), ~uint64_t( 0 ) }; + if ( start == end ) + { + // zero, and -0 IS zero + } + else if ( point > 40 ) + { + saturated = true; + if ( !negative ) { raw = is_signed ? signed_max : unsigned_max; } + else if ( is_signed ) { raw = signed_min; } + } + else if ( point < -40 ) + { + in.report->kind_mismatch++; // finer than any F can spell + return true; + } + else + { + // the fraction FIRST, so an inexact value is the wrong shape whatever + // its magnitude: its digits, with the zeros a negative point puts in + // front, doubled F times; each doubling's carry is the next bit, and + // the value is exact iff nothing is left after the last one + char fd[kTableJsonMaxNumber + 48]; + int32_t fn = 0; + for ( int64_t z = point; z < 0; z++ ) { fd[fn++] = 0; } + for ( int32_t k = (int32_t) ( point > 0 ? point : 0 ) + start; k < end; k++ ) + { + fd[fn++] = (char) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ); + } + TableJsonWide fraction = { 0, 0 }; + for ( int b = 0; b < frac; b++ ) + { + int carry = 0; + for ( int32_t k = fn - 1; k >= 0; k-- ) + { + int d = fd[k] * 2 + carry; + fd[k] = (char) ( d % 10 ); + carry = d / 10; + } + fraction = TableJsonWideShl( fraction, 1 ); + fraction.lo |= (uint64_t) carry; + } + for ( int32_t k = 0; k < fn; k++ ) + { + if ( fd[k] != 0 ) + { + in.report->kind_mismatch++; + return true; + } + } + // then the whole part, saturating past 128 bits + TableJsonWide whole = { 0, 0 }; + for ( int64_t k = start; k < start + point && !saturated; k++ ) + { + uint32_t digit = k < end ? (uint32_t) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ) : 0; + if ( TableJsonWideMulAdd( &whole, 10, digit ) != 0 ) { saturated = true; } + } + if ( !saturated && frac > 0 && !TableJsonWideZero( TableJsonWideShr( whole, 128 - frac ) ) ) { saturated = true; } + if ( !saturated ) + { + raw = TableJsonWideShl( whole, frac ); + raw.lo |= fraction.lo; + raw.hi |= fraction.hi; + } + if ( is_signed ) + { + if ( !saturated && !negative && TableJsonWideNegative( raw ) ) { saturated = true; } + if ( !saturated && negative && TableJsonWideCompare( raw, signed_min, false ) > 0 ) { saturated = true; } + if ( saturated ) { raw = negative ? signed_min : signed_max; } + else if ( negative ) { raw = TableJsonWideNeg( raw ); } + } + else + { + if ( saturated ) { raw = unsigned_max; } + if ( negative && !TableJsonWideZero( raw ) ) { raw.lo = 0; raw.hi = 0; saturated = true; } + } + } + if ( saturated ) { in.report->clamped++; } + if ( f->wide != NULL ) + { + TableJsonWide lo = { f->wide->lo[0], f->wide->lo[1] }; + TableJsonWide hi = { f->wide->hi[0], f->wide->hi[1] }; + if ( TableJsonWideCompare( raw, lo, is_signed ) < 0 ) { raw = lo; in.report->clamped++; } + else if ( TableJsonWideCompare( raw, hi, is_signed ) > 0 ) { raw = hi; in.report->clamped++; } + } + TableJsonWideStore( storage, f->elem_size, raw ); + return true; +} + +inline bool TableJsonSkipValue( TableJsonIn & in, int32_t depth ); + +inline bool TableJsonSkipContainer( TableJsonIn & in, char close, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; // the opening bracket + bool first = true; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == close ) { in.pos++; return true; } + if ( c == 0 ) { in.bad = true; return false; } + if ( close == '}' ) + { + // the key is kept, because a skipped OBJECT may still be a + // pointer's: an `&node` opening it names a node the storage could + // not hold, and the numbering has to survive the drop (§16.7). + // Anywhere but first, the prefix is the reserved key out of place + // — in a pointered unit; a fixed unit skips the value whole. + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + if ( key[0] == '&' && in.graph != NULL ) + { + if ( !first ) { in.report->malformed = true; in.bad = true; return false; } + if ( !TableJsonSkippedAmpersand( in, key, depth ) ) { return false; } + first = false; + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == close ) { in.pos++; return true; } + in.bad = true; + return false; + } + } + first = false; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == close ) { in.pos++; return true; } + in.bad = true; + return false; + } +} + +inline bool TableJsonSkipValue( TableJsonIn & in, int32_t depth ) +{ + char c = TableJsonPeek( in ); + switch ( c ) + { + case '{': return TableJsonSkipContainer( in, '}', depth ); + case '[': return TableJsonSkipContainer( in, ']', depth ); + case '"': return TableJsonScanString( in, NULL, 0, NULL ); + case 't': return TableJsonLiteral( in, "true" ); + case 'f': return TableJsonLiteral( in, "false" ); + case 'n': return TableJsonLiteral( in, "null" ); + case 0: in.bad = true; return false; + default: + { + // consumed, never converted: skipping needs no buffer, and this + // is the one walk a hostile text drives to the depth cap. It is + // the SAME production the value path scans, so an unknown key + // cannot smuggle past a number a named key would refuse. + bool integral = false; + if ( !TableJsonWalkNumber( in, &integral ) ) { in.bad = true; return false; } + return true; + } + } +} + +inline bool TableJsonReadTable( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth ); +// a UNION ARM that names no declaration reads through the field walk one key +// down (docs/SPEC-TABLES.md §2.6, §16.2), which is defined below +inline bool TableJsonReadField( TableJsonIn & in, void * base, const TableFieldInfo * f, int32_t depth ); + +// place one scalar at one storage address +inline bool TableJsonReadScalar( TableJsonIn & in, void * storage, const TableFieldInfo * f, int32_t depth ) +{ + if ( f->arms != NULL ) + { + // a union is an object with ONE key, the arm's name; {} is None, and + // two keys is a text this walk will not guess at + const TableUnionInfo * arms = f->arms(); + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + TableJsonSetRaw( (uint8_t *) storage + arms->tag_offset, arms->tag_size, 0 ); + if ( TableJsonPeek( in ) == '}' ) { in.pos++; return true; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t tag = 0; + for ( int64_t t = 1; t <= f->enum_max; t++ ) + { + if ( strcmp( f->enum_name( (uint64_t) t ), key ) == 0 ) { tag = t; break; } + } + if ( tag == 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + void * payload = (uint8_t *) storage + arms->arms[tag].offset; + const TableFieldInfo * arm = arms->arms[tag].field; + bool placed = true; + if ( arm != NULL ) + { + // THE ARM'S VALUE TAKES THE ARM'S OWN ROW (§16.2). A value of + // the wrong shape for that row is a KIND MISMATCH: the union + // reads None, the event is counted, and the enclosing object + // continues — the rule a FIELD's value lives under, one key + // down. A pointer arm's null is a null pointer, not a shape + // error, exactly as a pointer field's is (§16.7). + char got = TableJsonValueShape( in ); + if ( arm->kind == 17 && !arm->is_array && got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + memset( payload, 0, (size_t) arms->arms[tag].size ); + } + else if ( got != TableJsonShape( arm ) ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else if ( arm->kind == 17 && !arm->is_array ) + { + // A POINTER ARM'S VALUE IS THE POINTEE IN PLACE, or a + // node reference to one (§16.7) — the read a pointer + // FIELD takes, which is not the scalar walk + memset( payload, 0, (size_t) arms->arms[tag].size ); + if ( !TableJsonReadPointer( in, payload, arm, depth + 1 ) ) { return false; } + } + else + { + // SELECTION ZERO-ESTABLISHES THE ARM (SPEC §5): an arm + // takes no specified default, so zero is the establish + memset( payload, 0, (size_t) arms->arms[tag].size ); + if ( !TableJsonReadField( in, storage, arm, depth + 1 ) ) { return false; } + } + } + else if ( arms->arms[tag].table != NULL ) + { + if ( TableJsonValueShape( in ) != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else + { + arms->arms[tag].table->reset( payload ); + if ( !TableJsonReadTable( in, payload, arms->arms[tag].table, depth + 1 ) ) { return false; } + } + } + else + { + // A PAYLOAD-FREE ARM'S VALUE IS null (§2.6): the arm name + // selects it and there is nothing to place + if ( TableJsonValueShape( in ) != 'z' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else if ( !TableJsonLiteral( in, "null" ) ) + { + return false; + } + } + if ( placed ) + { + TableJsonSetRaw( (uint8_t *) storage + arms->tag_offset, arms->tag_size, (uint64_t) tag ); + } + } + char c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; c = TableJsonPeek( in ); } + if ( c == '}' ) { in.pos++; return true; } + in.bad = true; // a second key: a one-of with two arms is not a value + return false; + } + if ( f->kind == 13 ) + { + f->table->reset( storage ); + return TableJsonReadTable( in, storage, f->table, depth + 1 ); + } + if ( TableJsonIsEnum( f ) ) + { + char name[kTableJsonMaxKey]; + int32_t name_length = 0; + if ( !TableJsonScanString( in, name, kTableJsonMaxKey - 1, &name_length ) ) { return false; } + name[name_length] = 0; + for ( int64_t v = 0; v <= f->enum_max; v++ ) + { + if ( strcmp( f->enum_name( (uint64_t) v ), name ) == 0 ) + { + TableJsonSetRaw( storage, f->elem_size, (uint64_t) v ); + return true; + } + } + // a name this build cannot name reads as None and counts as unknown, + // exactly as an unknown variant id does on the wire (§4) + TableJsonSetRaw( storage, f->elem_size, 0 ); + in.report->unknown++; + return true; + } + if ( TableJsonIsFlags( f ) ) + { + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + in.pos++; + uint64_t bits = 0; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + if ( c != '"' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + char name[kTableJsonMaxKey]; + int32_t name_length = 0; + if ( !TableJsonScanString( in, name, kTableJsonMaxKey - 1, &name_length ) ) { return false; } + name[name_length] = 0; + bool found = false; + for ( int64_t bit = 0; bit <= f->enum_max; bit++ ) + { + if ( strcmp( f->enum_name( (uint64_t) bit ), name ) == 0 ) + { + bits |= uint64_t( 1 ) << bit; + found = true; + break; + } + } + if ( !found ) { in.report->unknown++; } + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + TableJsonSetRaw( storage, f->elem_size, bits ); + return true; + } + if ( f->kind == 1 ) + { + char c = TableJsonPeek( in ); + if ( c == 't' ) { if ( !TableJsonLiteral( in, "true" ) ) { return false; } TableJsonSetRaw( storage, f->elem_size, 1 ); return true; } + if ( !TableJsonLiteral( in, "false" ) ) { return false; } + TableJsonSetRaw( storage, f->elem_size, 0 ); + return true; + } + char token[kTableJsonMaxNumber]; + int32_t length = 0; + bool integral = false; + if ( !TableJsonScanNumber( in, token, kTableJsonMaxNumber, &length, &integral ) ) + { + in.bad = true; + return false; + } + if ( TableJsonKindWide( f->kind ) ) + { + return TableJsonReadWide( in, token, length, storage, f ); + } + if ( f->kind == 10 || f->kind == 11 ) + { + bool single = f->kind == 10; + double value = TableJsonTokenDouble( token, length, single ); + // A magnitude the field's format cannot hold is the WRONG SHAPE for + // the kind, and it never reaches storage: 1e400 is not a float64 and + // 1e300 is not a float32. Storing the infinity the conversion + // produced would leave an instance this walk called CLEAN that + // ToJsonMeasure then refuses forever (a non-finite float has no JSON + // spelling), and §16.1's one invariant is that a text which reads + // clean writes back. + if ( !TableJsonFinite( value ) ) + { + in.report->kind_mismatch++; + return true; + } + if ( f->has_range ) + { + if ( value < f->range_min ) { value = f->range_min; in.report->clamped++; } + else if ( value > f->range_max ) { value = f->range_max; in.report->clamped++; } + } + if ( single ) + { + float narrow = (float) value; + if ( !TableJsonFinite( (double) narrow ) ) + { + in.report->kind_mismatch++; + return true; + } + memcpy( storage, &narrow, sizeof( narrow ) ); + } + else + { + memcpy( storage, &value, sizeof( value ) ); + } + return true; + } + // AN ORDINARY FIELD'S POLICY over the one interpreted value: it CLAMPS to + // its domain and counts. JSON has one number type, so 2.0 IS the integer 2 + // and 1e3 IS 1000. A library that round-trips numbers through a double + // emits them that way, and this walker's own float writer emits 1e+21. Only + // a genuinely fractional value is the wrong shape for the kind. + const bool is_signed = f->kind >= 2 && f->kind <= 5; + const TableJsonInteger number = TableJsonInterpret( token, length, integral ); + if ( !number.finite || number.fractional ) + { + in.report->kind_mismatch++; + return true; + } + if ( number.saturated ) { in.report->clamped++; } // past what sixty-four bits hold + // THE DECLARED RANGE FIRST, THEN THE STORAGE WIDTH, the wire's order (§4), + // so a text and a wire loaded from the same data land the same instance. + // The comparison is on the value's OWN scale, correctly signed past + // INT64_MAX, where the storage's bit pattern is not a number to compare. + TableJsonInteger bounded = number; + if ( f->has_range ) + { + const double scale = number.negative ? -(double) number.magnitude : (double) number.magnitude; + if ( scale < f->range_min ) { bounded = TableJsonIntegerOf( f->range_min ); in.report->clamped++; } + else if ( scale > f->range_max ) { bounded = TableJsonIntegerOf( f->range_max ); in.report->clamped++; } + } + bool moved = false; + const int64_t value = TableJsonIntegerInDomain( bounded, is_signed, (int32_t) f->elem_size, moved ); + if ( moved ) { in.report->clamped++; } + TableJsonSetRaw( storage, f->elem_size, (uint64_t) value ); + return true; +} + +inline bool TableJsonReadField( TableJsonIn & in, void * base, const TableFieldInfo * f, int32_t depth ) +{ + uint8_t * storage = (uint8_t *) base + f->offset; + if ( TableJsonIsMap( f ) ) + { + return TableJsonReadMap( in, (void *) storage, f, depth ); + } + if ( TableJsonIsList( f ) ) + { + return TableJsonReadList( in, (void *) storage, f, depth ); + } + + if ( f->kind == 12 ) + { + int32_t length = 0; + if ( !TableJsonScanString( in, (char *) storage, f->array_bound, &length ) ) { return false; } + storage[length] = 0; + TableJsonSetCount( base, f, length ); + return true; + } + if ( f->kind == 33 ) + { + char16_t * units = (char16_t *) (void *) storage; + int32_t length = 0; + if ( !TableJsonScanWString( in, units, (int32_t) f->array_bound, &length ) ) { return false; } + units[length] = 0; // the terminating zero UNIT at index length (§7.2, SPEC.md §4.12) + TableJsonSetCount( base, f, length ); + return true; + } + if ( TableJsonIsBytes( f ) ) + { + // base64 decodes STRAIGHT INTO the field's storage, six bits at a + // time — no window, no temporary, so a bytes(N) of any declared + // extent reads the same way. A base64 body carries no escapes, so a + // backslash in one is simply not an alphabet character. + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + memset( storage, 0, (size_t) f->array_bound ); + TableJsonSetCount( base, f, 0 ); + const char * alphabet = TableJsonBase64Alphabet(); + int32_t placed = 0; + uint32_t accumulator = 0; + int32_t held = 0; + bool clamped = false; + bool malformed = false; + for ( ;; ) + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos++]; + if ( c == '"' ) { break; } + if ( c == '=' || malformed ) { continue; } + const char * at = c != 0 ? strchr( alphabet, c ) : NULL; + if ( at == NULL ) { malformed = true; continue; } + accumulator = ( accumulator << 6 ) | (uint32_t) ( at - alphabet ); + held += 6; + if ( held >= 8 ) + { + held -= 8; + if ( placed < f->array_bound ) + { + storage[placed++] = (uint8_t) ( ( accumulator >> held ) & 0xff ); + } + else + { + clamped = true; + } + } + } + if ( malformed ) + { + // a body that is not base64 is the wrong shape for the kind: the + // field keeps its default and the event is counted + in.report->kind_mismatch++; + return true; + } + if ( clamped ) { in.report->clamped++; } + TableJsonSetCount( base, f, placed ); + return true; + } + if ( TableJsonIsKeyed( f ) ) + { + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + // every slot back to its declared defaults first, so a key the text + // omits keeps them and a repeated field key cannot leave an earlier + // occurrence's slots standing + for ( int32_t i = 0; i < f->array_bound; i++ ) + { + void * slot = storage + (int64_t) i * f->elem_size; + if ( f->kind == 13 ) { f->table->reset( slot ); } + else { memset( slot, 0, (size_t) f->elem_size ); } + } + char shape = TableJsonElementShape( f ); + // A KEYED OBJECT'S KEYS ARE KEYS: a variant named twice is a duplicate + // key like any other, last-wins and counted (§16.2). Tracked the way + // a table's own field keys are — a bounded, allocation-free bitmask; + // a vocabulary wider than this still reads, its repeats simply stop + // being counted. + uint64_t seen[8] = {}; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t slot = -1; + for ( int64_t v = 0; v < f->array_bound; v++ ) + { + // nothing is stored for None, so "None" finds no slot and is + // an unknown key like any other name this reader cannot place + if ( !TableJsonKeyedSlotValid( f, v ) ) { continue; } + if ( strcmp( f->key_name( TableJsonKeyedSlotKey( v ) ), key ) == 0 ) { slot = v; break; } + } + if ( slot >= 0 && slot < 512 ) + { + uint64_t bit = uint64_t( 1 ) << ( slot & 63 ); + if ( ( seen[slot >> 6] & bit ) != 0 ) { in.report->duplicate++; } + seen[slot >> 6] |= bit; + } + if ( slot < 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( TableJsonValueShape( in ) != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadScalar( in, storage + slot * f->elem_size, f, depth + 1 ) ) + { + return false; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; + } + if ( f->is_array ) + { + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + in.pos++; + // LAST WINS has to be true of a repeated ARRAY key too, and it is + // wire-visible: a fixed array writes every slot, so a second, shorter + // occurrence overlaying a prefix would leave the first occurrence's + // tail standing. The field goes back to its declared defaults before + // this occurrence's elements are placed — the re-establishment a nested + // table and a union arm already get. A table element's defaults are + // its own (the reset hook); every other element kind's storage + // default is zero, which is what the generated array declares. + if ( f->kind == 13 ) + { + for ( int32_t i = 0; i < f->array_bound; i++ ) + { + f->table->reset( storage + (int64_t) i * f->elem_size ); + } + } + else + { + memset( storage, 0, (size_t) f->array_bound * (size_t) f->elem_size ); + } + TableJsonSetCount( base, f, 0 ); + int32_t placed = 0; + char shape = TableJsonElementShape( f ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + if ( placed >= f->array_bound ) + { + // more elements than the reader's bound: the bounded prefix + // is kept and the excess counts, the wire's rule (§4) + in.report->clamped++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( f->kind == 17 ) + { + // an element of an ARRAY OF POINTERS (§2.1): null is a null slot, an + // object is the pointee in place or an `&node` reference (§16.7) + char got = TableJsonValueShape( in ); + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( storage + (int64_t) placed * f->elem_size, f->elem_size, 0 ); + } + else if ( got != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, storage + (int64_t) placed * f->elem_size, f, depth + 1 ) ) { return false; } + placed++; + } + else if ( TableJsonValueShape( in ) != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed++; + } + else + { + if ( !TableJsonReadScalar( in, storage + (int64_t) placed * f->elem_size, f, depth + 1 ) ) { return false; } + placed++; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + // a fixed array's tail keeps the defaults the prefill left there, + // exactly as a short wire count does + TableJsonSetCount( base, f, placed ); + return true; + } + return TableJsonReadScalar( in, storage, f, depth ); +} + +inline bool TableJsonReadTableKeys( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth, const char * first_key ); + +// ONE table object: keys are field keys, unknown ones are skipped and +// counted, a repeated key is last-wins and counted. The instance is already +// at its declared defaults when this is entered, so a key the text never +// mentions keeps the default an absent field takes on the wire (§4). +inline bool TableJsonReadTable( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + return TableJsonReadTableKeys( in, base, info, depth, NULL ); +} + +// The keys of an object whose brace is already consumed. A pointer's object +// opens the same way a table's does, but its FIRST key may be `&node` (§16.7) +// and the adapter that reads it has to scan the key to know — so it hands the +// key it scanned in as `first_key`, with the colon consumed, and this places +// it before scanning the rest. +inline bool TableJsonReadTableKeys( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth, const char * first_key ) +{ + // duplicate tracking, bounded and allocation-free: a table with more + // fields than this still reads, its repeats simply stop being counted + uint64_t seen[8] = {}; + for ( ;; ) + { + char key[kTableJsonMaxKey]; + char c = 0; + if ( first_key != NULL ) + { + memcpy( key, first_key, strlen( first_key ) + 1 ); // scanned into a buffer this size by the caller + first_key = NULL; + } + else + { + c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; return true; } + if ( c == 0 ) { in.bad = true; return false; } + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + } + int32_t index = -1; + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + if ( strcmp( info->fields[i].json, key ) == 0 ) { index = i; break; } + } + if ( key[0] == '&' ) + { + // THE AMPERSAND PREFIX IS RESERVED TO THE FORM (docs/SPEC-TABLES.md + // §16.7). No declaration may take a key beginning with it, so this + // is never a field this build lacks — it is the sharing construct + // somewhere it cannot stand: `&node` is the FIRST key of a pointer's + // object and nothing else, and the adapter that reads a pointer + // has consumed it before these keys are read. MALFORMED, refused + // and counted; never counted as unknown, never skipped. + in.report->malformed = true; + in.bad = true; + return false; + } + if ( index < 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + const TableFieldInfo * f = &info->fields[index]; + if ( index < 512 ) + { + uint64_t bit = uint64_t( 1 ) << ( index & 63 ); + if ( ( seen[index >> 6] & bit ) != 0 ) { in.report->duplicate++; } + seen[index >> 6] |= bit; + } + // PRESENCE OF THE KEY IS THE PRESENCE (§16.2): reaching this line + // is the key being present, so an optional is set present + // whatever its value — with one exception the page names: a JSON + // null, which reads as ABSENT rather than as a value. + char got = TableJsonValueShape( in ); + if ( f->kind == 17 && !f->is_array ) + { + // a pointer: null is a null pointer, an object is the pointee + // in place or an `&node` reference to one (§16.7), a string is + // a BYTE BUFFER's bytes (§2.5), and anything else is the wrong + // shape for the kind + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) base + f->offset, f->elem_size, 0 ); + } + else if ( got != TableJsonShape( f ) ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, (uint8_t *) base + f->offset, f, depth ) ) + { + return false; + } + } + else if ( f->optional && got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + // absent, and back at its defaults: a repeated key whose last + // occurrence is null must not leave an earlier value standing + if ( f->table != NULL ) { f->table->reset( (uint8_t *) base + f->offset ); } + else { memset( (uint8_t *) base + f->offset, 0, (size_t) f->elem_size ); } + TableJsonSetRaw( (uint8_t *) base + f->present_offset, 1, 0 ); + } + else + { + if ( got != TableJsonShape( f ) ) + { + // the wrong JSON type for the kind: skipped, never coerced + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadField( in, base, f, depth ) ) + { + return false; + } + if ( f->optional ) + { + TableJsonSetRaw( (uint8_t *) base + f->present_offset, 1, 1 ); + } + } + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; return true; } + in.bad = true; + return false; + } +} + +// ---- the two entry points the per-table wrappers name ---- + +inline bool TableJsonRead( void * value, const TableTypeInfo * info, const char * text, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableJsonIn in; + in.text = text; + in.size = bytes; + in.pos = 0; + in.report = report != NULL ? report : &ignored; + in.bad = false; + in.graph = NULL; + info->reset( value ); + if ( text == NULL || bytes < 0 ) + { + in.report->malformed = true; + return false; + } + bool ok = TableJsonReadTable( in, value, info, 0 ); + if ( ok ) + { + TableJsonSpace( in ); + if ( in.pos != in.size ) { in.bad = true; } // trailing rubbish is not one text + } + if ( in.bad || !ok ) + { + in.report->malformed = true; + return false; + } + return true; +} + +inline int64_t TableJsonWrite( const void * value, const TableTypeInfo * info, char * buffer, int64_t capacity ) +{ + TableJsonOut out; + out.buffer = buffer; + out.capacity = capacity; + out.offset = 0; + out.overflow = false; + out.graph = NULL; + if ( !TableJsonWriteValue( out, value, info, 0 ) ) { return -1; } + // THE CANONICAL TEXT ENDS WITH EXACTLY ONE NEWLINE (docs/SPEC-TABLES.md + // §16.1). Every writer emits it — this walk, the C# walk and + // "schema unpack" — and every reader accepts a text with or without one, + // because the trailing whitespace a read already skips is what makes the + // two the same text. It is a byte of the FORM rather than a file + // convention: a text that is written to a file, pasted into a diff and + // handed back through a pipe has to be one text in all three places, and a + // buffer whose last byte is a closing brace is the one shape that is not. + out.put( '\n' ); + if ( out.overflow ) { return -1; } + return out.offset; +} + +// ---- json walk: end ---- + +// ---- json graph walk: begin ---- +// +// THE VARIABLE CLASS's half of the text form (docs/SPEC-TABLES.md §16.7). The +// walk above places every kind but one; this defines the three adapters it +// calls for that one, and the two entry points a pointered table's wrappers +// name. The text is the fixed class's — a pointee is an object in place — and a +// node named more than once carries `&node`: defined once, with its fields, +// and referenced after by `{ "&node": N }` alone. + +// ---- the identity map ---- +// +// ONE map shape serves both directions. Writing keys it by a node's ADDRESS and +// counts the slots that name the node, so the second pass knows at a node's +// first occurrence whether it will be named again; reading keys it by the +// text's own label and answers the node it defined. Open addressing, a +// multiply-shift hash and quadrupling growth — TablePackMap's shape (§6.2), on +// the same terms: proportional to nodes, never to bytes, on the authoring +// side, and released before the call returns. + +struct TableJsonGraphEntry +{ + uint64_t key; // a node's address (write) or a label (read); 0 is an empty slot + int64_t count; // write: how many slots name this node + int64_t label; // write: the `&node` label assigned at its first write, 0 until then + uint8_t open; // the descent is still open: a reference here is a cycle (write), a self-reference (read) + uint32_t node; // read: the node's arena offset; 0 for a definition the reader dropped + const TableTypeInfo * type; // read: the node's table; NULL for a dropped one +}; + +struct TableJsonGraphMap +{ + TableJsonGraphEntry * entries; + int64_t capacity; // a power of two, or zero while empty + int64_t count; + TableAllocator allocator; // the caller's pair (§6.5): the builder's on read, the one handed to ToJson on write +}; + +inline void TableJsonGraphMapInit( TableJsonGraphMap & map, TableAllocator allocator ) +{ + map.entries = NULL; + map.capacity = 0; + map.count = 0; + map.allocator = allocator; +} + +inline void TableJsonGraphMapShutdown( TableJsonGraphMap & map ) +{ + map.allocator.free( map.allocator.context, map.entries ); + TableJsonGraphMapInit( map, map.allocator ); +} + +inline int64_t TableJsonGraphMapSlot( const TableJsonGraphMap & map, uint64_t key ) +{ + uint64_t hash = key * 0x9E3779B97F4A7C15ull; + hash ^= hash >> 29; + int64_t mask = map.capacity - 1; + int64_t at = (int64_t) ( hash & (uint64_t) mask ); + while ( map.entries[at].key != 0 && map.entries[at].key != key ) + { + at = ( at + 1 ) & mask; + } + return at; +} + +inline TableJsonGraphEntry * TableJsonGraphMapFind( TableJsonGraphMap & map, uint64_t key ) +{ + if ( map.capacity == 0 ) { return NULL; } + TableJsonGraphEntry * entry = &map.entries[ TableJsonGraphMapSlot( map, key ) ]; + return entry->key == key ? entry : NULL; +} + +inline bool TableJsonGraphMapGrow( TableJsonGraphMap & map ) +{ + TableJsonGraphMap grown; + grown.allocator = map.allocator; + grown.capacity = map.capacity != 0 ? map.capacity * 4 : 64; + grown.count = 0; + grown.entries = (TableJsonGraphEntry *) map.allocator.alloc( map.allocator.context, grown.capacity * (int64_t) sizeof( TableJsonGraphEntry ) ); // zeroed, by the pair's contract + if ( grown.entries == NULL ) { return false; } + for ( int64_t i = 0; i < map.capacity; i++ ) + { + if ( map.entries[i].key == 0 ) { continue; } + grown.entries[ TableJsonGraphMapSlot( grown, map.entries[i].key ) ] = map.entries[i]; + grown.count++; + } + map.allocator.free( map.allocator.context, map.entries ); + map = grown; + return true; +} + +// the entry for a key, made if it was not there; `taken` says which. NULL is the +// allocator refusing, and the walk refuses with it. +inline TableJsonGraphEntry * TableJsonGraphMapReach( TableJsonGraphMap & map, uint64_t key, bool & taken ) +{ + if ( ( map.count + 1 ) * 4 >= map.capacity * 3 ) // keep the load factor under three quarters + { + if ( !TableJsonGraphMapGrow( map ) ) { return NULL; } + } + TableJsonGraphEntry * entry = &map.entries[ TableJsonGraphMapSlot( map, key ) ]; + taken = entry->key != key; + if ( taken ) + { + entry->key = key; + map.count++; + } + return entry; +} + +// ---- reading: into a builder ---- + +struct TableJsonGraphIn +{ + TableWorker * worker; // where every node comes from + TableJsonGraphMap labels; // a label -> the node it defined +}; + +// `&node`'s value, the LABEL: a positive integer spelled as one — digits, no sign, no +// fraction, no exponent, no leading zero (§16.7). Anything else is malformed. +inline bool TableJsonScanLabel( TableJsonIn & in, uint64_t & label ) +{ + TableJsonSpace( in ); + if ( in.pos >= in.size || in.text[in.pos] < '1' || in.text[in.pos] > '9' ) + { + in.report->malformed = true; + in.bad = true; + return false; + } + uint64_t value = 0; + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) + { + uint64_t digit = (uint64_t) ( in.text[in.pos] - '0' ); + if ( value > ( UINT64_MAX - digit ) / 10 ) + { + in.report->malformed = true; + in.bad = true; + return false; + } + value = value * 10 + digit; + in.pos++; + } + label = value; + return true; +} + +// A BYTE BUFFER's text (docs/SPEC-TABLES.md §2.5, §16.2): a string. For a +// *string the string's bytes become the blob; for a *bytes the string is base64 +// and its decoded bytes do. The blob is allocated at EXACTLY the decoded +// length — the string is scanned once without keeping it to learn the length, +// and once into the node — so a blob of any size reads with no window and no +// bound to clamp against. A *bytes body that is not base64 is the wrong shape +// for the kind: the reference stays null and the event is counted. +inline bool TableJsonReadBlob( TableJsonIn & in, void * slot, const TableFieldInfo * f ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + TableRef * ref = (TableRef *) slot; + ref->value = 0; + if ( strcmp( f->type_name, "string" ) == 0 ) + { + const int64_t mark = in.pos; + int32_t length = 0; + if ( !TableJsonScanString( in, NULL, 0, &length ) ) { return false; } + in.pos = mark; + char * data = TableStringEmplace( *graph->worker, *ref, NULL, (int64_t) length ); + if ( data == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + int32_t placed = 0; + return TableJsonScanString( in, data, length, &placed ); + } + // base64: the alphabet characters decide the length, six bits apiece + const char * alphabet = TableJsonBase64Alphabet(); + const int64_t mark = in.pos + 1; + int64_t symbols = 0; + bool malformed = false; + in.pos++; + for ( ;; ) + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos++]; + if ( c == '"' ) { break; } + if ( c == '=' || malformed ) { continue; } + if ( c == 0 || strchr( alphabet, c ) == NULL ) { malformed = true; continue; } + symbols++; + } + if ( malformed ) + { + in.report->kind_mismatch++; + return true; + } + const int64_t length = ( symbols * 6 ) / 8; + uint8_t * data = TableBytesEmplace( *graph->worker, *ref, length ); + if ( data == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + int64_t placed = 0; + uint32_t accumulator = 0; + int32_t held = 0; + for ( int64_t at = mark; ; at++ ) + { + char c = in.text[at]; + if ( c == '"' ) { break; } + const char * symbol = c != '=' ? strchr( alphabet, c ) : NULL; + if ( symbol == NULL ) { continue; } + accumulator = ( accumulator << 6 ) | (uint32_t) ( symbol - alphabet ); + held += 6; + if ( held >= 8 ) + { + held -= 8; + if ( placed < length ) { data[placed++] = (uint8_t) ( ( accumulator >> held ) & 0xff ); } + } + } + return true; +} + +// A pointer's object. Its FIRST key decides what it is: `&node` naming a label not +// yet defined, with fields after it, is a DEFINITION; `&node` naming one already +// defined, alone, is a REFERENCE; any other key is a node named once, its +// object in place. The node comes from the +// builder's arena, and the slot holds its arena offset (§6.3). A pointer whose +// target is a BYTE BUFFER — no table — takes a string instead (§2.5). +inline bool TableJsonReadPointer( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( f->table == NULL ) { return TableJsonReadBlob( in, slot, f ); } + // the pointee nests one level down, exactly as a by-value table does, and + // takes the same cap: a chain nests as deep as it is long (§16.7) + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + char c = TableJsonPeek( in ); + if ( c == '}' ) + { + // an empty object: a node at its defaults, named once + in.pos++; + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + return true; + } + if ( c == 0 ) { in.bad = true; return false; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + if ( strcmp( key, "&node" ) != 0 ) + { + // a node named once: the pointee's object in place, and this key is + // its first field — unless it is the reserved prefix under a spelling + // this form does not have, which ReadTableKeys refuses + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } + return TableJsonReadTableKeys( in, node, f->table, depth + 1, key ); + } + uint64_t label = 0; + if ( !TableJsonScanLabel( in, label ) ) { return false; } + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph->labels, label, taken ); + if ( entry == NULL ) { in.report->malformed = true; in.bad = true; return false; } + // ONE SPELLING, and what follows the label says which half it is: fields + // after a label the text has not defined DEFINE it, and a label alone that + // the text has defined REFERS to it. The other two are malformed — a label + // alone that the text never defined, which would otherwise read as a default + // node under a silent report, and a field after a label already defined, + // which would be a second definition. That is what keeps a typo loud. + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; c = TableJsonPeek( in ); } + bool bare = c == '}'; + if ( bare == taken ) { in.report->malformed = true; in.bad = true; return false; } + if ( bare ) + { + // A REFERENCE. A label is defined when its object CLOSES, so a + // reference met inside its own definition — at any depth of by-value + // nesting — names a node whose descent is still open: the cycle the + // wire refuses (§3.1), refused here where it is written. A definition + // the reader dropped names no node, so the slot stays null with + // nothing more counted — the drop was counted where it happened. A + // node of another table than the slot declares is a kind mismatch, as + // on the wire. + in.pos++; + if ( entry->open != 0 ) { in.report->malformed = true; in.bad = true; return false; } + TableRef ref; + if ( entry->type == NULL ) + { + memcpy( slot, &ref, sizeof( ref ) ); + return true; + } + if ( entry->type != f->table ) + { + memcpy( slot, &ref, sizeof( ref ) ); + in.report->kind_mismatch++; + return true; + } + ref.value = (int64_t) entry->node; + memcpy( slot, &ref, sizeof( ref ) ); + return true; + } + // A DEFINITION: the node is allocated, the label is its, and the keys after + // `&node` are its fields. The entry is OPEN until the object closes, so a + // reference to the label from inside the node's own fields is refused as + // the cycle it is; the node and its table are filled in at the close. + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } + entry->open = 1; + if ( !TableJsonReadTableKeys( in, node, f->table, depth + 1, NULL ) ) { return false; } + entry = TableJsonGraphMapFind( graph->labels, label ); // the map may have grown under the descent + if ( entry == NULL ) { in.report->malformed = true; in.bad = true; return false; } + TableRef ref; + memcpy( &ref, slot, sizeof( ref ) ); + entry->node = (uint32_t) ref.value; + entry->type = f->table; + entry->open = 0; + return true; +} + +// An `&`-prefixed key opening an object the walk is SKIPPING — a value past an +// array's bound, an unknown key's value, a value of the wrong shape. A +// definition in there still takes its label, so the numbering survives whatever +// the storage could not hold (§16.7): the label is registered with no node, and a +// reference to it reads null. Any other prefixed key is the reserved prefix +// out of place. +inline bool TableJsonSkippedAmpersand( TableJsonIn & in, const char * key, int32_t ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL || strcmp( key, "&node" ) != 0 ) { in.report->malformed = true; in.bad = true; return false; } + uint64_t label = 0; + if ( !TableJsonScanLabel( in, label ) ) { return false; } + bool taken = false; + if ( TableJsonGraphMapReach( graph->labels, label, taken ) == NULL ) { in.report->malformed = true; in.bad = true; return false; } + return true; // a fresh entry is node 0, type NULL: a definition with no node +} + +// ---- writing: from a region's const root ---- + +struct TableJsonGraphOut +{ + TableJsonGraphMap nodes; // a node's address -> how many slots name it, and its `&node` once assigned + bool counting; // PASS ONE: count the references, refuse a cycle, emit nothing + int64_t next_label; +}; + +// The node a slot names: null as `null`, a node named once as its object in +// place, and a node named more than once under the construct. Which of the +// last two it is was learned in pass one; pass two spells it. +inline bool TableJsonWritePointer( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphOut * graph = (TableJsonGraphOut *) out.graph; + if ( graph == NULL ) { return false; } + const void * node = f->resolve( slot ); + if ( node == NULL ) + { + out.raw( "null", 4 ); + return true; + } + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph->nodes, (uint64_t) (uintptr_t) node, taken ); + if ( entry == NULL ) { return false; } + if ( f->table == NULL ) + { + // A BYTE BUFFER (§2.5, §16.7): its text is a string, which has no + // first key to carry `&node`, so a blob named from more than one + // slot has no spelling this form can carry and the graph is refused — + // as a shared node with nothing to write is. A blob named once is its + // bytes in place: base64 for a *bytes, the string itself for a *string. + if ( graph->counting ) { entry->count++; return true; } + if ( entry->count > 1 ) { return false; } + const TableBlob * blob = (const TableBlob *) node; + if ( blob->length > (uint32_t) 0x7fffffff ) { return false; } + if ( strcmp( f->type_name, "string" ) == 0 ) { TableJsonWriteString( out, (const char *) ( blob + 1 ), (int32_t) blob->length ); } + else { TableJsonWriteBase64( out, (const uint8_t *) ( blob + 1 ), (int32_t) blob->length ); } + return true; + } + if ( graph->counting ) + { + // PASS ONE: one visit per node, every slot that names it counted, and + // a reference to a node whose descent is still open is a cycle — + // refused here as the wire refuses it (§3.1) + entry->count++; + if ( !taken ) { return entry->open == 0; } + entry->open = 1; + if ( !TableJsonWriteValue( out, node, f->table, depth ) ) { return false; } + entry = TableJsonGraphMapFind( graph->nodes, (uint64_t) (uintptr_t) node ); // the map may have grown under the descent + if ( entry == NULL ) { return false; } + entry->open = 0; + return true; + } + // PASS TWO: a node named once is its object in place; a node named more + // than once is DEFINED at its first occurrence — `&node` first, then its + // fields — and REFERENCED by `&node` alone after that, spelled the same way at + // every site. Labels run from 1 in first-write order and are the text's own, + // so a stray number in a hand-edited text is most often one never defined. + if ( entry->count <= 1 ) + { + return TableJsonWriteValue( out, node, f->table, depth ); + } + if ( depth > kTableJsonMaxDepth ) { return false; } + if ( entry->label != 0 ) + { + out.put( '{' ); + out.line( depth + 1 ); + out.raw( "\"&node\": ", 9 ); + TableJsonWriteUnsigned( out, (uint64_t) entry->label ); + out.line( depth ); + out.put( '}' ); + return true; + } + entry->label = ++graph->next_label; + out.put( '{' ); + out.line( depth + 1 ); + out.raw( "\"&node\": ", 9 ); + TableJsonWriteUnsigned( out, (uint64_t) entry->label ); + bool any = true; + int64_t before = out.offset; + if ( !TableJsonWriteFields( out, node, f->table, depth, any ) ) { return false; } + // a definition carries at least one field, because a label alone is a + // reference: a shared node with nothing to write has no definition this + // form can spell, and the writer refuses it as it refuses any value it + // cannot spell (§16.3) + if ( out.offset == before ) { return false; } + out.line( depth ); + out.put( '}' ); + return true; +} + +// ---- the two entry points a pointered table's wrappers name ---- + +// The text into the builder's root. Every node the text names is allocated in +// the builder's arena through the field's own Emplace; the label map is the +// walk's, released before this returns. The root itself takes no label — nothing +// may name it (§16.7) — so an `&node` at the root is refused like any other key +// of the prefix. +inline bool TableJsonReadGraph( TableWorker & worker, void * root, const TableTypeInfo * info, const char * text, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + if ( worker.arena == NULL ) { if ( report != NULL ) { report->malformed = true; } return false; } + TableJsonGraphIn graph; + graph.worker = &worker; + TableJsonGraphMapInit( graph.labels, worker.arena->allocator ); + TableJsonIn in; + in.text = text; + in.size = bytes; + in.pos = 0; + in.report = report != NULL ? report : &ignored; + in.bad = false; + in.graph = &graph; + info->reset( root ); + if ( text == NULL || bytes < 0 ) + { + in.report->malformed = true; + return false; + } + bool ok = TableJsonReadTable( in, root, info, 0 ); + if ( ok ) + { + TableJsonSpace( in ); + if ( in.pos != in.size ) { in.bad = true; } // trailing rubbish is not one text + } + TableJsonGraphMapShutdown( graph.labels ); + if ( in.bad || !ok ) + { + in.report->malformed = true; + return false; + } + return true; +} + +// The text of a region's const root: measured when the buffer is NULL, written +// when it is not, over one code path. Two passes over one walk — the first +// counts how many slots name each node and refuses a cycle, the second writes +// — so a node's first occurrence knows whether it will be named again. The +// ROOT's entry is open for the whole first pass, so a reference back at it is +// the cycle it is (§3.1), and it takes no label. +inline int64_t TableJsonWriteGraph( const void * root, const TableTypeInfo * info, char * buffer, int64_t capacity, TableAllocator allocator ) +{ + if ( root == NULL ) { return -1; } + TableJsonGraphOut graph; + TableJsonGraphMapInit( graph.nodes, allocator ); + graph.counting = true; + graph.next_label = 0; + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph.nodes, (uint64_t) (uintptr_t) root, taken ); + if ( entry == NULL ) { TableJsonGraphMapShutdown( graph.nodes ); return -1; } + entry->open = 1; + TableJsonOut count; + count.buffer = NULL; + count.capacity = 0; + count.offset = 0; + count.overflow = false; + count.graph = &graph; + bool ok = TableJsonWriteValue( count, root, info, 0 ); + graph.counting = false; + TableJsonOut out; + out.buffer = buffer; + out.capacity = capacity; + out.offset = 0; + out.overflow = false; + out.graph = &graph; + if ( ok ) { ok = TableJsonWriteValue( out, root, info, 0 ); } + TableJsonGraphMapShutdown( graph.nodes ); + if ( !ok ) { return -1; } + out.put( '\n' ); // the canonical text ends with exactly one newline (§16.1) + if ( out.overflow ) { return -1; } + return out.offset; +} + +// ---- json graph walk: end ---- + +// ---- the out-of-line array's slot (docs/SPEC-TABLES.md §8.1) ---- + +inline int32_t TableJsonExtentCount( const void * slot ) +{ + int32_t count = 0; + memcpy( &count, (const uint8_t *) slot + 8, sizeof( count ) ); + return count < 0 ? 0 : count; +} + +inline const uint8_t * TableJsonExtentElements( const void * slot ) +{ + int64_t delta = 0; + memcpy( &delta, slot, sizeof( delta ) ); + return delta != 0 ? (const uint8_t *) slot + delta : NULL; +} + +// ---- json map walk: begin ---- + +inline bool TableJsonIsMap( const TableFieldInfo * f ) +{ + return f->is_array && f->array_bound == 0 && strncmp( f->type_name, "map[", 4 ) == 0; +} + +// the entry's two rows: fields[0] IS the key and fields[1] IS the value, which +// is what makes a user's own table of pairs the same bytes (§2.8) +inline const TableFieldInfo * TableJsonMapKeyField( const TableFieldInfo * f ) { return &f->table->fields[0]; } +inline const TableFieldInfo * TableJsonMapValueField( const TableFieldInfo * f ) { return &f->table->fields[1]; } + +inline bool TableJsonMapKeyIsString( const TableFieldInfo * key ) { return key->kind == 12; } +inline bool TableJsonMapKeySigned( const TableFieldInfo * key ) { return key->kind >= 2 && key->kind <= 5; } + +// AN INTEGER KEY IS THE INTEGER'S DECIMAL SPELLING, QUOTED, because a JSON +// object's keys are strings. Written digit by digit so no locale can move it. +inline void TableJsonWriteMapIntegerKey( TableJsonOut & out, const void * storage, const TableFieldInfo * key ) +{ + uint64_t magnitude = 0; + bool negative = false; + if ( TableJsonMapKeySigned( key ) ) + { + int64_t value = 0; + switch ( key->kind ) + { + case 2: value = (int64_t) *(const int8_t *) storage; break; + case 3: value = (int64_t) *(const int16_t *) storage; break; + case 4: value = (int64_t) *(const int32_t *) storage; break; + default: value = *(const int64_t *) storage; break; + } + negative = value < 0; + magnitude = negative ? ( ~(uint64_t) value ) + 1 : (uint64_t) value; + } + else + { + switch ( key->kind ) + { + case 6: magnitude = (uint64_t) *(const uint8_t *) storage; break; + case 7: magnitude = (uint64_t) *(const uint16_t *) storage; break; + case 8: magnitude = (uint64_t) *(const uint32_t *) storage; break; + default: magnitude = *(const uint64_t *) storage; break; + } + } + char digits[24]; + int32_t at = (int32_t) sizeof( digits ); + do { digits[--at] = (char) ( '0' + ( magnitude % 10 ) ); magnitude /= 10; } while ( magnitude != 0 ); + if ( negative ) { digits[--at] = '-'; } + TableJsonWriteString( out, digits + at, (int32_t) sizeof( digits ) - at ); +} + +inline void TableJsonWriteMapKey( TableJsonOut & out, const void * entry, const TableFieldInfo * key ) +{ + const uint8_t * storage = (const uint8_t *) entry + key->offset; + if ( TableJsonMapKeyIsString( key ) ) + { + // A STRING KEY IS THE STRING (§2.8): every JSON key of a map object is + // a KEY OF THE MAP and none is a field key, so the `&` prefix §16.7 + // reserves for field keys is ordinary data here. + TableJsonWriteString( out, (const char *) storage, *(const int32_t *) ( (const uint8_t *) entry + key->count_offset ) ); + return; + } + TableJsonWriteMapIntegerKey( out, (const void *) storage, key ); +} + +// ToJson WRITES ENTRIES IN ASCENDING KEY ORDER, so unpack then pack is +// byte-stable and a diff of two texts is a diff of two maps (§2.8, §17.2). +// A region holds them in that order already, so this is the array in place. +inline bool TableJsonWriteMap( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + const int32_t count = TableJsonExtentCount( slot ); + if ( count == 0 ) { out.raw( "{}", 2 ); return true; } + const TableFieldInfo * key = TableJsonMapKeyField( f ); + const TableFieldInfo * value = TableJsonMapValueField( f ); + const uint8_t * entries = TableJsonExtentElements( slot ); + out.put( '{' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + const void * entry = (const void *) ( entries + (int64_t) i * f->elem_size ); + TableJsonWriteMapKey( out, entry, key ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteField( out, entry, value, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( '}' ); + return true; +} + +// AN INTEGER KEY IS READ BY §16.2's INTEGER RULE AND BY NOTHING ELSE, so +// "2.0" and "1e3" are the integers 2 and 1000 and "-0" is zero. The token is +// walked as a JSON number over its own bytes; a token that rule calls +// malformed makes the KEY malformed, and a genuinely fractional value, or one +// outside the key kind's range, is kind_mismatch for that entry. +// +// THE KEY IS THE SPELLING AND NOTHING AROUND IT. The number walk steps over +// leading whitespace and comments, which is right BETWEEN tokens and wrong +// INSIDE one: a key is an identity, and a padded spelling that resolved to the +// same integer would be a second name for one entry. So the walk must begin at +// the token's first byte, and a token with anything before the number is not a +// JSON number at all, which is malformed on the terms "1-2" is. +inline bool TableJsonMapKeyValue( const char * token, int32_t length, const TableFieldInfo * key, + int64_t & value, bool & fits ) +{ + fits = false; + TableReport scratch; + TableJsonIn probe = { token, (int64_t) length, 0, &scratch, false, NULL }; + bool integral = false; + TableJsonSpace( probe ); + if ( probe.pos != 0 ) { return false; } // whitespace is never part of a key + if ( !TableJsonWalkNumber( probe, &integral ) ) { return false; } + if ( probe.pos != (int64_t) length ) { return false; } // trailing bytes: not a number + // A MAP KEY'S POLICY over the one interpreted value: it REJECTS THE WHOLE + // ENTRY. A key is an identity, so a clamped one is two entries merged, and + // a value the key kind does not hold is kind_mismatch for that entry, + // dropped and counted, never clamped. + const TableJsonInteger number = TableJsonInterpretExact( token, length ); + if ( number.fractional || number.saturated ) { return true; } + bool moved = false; + value = TableJsonIntegerInDomain( number, TableJsonMapKeySigned( key ), (int32_t) key->elem_size, moved ); + fits = !moved; + return true; +} + +// FromJson READS KEYS IN WHATEVER ORDER THE TEXT GIVES THEM. A repeated key is +// last-wins and counted duplicate, the object rule (§16.2) applied inside the +// map. An empty object is an empty map, and null is kind_mismatch. +inline bool TableJsonReadMap( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; + const TableFieldInfo * key = TableJsonMapKeyField( f ); + const TableFieldInfo * value = TableJsonMapValueField( f ); + const char shape = TableJsonShape( value ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + char token[kTableJsonMaxKey]; + int32_t token_length = 0; + bool key_over = false; // longer than THIS buffer: never truncated into a key + if ( !TableJsonScanString( in, token, kTableJsonMaxKey - 1, &token_length, &key_over ) ) { return false; } + token[token_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t key_value = 0; + bool place = true; + if ( !TableJsonMapKeyIsString( key ) ) + { + // AN INTEGER KEY PAST THIS SCAN'S BUFFER DROPS AS kind_mismatch, + // here and in the tool's walker. The bytes kept are a PREFIX, and a + // prefix is a different token, so the entry drops rather than a + // truncation being read as a value. The length alone does not + // settle it: a token that long can still spell a number an integer + // kind holds, "1" padded by an exponent of zeroes for one, and this + // read declines to find out. + bool fits = false; + if ( key_over ) { in.report->kind_mismatch++; place = false; } + else if ( !TableJsonMapKeyValue( token, token_length, key, key_value, fits ) ) + { + // A MALFORMED KEY STOPS THE READ where §16.1's rule stops it, + // with the instance holding what was placed before the stop. + in.report->malformed = true; + in.bad = true; + return false; + } + else if ( !fits ) { in.report->kind_mismatch++; place = false; } + } + else if ( key_over || token_length > key->array_bound ) + { + // A KEY LONGER THAN N DROPS ITS ENTRY AND COUNTS clamped, the + // wire's rule, because a clamped key is a merged entry (§2.8). The + // BOUND IS THE WALKER'S, tested here against the key field's own + // descriptor, so placement is left with one failure to report. A + // key past this scan's own buffer is the SAME event, because a + // truncated key is the merged entry the rule exists to prevent. + in.report->clamped++; + place = false; + } + const int32_t before = TableJsonExtentCount( (const void *) slot ); + void * entry = place ? f->place( *graph->worker, slot, token, token_length, key_value ) : NULL; + if ( place && entry == NULL ) + { + // AN ALLOCATION FAILURE IS NOT AN OVERSIZED KEY (§2.8, §16.1). The + // key was checked above, so the arena is what refused, and the read + // stops where the list, blob and pointer paths stop on one rather + // than handing back an instance short of entries the text spelled + // and calling itself clean. + in.report->malformed = true; + in.bad = true; + return false; + } + else if ( entry != NULL && TableJsonExtentCount( (const void *) slot ) == before ) + { + in.report->duplicate++; // last-wins, the object rule inside the map + } + const char got = TableJsonValueShape( in ); + if ( entry == NULL ) + { + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( value->kind == 17 && !value->is_array ) + { + // A POINTER VALUE IS SHARED EXACTLY AS A POINTER FIELD IS (§2.8): + // null is a null slot, an object is the pointee in place or an + // &node reference to one (§16.7), anything else is the wrong shape — + // the same three the field-key loop gives a pointer field, because + // an entry's value IS a field line. + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) entry + value->offset, value->elem_size, 0 ); + } + else if ( got != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, (uint8_t *) entry + value->offset, value, depth + 1 ) ) + { + return false; + } + } + else if ( got != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadField( in, entry, value, depth + 1 ) ) + { + return false; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; +} + +// ---- json map walk: end ---- + +// ---- json list walk: begin ---- + +// an unbounded array is the out-of-line array that is not a map (§8.1) +inline bool TableJsonIsList( const TableFieldInfo * f ) +{ + return f->is_array && f->array_bound == 0 && !TableJsonIsMap( f ); +} + +// ToJson WRITES THE ELEMENTS IN INDEX ORDER, which is the only order there is, +// so unpack then pack is byte-stable without a rule of its own (§2.9, §17.2). +// A region holds the array in place, so this steps it at the descriptor's pitch. +inline bool TableJsonWriteList( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + const int32_t count = TableJsonExtentCount( slot ); + if ( count == 0 ) { out.raw( "[]", 2 ); return true; } + const uint8_t * elements = TableJsonExtentElements( slot ); + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + const uint8_t * element = elements + (int64_t) i * f->elem_size; + if ( f->kind == 17 ) + { + // a []*T's elements take the pointer row (§16.7): the pointee's + // object in place, null, or `&node` for a shared one + if ( !TableJsonWritePointer( out, element, f, depth + 1 ) ) { return false; } + } + else if ( !TableJsonWriteScalar( out, element, f, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( ']' ); + return true; +} + +// FromJson READS EVERY ELEMENT THE TEXT CARRIES, appending each through the +// descriptor's place resolver: `[]` is an empty list, and null is +// kind_mismatch, the array row's own rule (§16.2). LAST WINS holds for a +// repeated key: the list goes back to EMPTY before this occurrence's elements +// land, the builder's storage being reclaimed at reset (§2.9). +inline bool TableJsonReadList( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; + TableJsonSetRaw( (uint8_t *) slot, 8, 0 ); + TableJsonSetRaw( (uint8_t *) slot + 8, 4, 0 ); + const char shape = TableJsonElementShape( f ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + void * element = f->place( *graph->worker, slot, NULL, 0, 0 ); + if ( element == NULL ) + { + // NOT ADDED: the arena could not carve another segment, or the + // count met the int32 cap. The text cannot be placed whole, and + // the read stops where §16.1's rule stops it. + in.report->malformed = true; + in.bad = true; + return false; + } + if ( f->kind == 17 ) + { + // an element of a []*T (§2.9): null is a null slot, an object is the + // pointee in place or an `&node` reference (§16.7) + char got = TableJsonValueShape( in ); + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) element, f->elem_size, 0 ); + } + else if ( got != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, element, f, depth + 1 ) ) { return false; } + } + else if ( TableJsonValueShape( in ) != shape ) + { + // the wrong shape for the element kind: the slot keeps its + // defaults and the event counts, the array row's rule (§16.2) + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadScalar( in, element, f, depth + 1 ) ) { return false; } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; +} + +// ---- json list walk: end ---- + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_JSON + +namespace mapdemo { + +bool CrewsFromJson( CrewsBuilder & builder, const char * text, int64_t bytes, TableReport * report ) +{ + Crews * root = builder.GetRoot(); + if ( root == NULL ) { if ( report != NULL ) { report->malformed = true; } return false; } // locked, or the root allocation failed + return TableJsonReadGraph( builder.main, root, CrewsTableType(), text, bytes, report ); +} + +int64_t CrewsToJsonMeasure( const Crews * root, TableAllocator allocator ) +{ + return TableJsonWriteGraph( root, CrewsTableType(), NULL, 0, allocator ); +} + +int64_t CrewsToJson( const Crews * root, char * buffer, int64_t capacity, TableAllocator allocator ) +{ + return TableJsonWriteGraph( root, CrewsTableType(), buffer, capacity, allocator ); +} + +} // namespace mapdemo diff --git a/testdata/golden/tables/maps/CrewsTable.h b/testdata/golden/tables/maps/CrewsTable.h new file mode 100644 index 000000000..0acb2ac69 --- /dev/null +++ b/testdata/golden/tables/maps/CrewsTable.h @@ -0,0 +1,9466 @@ +// Code generated by the schema compiler from Crews.schema. DO NOT EDIT. +// SPDX-License-Identifier: NONE — this generated output is yours, under terms of +// your choice. See the LICENSE exception in the schema compiler; the compiler is +// AGPL-3.0, its output is not. +// package mapdemo — protocol id 0x1ac124decde5b2aa (packets only: tables version by field id, not by protocol id) +// The TABLE wire (evolution-tolerant, docs/SPEC-TABLES.md): no serialize +// dependency — includable from any TU. + +#pragma once + +#include +#include // the prefill's scalar-array fills +#include // offsetof, for the reflection descriptors + +// ---- the hooks (docs/USAGE.md, "the C++ table runtime's hooks") ---- +// +// schema_assert — the runtime's own assert, and the refusal a debugger reads. +// NDEBUG removes it, exactly as it removes assert. A caller who already routes +// serialize's asserts writes `#define schema_assert serialize_assert` before +// including this header and both halves land in one handler. +#ifndef schema_assert +#include +#define schema_assert assert +#endif // #ifndef schema_assert + +// schema_fatal — what stands after the assert on a path that cannot continue. +// NDEBUG does not remove it. Supply it and is never included. +#ifndef schema_fatal +#include // abort +#define schema_fatal abort +#endif // #ifndef schema_fatal + +// schema_allocate / schema_release — what "no allocator handed in" means for +// this program. schema_allocate hands back ZEROED bytes and NULL on failure: +// an arena segment is copied whole, padding included, so anything left +// uninitialized here would reach a packed region. Supply both and +// is never included; hand a TableAllocator to a builder to route one +// structure's allocations somewhere else again. +#ifndef schema_allocate +#include // calloc, free +#define schema_allocate( bytes ) calloc( (size_t) 1, (size_t) ( bytes ) ) +#define schema_release( pointer ) free( pointer ) +#endif // #ifndef schema_allocate +#include // a node's lifetime starts in arena storage (placement new) +#include // one atomic per slab: the arena is lock-free by ownership + +#include "Crews.h" +#include "FleetTable.h" + +#ifndef MAPDEMO_SCHEMA_TABLE_PRIMITIVES +#define MAPDEMO_SCHEMA_TABLE_PRIMITIVES + +// THE CODEC DOES NOT DEPEND ON THE COMPILER'S INLINING BUDGET. A table of a +// realistic field count emits one large body per type, and the cursor a body +// writes through lives in the caller's `TableWriter`: across a call boundary +// that cursor round-trips through memory, and a `uint8_t *` store may alias the +// writer itself, so every put reloads it. When a budget runs out mid-body the +// codec silently degrades to that shape. Forcing the primitives and the +// fixed-class bodies inline is what keeps the cursor in registers and lets +// adjacent constant framing bytes merge into one store. +#if defined( _MSC_VER ) +#define MAPDEMO_TABLE_INLINE __forceinline +#elif defined( __GNUC__ ) || defined( __clang__ ) +#define MAPDEMO_TABLE_INLINE inline __attribute__(( always_inline )) +#else +#define MAPDEMO_TABLE_INLINE inline +#endif + +namespace mapdemo { + +// WHY A READ WAS REFUSED, by name (docs/SPEC-TABLES.md §3.3, §11). A REFUSAL +// is not one of §4's events: nothing is decoded, no counter moves and no +// damage is reported, so five zero counters and a false flag are what a clean +// read prints too and only the verdict tells them apart. The reason says which +// refusal it was. +// +// This is the MESSAGE PATH's vocabulary and not the cooked form's (§7.4): a +// caller meeting one of these has been refused a MESSAGE on a connection, +// which is a different recovery with a different owner than a file a header +// match turned down. +enum TableMessageReason +{ + newer_form, // a FORM BYTE this reader does not carry (§3) + no_vocabulary, // no table for this connection: the message arrived before the announcement, or after a refused one + second_announcement, // a second announcement on a connection: it sets nothing, amends nothing, and the connection closes + vocabulary_too_large, // an announcement above the receiver's declared bound, refused before an entry is touched + message_form_as_file, // a form 2 wire where a FILE was expected: its table is somewhere else + batch_too_large // a batch of more than 256 bodies on the write side, or of more than the caller has room for on the read side: nothing is written or decoded, and the count says what the wire carries +}; + +// The table-wire read report — the permissive contract's ledger. Silence +// (all zero) means the data matched this reader's schema exactly. +struct TableReport +{ + int32_t unknown = 0; // unknown field ids skipped (newer data) + int32_t kind_mismatch = 0; // known id, changed type — skipped, never misdecoded + // a kind that GREW since the writer (docs/SPEC-TABLES.md §4): an integer + // kind read into a wider one of the same signedness, or f32 into f64, + // decoded EXACTLY. One count per field or per map. It is the one counter + // that names no loss: the bytes were not the shape this reader declares, + // and the number survived. + int32_t widened = 0; + int32_t clamped = 0; // out-of-range values clamped to declared bounds + // a key the TEXT form saw twice: last wins, and the repeat is counted + // (docs/SPEC-TABLES.md §16.2). The wire never raises it — a body carrying an + // id twice is legal input whose last occurrence wins, silently (§3). + int32_t duplicate = 0; + bool malformed = false; // framing damage; decode stopped, partial result kept + // THE REFUSAL VERDICT, which is not one of §4's events and moves no counter + // (docs/SPEC-TABLES.md §3): a FORM BYTE this reader does not carry. Five + // zero counters and a false flag are what a clean read prints too, so the + // verdict is what tells the two apart. + bool refused = false; + // WHICH refusal, and it is read only when refused is set: a read that + // was not refused has no reason, and this member is the one the caller + // must not look at then (docs/SPEC-TABLES.md §3.3). + TableMessageReason reason = newer_form; + // RETAIN-UNKNOWN's pair (docs/SPEC-TABLES.md §6.6), on the same struct for + // the reason duplicate is: a caller has one report type and not two. Both + // are ZERO in every read that did not opt in, and retention moves no + // counter above. A retained field still counts unknown, because unknown + // says what a READER could not name and that stays true. + int32_t retained = 0; // unknown fields whose bytes were kept + int32_t retain_lost = 0; // every unknown this load or save could not keep +}; + + +// WHY A FILE WAS REFUSED, by name (docs/SPEC-TABLES.md §6.5, §7, §19.2): the +// one vocabulary a cook's Open, a block's BlockOpen and a load measure's -1 +// share, because a caller asking "why can I not have this file" is +// asking one question whichever call refused it. The FIRST failing clause names the +// reason, in the order §7 enumerates, so one file answers one value in every +// language. A refusal moves no counter, and a match writes nothing: the +// out-parameter is touched on the refusal path only. +// +// It is not the MESSAGE FORM's vocabulary (TableMessageReason, §3.3): a caller +// meeting one of these has been refused a FILE, by a header match or by a +// measure. +enum TableRefuseReason +{ + ok, // no clause failed: the only value beside a non-null root (§7) + not_a_cook, // the magic is neither this build's constant nor its byte reversal, or the byte-order word contradicts the magic + foreign_order, // the magic byte-reversed: a cook of the other byte order (§7.1) + wrong_build_version, // the build_version word is not this build's (§20) + reserved_not_zero, // a reserved header word is not zero (§7.1) + bad_alignment, // the alignment word is not a power of two, is below eight, is above sixty-four, or is not a multiple of the root's own alignof + truncated, // the part lengths against the caller's length, or a data part too short to hold the root + unaligned_base, // the pointer the caller passed is not aligned for the region: the caller's defect, not the file's + bad_layout, // BlockOpen (§19.2): a pitch, a count, an offset or an extent that disagrees with this build's or leaves the block + unknown_form, // at a MEASURE (§3, §6.5): a form byte this build does not carry, refused before any read + count_over_length, // an array or map count whose elements cannot fit the field's own L (§2.8, §2.9) + count_over_extent_cap, // a count above the int32 extent cap (§2.2), which no region can hold whatever its size + blob_over_size_cap, // a blob whose length is past the derived-size cap (§3.1, §11) + data_cycle // a data cycle reached from a builder: the AUTHORING side's -1 (§3.1, §7.6) +}; +// ---- reflection (tables only, docs/SPEC-TABLES.md) ---- +// +// Static field descriptors for every type in the table closure: name, wire +// id/kind, storage offset, bounds, ranges, enum names and branch guards — +// enough to walk, print, diff, edit or bind any table value at runtime with +// no RTTI and no schema files. TableType() returns X's descriptor. + +struct TableTypeInfo; + +// One arm of a union field: where its payload sits inside the union's storage +// and what its payload looks like. The arm's NAME and its table-wire id come +// from the field's enum_name/variant_id functions at the same tag, so nothing +// is spelled twice (docs/SPEC-TABLES.md §8). +struct TableFieldInfo; + +struct TableUnionArmInfo +{ + uint32_t offset; // offsetof the arm's payload within the union storage + const TableTypeInfo * table; // the arm payload's descriptor, or NULL + // AN ARM IS A FIELD LINE (docs/SPEC-TABLES.md §2.6): an arm that names no + // declared type or table carries the FIELD descriptor a field of that + // type would carry instead — offsets taken within the union storage — so + // a generic walk meets an arm's kind, width, bounds and companions where + // it meets a field's. Exactly one of the two is non-NULL on a set arm. + const TableFieldInfo * field; + uint32_t size; // the arm's whole storage, which selection zero-establishes +}; + +// A union field's shape: the tag, and the arms indexed by it. Arms run +// [0, enum_max]; index 0 is the EMPTY arm and carries no payload. +struct TableUnionInfo +{ + uint32_t tag_offset; // offsetof the tag within the union storage + uint32_t tag_size; // sizeof the tag + const TableUnionArmInfo * arms; +}; + +// The exact raw range of a wide-kind field (docs/SPEC-TABLES.md §8.2): two 128-bit +// values as 64-bit lanes, low lane first, two's complement for the signed kinds. +struct TableWideRange +{ + uint64_t lo[2]; + uint64_t hi[2]; +}; + +// THE SHARED EMPTY DOC (docs/SPEC-TABLES.md §8.1): a declaration with no /// +// block carries a doc column pointing at this one object, so absence costs a +// unit no string data and a printer concatenates doc columns with no null +// test. One definition for the whole unit: every absent doc compares equal by +// address. +inline const char TableDocNone[1] = ""; + +// the arena's allocation front, defined with the variable-length runtime +// below; a descriptor names it only through a pointer parameter. +struct TableWorker; + +struct TableFieldInfo +{ + const char * name; // schema field name, e.g. "health" + const char * json; // the TEXT form's key: the json = "key" attribute, else name (§16.3) + const char * type_name; // schema type name, e.g. "float32", "Grade" + uint64_t id; // table-wire field id: fnv1a64 of the name, of the was alias after a rename (§5) + uint8_t kind; // table-wire kind; for arrays/strings/bytes, the ELEMENT kind + bool is_array; // fixed or counted array (bytes included) + bool is_pointer; // a *T pointer field: storage is an 8-byte TableRef; the target is a table + // THE TWO THE TEXT FORM NEEDS (docs/SPEC-TABLES.md §16.7), and they + // are here for the same reason is_pointer is: the walk is ONE walk + // over descriptors and cannot spell a target's own At or + // Emplace. `resolve` reads a slot in a REGION and answers the + // node it names, or NULL; `emplace` allocates one in a BUILDER's + // arena and points the slot at it. NULL on every field that is not + // a pointer, and emitted only in a unit that declares one. + const void * (*resolve)( const void * slot ); + void * (*emplace)( TableWorker & worker, void * slot ); + bool counted; // a _count/_length int32 companion exists (counted arrays, strings, bytes) + bool optional; // a ?T field: a _present bool companion decides whether it rides + int32_t array_bound; // array capacity / string max length; 0 for plain scalars + uint32_t offset; // offsetof the storage member + uint32_t elem_size; // sizeof the member (element size for arrays) + uint32_t count_offset; // offsetof the _count/_length companion, or 0xffffffff + uint32_t present_offset; // offsetof the _present companion, or 0xffffffff + const TableTypeInfo * table; // nested table's descriptor, or NULL + bool has_range; // a declared [min, max] (int or float) + double range_min; // NOTE: int64 ranges beyond 2^53 lose precision here + double range_max; + // the WIDE kinds (18-29, docs/SPEC-TABLES.md §3, §8.2): frac_bits is a fixed + // field's F — its storage holds units × 2^F — and wide is the declared + // range on that RAW scale, exact, as two 128-bit two's-complement values + // in 64-bit lanes (low lane first). NULL where the declaration bounds + // nothing (a bare uint128) and for every other kind; frac_bits is 0 for + // every kind that is not fixed-point. range_min/range_max still carry + // the declared bounds as doubles — whole units for a fixed field — for + // a walker that only shows them. + uint8_t frac_bits; + const TableWideRange * wide; + int64_t enum_max; // enums: highest valid value (None = 0 always valid); + // unions: the arm count (tag range [0, enum_max]); + // flags: the highest declared BIT INDEX; else -1 + // the vocabulary's names, indexed the same way enum_max bounds: an enum's + // value -> name, a union's tag -> arm name, a FLAGS field's bit index -> + // variant name. NULL for every other kind. + const char * (*enum_name)( uint64_t value ); + // the TABLE-WIRE id of one variant (docs/SPEC-TABLES.md §5): for an enum, the + // hash of the variant's name; for a union, the hash of the arm's name. + // 0 is the reserved id — an enum's None, a union's empty. NULL for every + // other kind — a FLAGS field's variants have no per-variant wire id (§4), + // so a NULL here beside a non-NULL enum_name is what says "flags". + // Walk [0, enum_max] to enumerate a vocabulary and its ids. + uint64_t (*variant_id)( uint64_t value ); + // an ENUM-KEYED array (docs/SPEC-TABLES.md §2.4): the array has one slot per + // variant of key_type_name, indexed by the variant's value, and its slots + // ride under variant ids rather than positions. key_name and key_id are + // the key's vocabulary — walk [0, array_bound) to print slots by name. + // NULL on every other field. + const char * key_type_name; + const char * (*key_name)( uint64_t value ); + uint64_t (*key_id)( uint64_t value ); + // union fields: the tag and its arms, behind a function so the whole + // descriptor stays CONSTANT-INITIALISED (a captureless lambda converts to + // a function pointer at compile time; the arms themselves are a static + // inside it). NULL for every other kind. + const TableUnionInfo * (*arms)(); + // an OUT-OF-LINE array (docs/SPEC-TABLES.md §8.1): place one element and + // hand it back at its defaults. A MAP places BY KEY, a string key comes + // in as the bytes and the length, an integer key as the value, and NULL + // is NOT INSERTED: a key past the bound, or an arena that could not carve + // another segment. A LIST ignores the key and APPENDS, NULL at the arena + // or the int32 cap. NULL on every field that is neither. + void * ( * place )( TableWorker & worker, void * slot, const char * key, int32_t key_length, int64_t key_value ); + const char * guard; // branch guard, e.g. "at_rest" or "!at_rest"; "" if unguarded + // what a PERSON wrote about the field (docs/SPEC-TABLES.md §8.1): the /// + // block above it, verbatim (SPEC §4.1). It is TableDocNone when there is + // none, never NULL. Its tags (SPEC §4.2) follow in declared order, and an + // untagged field is 0 beside NULL. Static, constant-initialized, + // allocating nothing. + const char * doc; + int32_t num_tags; + const char * const * tags; +}; + +struct TableTypeInfo +{ + const char * name; // schema type name + uint32_t size; // sizeof the storage struct + int32_t num_fields; + const TableFieldInfo * fields; + // put one instance back at its declared defaults, in place. A generic + // walker that fills a value has to be able to establish the defaults an + // absent field takes, and it holds no type to spell — this is the one + // thing the descriptors could not express without it. Placement-new + // value-init, exactly what the wire's read path does, and no temporary. + void (*reset)( void * storage ); + // the DERIVED mode (docs/SPEC-TABLES.md): false = fixed-size, a plain + // relocatable struct; true = variable-length, built through a Builder + // and read through a region root. Nobody declares it; the compiler + // works it out. + bool variable; + // the declaration's own doc and tags, on the same terms as a field's + // (docs/SPEC-TABLES.md §8.1) + const char * doc; + int32_t num_tags; + const char * const * tags; +}; + +struct TableWriter +{ + uint8_t * buffer; + int64_t capacity; + int64_t offset = 0; + bool overflow = false; + + // the parameters do not repeat the member names: a parameter that hides a + // member is a warning the estate's compilers disagree about (gcc's + // -Wshadow and cl's C4458 refuse it, clang's -Wshadow does not), and this + // is a header a consumer compiles under its OWN flags + TableWriter( uint8_t * to_buffer, int64_t to_capacity ) : buffer( to_buffer ), capacity( to_capacity ) {} + + MAPDEMO_TABLE_INLINE void raw( const void * data, int64_t bytes ) + { + if ( offset + bytes > capacity ) { overflow = true; return; } + memcpy( buffer + offset, data, (size_t) bytes ); + offset += bytes; + } + MAPDEMO_TABLE_INLINE void put8( uint8_t v ) { raw( &v, 1 ); } + MAPDEMO_TABLE_INLINE void put16( uint16_t v ) { uint8_t b[2] = { uint8_t( v ), uint8_t( v >> 8 ) }; raw( b, 2 ); } + MAPDEMO_TABLE_INLINE void put32( uint32_t v ) { uint8_t b[4] = { uint8_t( v ), uint8_t( v >> 8 ), uint8_t( v >> 16 ), uint8_t( v >> 24 ) }; raw( b, 4 ); } + MAPDEMO_TABLE_INLINE void put64( uint64_t v ) { put32( uint32_t( v ) ); put32( uint32_t( v >> 32 ) ); } + // a 128-bit value as two lanes, the low half first (docs/SPEC-TABLES.md §3) + MAPDEMO_TABLE_INLINE void put128( uint64_t lo, uint64_t hi ) { put64( lo ); put64( hi ); } + // EVERY LENGTH, COUNT, INDEX AND ID REFERENCE IS ONE CANONICAL UNSIGNED + // LEB128 (docs/SPEC-TABLES.md §3): seven value bits a byte, the lowest + // group first, the high bit set on every byte but the last. One value has + // one spelling, so two conforming writers agree byte for byte. + MAPDEMO_TABLE_INLINE void putleb( uint64_t v ) + { + while ( v >= 0x80 ) { put8( uint8_t( v ) | 0x80 ); v >>= 7; } + put8( uint8_t( v ) ); + } +}; + +// TableLebBytes is one value's spelling length, which a MEASURE needs before +// the bytes exist — the length of a body has to be known before it is written, +// because a length whose own width moves cannot be patched in place. +inline int64_t TableLebBytes( uint64_t v ) +{ + int64_t n = 1; + while ( v >= 0x80 ) { v >>= 7; n++; } + return n; +} + +// THE ID TABLE, WRITER SIDE (docs/SPEC-TABLES.md §3). It holds every id the +// body used, once each, in FIRST-USE order over the whole wire, and the body +// names them by position: reference k is the kth entry, counted from 1, and +// reference 0 names NO ID. +// +// Its capacity is a COMPILE-TIME fact of the unit — the distinct names its +// table closure can spell — so a save allocates nothing: the table is a local +// of Measure and of Save. The bucket chain makes ref constant time and makes +// truncate constant time too, which is what an ELIDED field needs: a field +// that turns out not to ride costs nothing in the id table either, so the walk +// interns its id, builds the payload that decides, and undoes the entry when +// nothing rides. +struct TableIds +{ + static const int32_t kCapacity = 76; + static const int32_t kBuckets = 256; + + uint64_t ids[ kCapacity ]; + int32_t chain[ kCapacity ]; + int32_t head[ kBuckets ]; + int32_t count; + bool overflow; + + TableIds() : count( 0 ), overflow( false ) + { + for ( int32_t i = 0; i < kBuckets; i++ ) { head[i] = -1; } + } + + static MAPDEMO_TABLE_INLINE uint32_t bucket_of( uint64_t id ) + { + return uint32_t( ( id * 0x9E3779B97F4A7C15ull ) >> 56 ) & uint32_t( kBuckets - 1 ); + } + + // the reference an id takes: the file's own first-use entry, appended on + // first use. The MESSAGE form names no id at all: its references are + // compile-time slots of the announced vocabulary (docs/SPEC-TABLES.md §3.3). + MAPDEMO_TABLE_INLINE uint64_t ref( uint64_t id ) + { + const uint32_t b = bucket_of( id ); + for ( int32_t i = head[b]; i >= 0; i = chain[i] ) + { + if ( ids[i] == id ) { return uint64_t( i ) + 1; } + } + if ( count >= kCapacity ) { overflow = true; return 1; } + ids[count] = id; chain[count] = head[b]; head[b] = count; count++; + return uint64_t( count ); + } + + // undo every entry appended since mark. An entry removed is the most + // recent one in its bucket, so it sits at that bucket's head. + void truncate( int32_t mark ) + { + while ( count > mark ) + { + count--; + head[ bucket_of( ids[count] ) ] = chain[count]; + } + } +}; + +// TableIdsBytes is the trailer's own size: the entries, each a fixed +// little-endian u64, and the ENTRY COUNT, the one fixed-width number on the +// wire (docs/SPEC-TABLES.md §3). +inline int64_t TableIdsBytes( const TableIds & ids ) { return int64_t( ids.count ) * 8 + 8; } + +// TableIdsWrite puts the trailer where the walk ended: a writer never patches, +// because first-use order is known only when the walk ends. +inline void TableIdsWrite( TableWriter & w, const TableIds & ids ) +{ + for ( int32_t i = 0; i < ids.count; i++ ) { w.put64( ids.ids[i] ); } + w.put64( uint64_t( ids.count ) ); +} + +// THE ID TABLE, READER SIDE (docs/SPEC-TABLES.md §3). A reader locates it from +// the END of the wire and resolves it ONCE, at open: the entries are eight +// bytes each and a body names them by position, so every field dispatches +// through an index rather than through a search over hashes. +struct TableIdTable +{ + const uint8_t * entries = NULL; + int64_t count = 0; + + // the id a reference names. ref is 1-based and bounds-checked by the + // caller: a reference ABOVE the entry count is framing damage on the body + // that carries it, and 0 names no id at all. + uint64_t at( uint64_t ref ) const + { + const uint8_t * e = entries + ( ref - 1 ) * 8; + uint64_t lo = uint64_t( e[0] ) | uint64_t( e[1] ) << 8 | uint64_t( e[2] ) << 16 | uint64_t( e[3] ) << 24; + uint64_t hi = uint64_t( e[4] ) | uint64_t( e[5] ) << 8 | uint64_t( e[6] ) << 16 | uint64_t( e[7] ) << 24; + return lo | ( hi << 32 ); + } +}; + +struct TableReader +{ + const uint8_t * buffer; + int64_t size; + int64_t offset = 0; + TableReport * report; + const TableIdTable * ids = NULL; + // ONLY THE ROOT BODY CARRIES THE NODE TABLE (docs/SPEC-TABLES.md §3.1), so + // a body has to know which it is: the reserved id inside a NESTED body is + // malformed, because a second numbering cannot exist. Every reader made + // for a payload is nested; the two the wire surfaces make for a root say so. + bool nested = true; + + TableReader( const uint8_t * from_buffer, int64_t from_size, TableReport * to_report ) + : buffer( from_buffer ), size( from_size ), report( to_report ) {} + + TableReader( const uint8_t * from_buffer, int64_t from_size, TableReport * to_report, const TableIdTable * to_ids ) + : buffer( from_buffer ), size( from_size ), report( to_report ), ids( to_ids ) {} + + MAPDEMO_TABLE_INLINE bool has( int64_t bytes ) const { return offset + bytes <= size; } + // A LENGTH IS A 64-BIT NUMBER AND A BUFFER IS NOT (docs/SPEC-TABLES.md + // §3): every length, count and index on this wire has sixty-four bits of + // capability, so one past what remains must be compared UNSIGNED. Casting + // it to int64 first turns 0xFFFFFFFFFFFFFFFF into -1, and a negative + // length looks like room. + MAPDEMO_TABLE_INLINE bool room( uint64_t bytes ) const { return bytes <= (uint64_t) ( size - offset ); } + MAPDEMO_TABLE_INLINE uint8_t get8() { return buffer[offset++]; } + MAPDEMO_TABLE_INLINE uint16_t get16() { uint16_t v = uint16_t( buffer[offset] ) | uint16_t( buffer[offset+1] ) << 8; offset += 2; return v; } + MAPDEMO_TABLE_INLINE uint32_t get32() { uint32_t v = uint32_t( buffer[offset] ) | uint32_t( buffer[offset+1] ) << 8 | uint32_t( buffer[offset+2] ) << 16 | uint32_t( buffer[offset+3] ) << 24; offset += 4; return v; } + MAPDEMO_TABLE_INLINE uint64_t get64() { uint64_t lo = get32(); uint64_t hi = get32(); return lo | ( hi << 32 ); } + MAPDEMO_TABLE_INLINE void get128( uint64_t & lo, uint64_t & hi ) { lo = get64(); hi = get64(); } + + // ONE CANONICAL UNSIGNED LEB128 (docs/SPEC-TABLES.md §3), and a + // non-minimal spelling is MALFORMED: 0x80 0x00 and 0x00 both spell zero, + // and only the second is legal input. An encoding past ten bytes, or a + // tenth byte with a bit above the 64th value bit, is malformed on the same + // rule. false = framing damage on the body carrying it. + bool getleb( uint64_t & value ) + { + // A NUMBER THIS READER REFUSES LEAVES THE CURSOR WHERE IT WAS. The + // caller's next question is often "did this body end exactly at its + // L", and a rejected number that had moved the cursor would answer + // that question with the damage already stepped over. + const int64_t at = offset; + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( !has( 1 ) ) { offset = at; return false; } + const uint8_t b = get8(); + if ( i == 9 && b > 1 ) { offset = at; return false; } + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) + { + if ( i > 0 && b == 0 ) { offset = at; return false; } // a redundant continuation + return true; + } + shift += 7; + } + offset = at; + return false; + } + + // resolve one id reference against the file's table. false = a reference + // ABOVE the entry count, or a 0 where an id is required, both of which + // are framing damage on the body that carries it. + bool getid( uint64_t & id ) + { + uint64_t ref = 0; + if ( !getleb( ref ) ) { return false; } + if ( ref == 0 || ids == NULL || ref > (uint64_t) ids->count ) { return false; } + id = ids->at( ref ); + return true; + } + + // skip one payload by kind; false = framing damage. FOUR RULES COVER THE + // SET (docs/SPEC-TABLES.md §3), and a kind outside it is not skippable — + // which is why the set is closed and why kind 31 exists. + bool skip( uint8_t kind ) + { + switch ( kind ) + { + // the fixed-width kinds, each by its width: 18-29 are the 128-bit integers and + // the fixed-point family at every storage width (docs/SPEC-TABLES.md §3) + case 1: case 2: case 6: case 20: case 25: return has( 1 ) ? ( offset += 1, true ) : false; + case 3: case 7: case 21: case 26: return has( 2 ) ? ( offset += 2, true ) : false; + case 4: case 8: case 10: case 22: case 27: return has( 4 ) ? ( offset += 4, true ) : false; + case 5: case 9: case 11: case 23: case 28: return has( 8 ) ? ( offset += 8, true ) : false; + case 18: case 19: case 24: case 29: return has( 16 ) ? ( offset += 16, true ) : false; + case 17: case 30: // a NODE INDEX (§3.1) and an ENUM's variant reference: one LEB128 and stop + { + uint64_t ignored = 0; + return getleb( ignored ); + } + case 12: case 13: case 14: case 16: case 31: case 32: case 33: // 31 is the ESCAPE, 32 the payload-free kind, 33 wide text + { + uint64_t n = 0; + if ( !getleb( n ) ) return false; + return room( n ) ? ( offset += (int64_t) n, true ) : false; + } + case 15: // union: the arm id reference, then its kind, its L and its payload (reference 0 = empty) + { + uint64_t arm = 0; + if ( !getleb( arm ) ) return false; + if ( arm == 0 ) return true; + if ( !has( 1 ) ) return false; + offset += 1; // the arm's kind byte + uint64_t n = 0; + if ( !getleb( n ) ) return false; + return room( n ) ? ( offset += (int64_t) n, true ) : false; + } + // KIND 34 IS RESERVED FOR float16 AND IS NOT PART OF THIS MAJOR (§3): + // no writer emits it and no reader has a rule for it, so a reader + // meets it only as DAMAGE, exactly as it meets 35 or 200. A bare 34 + // is a writer that ignored the escape kind 31. + case 34: return false; + } + return false; + } +}; + + +// WIDENING (docs/SPEC-TABLES.md §4): a payload under a kind BELOW the reader's +// on the same ladder decodes exactly. The signed ladder is kinds 2, 3, 4, 5, +// 18, the unsigned one 6, 7, 8, 9, 19, and 10 into 11 is the float rung. Every +// other pair is a kind mismatch. The declared kind is a constant at every call +// site, so this folds to one or two comparisons on the mismatch path and to +// nothing on the matching one. +inline bool TableKindWidens( uint8_t kind, uint8_t declared ) +{ + switch ( declared ) + { + case 3: case 4: case 5: return kind >= 2 && kind < declared; + case 18: return kind >= 2 && kind <= 5; + case 7: case 8: case 9: return kind >= 6 && kind < declared; + case 19: return kind >= 6 && kind <= 9; + case 11: return kind == 10; + } + return false; +} + +// a fixed-width kind's payload width, for the one place the width is a +// runtime fact: an arm whose kind byte the reader widens, whose L must be the +// wire kind's own width (§3) +inline int64_t TableKindWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + } + return 0; +} + +// the payload of a kind on the SIGNED ladder (2 to 5), sign-extended to +// sixty-four bits; false = the body cannot cover it, which is framing damage +inline bool TableReadSignedAt( TableReader & r, uint8_t kind, int64_t & out ) +{ + switch ( kind ) + { + case 2: if ( !r.has( 1 ) ) { return false; } out = (int8_t) r.get8(); return true; + case 3: if ( !r.has( 2 ) ) { return false; } out = (int16_t) r.get16(); return true; + case 4: if ( !r.has( 4 ) ) { return false; } out = (int32_t) r.get32(); return true; + default: if ( !r.has( 8 ) ) { return false; } out = (int64_t) r.get64(); return true; + } +} + +// the payload of a kind on the UNSIGNED ladder (6 to 9), zero-extended +inline bool TableReadUnsignedAt( TableReader & r, uint8_t kind, uint64_t & out ) +{ + switch ( kind ) + { + case 6: if ( !r.has( 1 ) ) { return false; } out = r.get8(); return true; + case 7: if ( !r.has( 2 ) ) { return false; } out = r.get16(); return true; + case 8: if ( !r.has( 4 ) ) { return false; } out = r.get32(); return true; + default: if ( !r.has( 8 ) ) { return false; } out = r.get64(); return true; + } +} + +// f32 into f64, exact: a NaN's payload is data and rides on the bits, since +// the hardware conversion would set the quiet bit (§4) +inline double TableWidenF32( uint32_t bits ) +{ + if ( ( bits & 0x7F800000u ) == 0x7F800000u && ( bits & 0x007FFFFFu ) != 0 ) + { + const uint64_t sign = (uint64_t) ( bits >> 31 ) << 63; + const uint64_t payload = (uint64_t) ( bits & 0x007FFFFFu ) << 29; + const uint64_t nan_bits = sign | 0x7FF0000000000000ull | payload; + double d; memcpy( &d, &nan_bits, 8 ); return d; + } + float f; memcpy( &f, &bits, 4 ); return (double) f; +} + +// ILL-FORMED TEXT IS DAMAGE (docs/SPEC-TABLES.md §3, §4): a kind 12 payload is +// well-formed UTF-8 with no zero byte among its bytes, checked AS IT ARRIVES +// and before the reader's own bound, because a payload that is not text is not +// text at whatever length the reader would have kept. Rejects a zero byte, a +// truncated sequence, a bare continuation, an overlong encoding, a surrogate +// and a code point past U+10FFFF, which is SPEC.md §4.7's rule in this wire's +// idiom: the field reads its declared default, one malformed counts, and the +// parent reads on past L. +// +// A LENGTH IS A 64-BIT NUMBER (§3), so it arrives as one: a payload length is +// whatever the wire spelled, and narrowing it to a signed count would read +// 0xFFFFFFFFFFFFFFFF as an empty payload. +inline bool TableUtf8Valid( const uint8_t * bytes, uint64_t length ) +{ + uint64_t i = 0; + while ( i < length ) + { + const uint8_t lead = bytes[i]; + uint64_t continuations; + uint32_t code_point; + if ( lead == 0 ) { return false; } + if ( lead < 0x80 ) { i++; continue; } + else if ( ( lead & 0xE0 ) == 0xC0 ) { continuations = 1; code_point = lead & 0x1F; } + else if ( ( lead & 0xF0 ) == 0xE0 ) { continuations = 2; code_point = lead & 0x0F; } + else if ( ( lead & 0xF8 ) == 0xF0 ) { continuations = 3; code_point = lead & 0x07; } + else { return false; } + if ( i + continuations >= length ) { return false; } + for ( uint64_t k = 1; k <= continuations; k++ ) + { + if ( ( bytes[i + k] & 0xC0 ) != 0x80 ) { return false; } + code_point = ( code_point << 6 ) | uint32_t( bytes[i + k] & 0x3F ); + } + if ( continuations == 1 && code_point < 0x80 ) { return false; } + if ( continuations == 2 && ( code_point < 0x800 || ( code_point >= 0xD800 && code_point <= 0xDFFF ) ) ) { return false; } + if ( continuations == 3 && ( code_point < 0x10000 || code_point > 0x10FFFF ) ) { return false; } + i += 1 + continuations; + } + return true; +} + +// A CLAMP CUTS AT A CODE POINT BOUNDARY (§3, §16.2): the last whole code point +// that fits within the bound, over a payload the check above already accepted, +// so a clamp can never invent ill-formed storage. +// +// THE ANSWER IS NEVER ABOVE THE BOUND. The length arrives as the wire's own +// 64-bit number and the caller turns the answer back into the size of a copy, +// so a length no reader could have bounded has to leave here bounded: taken as +// a signed count, 0xFFFFFFFFFFFFFFFF is -1, -1 is under every bound, and the +// copy would run at SIZE_MAX. +inline int64_t TableUtf8Clamp( const uint8_t * bytes, uint64_t length, int64_t bound ) +{ + if ( length <= (uint64_t) bound ) { return (int64_t) length; } + int64_t cut = bound; + while ( cut > 0 && ( bytes[cut] & 0xC0 ) == 0x80 ) { cut--; } + return cut; +} + +// ONE CODE UNIT off the wire: two bytes LITTLE-ENDIAN, this wire's order for +// every fixed-width number (docs/SPEC-TABLES.md §3). No unit can exceed +// 0xFFFF, because two bytes cannot spell one. +inline uint16_t TableUtf16Unit( const uint8_t * bytes, int64_t index ) +{ + return uint16_t( uint16_t( bytes[index * 2] ) | ( uint16_t( bytes[index * 2 + 1] ) << 8 ) ); +} + +// ILL-FORMED WIDE TEXT IS DAMAGE (docs/SPEC-TABLES.md §3, §4): a kind 33 +// payload carrying an UNPAIRED SURROGATE or a ZERO CODE UNIT among its units, +// checked AS IT ARRIVES and before the reader's own bound, on the rule kind 12 +// takes for UTF-8. An ODD L is framing damage and the caller rejects it ahead +// of this, because units is L / 2. SPEC.md §4.12 refuses the same content +// TERMINALLY on the packet wire; here the field reads its declared default, +// one malformed counts, and the parent reads on past L. +inline bool TableUtf16Valid( const uint8_t * bytes, int64_t units ) +{ + int64_t i = 0; + while ( i < units ) + { + const uint16_t unit = TableUtf16Unit( bytes, i ); + if ( unit == 0 ) { return false; } + if ( unit >= 0xD800 && unit <= 0xDBFF ) + { + if ( i + 1 >= units ) { return false; } // a high surrogate with no low half + const uint16_t low = TableUtf16Unit( bytes, i + 1 ); + if ( low < 0xDC00 || low > 0xDFFF ) { return false; } + i += 2; + continue; + } + if ( unit >= 0xDC00 && unit <= 0xDFFF ) { return false; } // a low surrogate first + i++; + } + return true; +} + +// A CLAMP CUTS AT A CODE UNIT BOUNDARY AND NEVER SPLITS A PAIR (§3, §16.2): +// the first bound units of a payload the check above already accepted, and +// where the last kept unit is a HIGH SURROGATE whose low half did not fit, +// that unit is dropped with it. So a clamp can never invent an unpaired +// surrogate, exactly as kind 12's clamp can never invent a broken sequence. +inline int64_t TableUtf16Clamp( const uint8_t * bytes, int64_t units, int64_t bound ) +{ + if ( units <= bound ) { return units; } + int64_t cut = bound; + if ( cut > 0 ) + { + const uint16_t last = TableUtf16Unit( bytes, cut - 1 ); + if ( last >= 0xD800 && last <= 0xDBFF ) { cut--; } + } + return cut; +} + +// The RESERVED node-table id, the one id the language holds back +// (docs/SPEC-TABLES.md §3.1, §5). It rides in every unit, pointered or not, +// because every body has to know that a NESTED body claiming one is damaged. +static const uint64_t kTableNodeTableFieldId = 0xFFFFFFFFFFFFFFFFull; + +// TableWireForm is the FORM BYTE, and it is the whole header +// (docs/SPEC-TABLES.md §3). A reader that meets a byte it does not know +// refuses the wire by name and never reports damage. +const uint8_t kTableWireForm = 1; + +// TableOpen reads the form byte and the trailer, in that order, and hands back +// the ROOT BODY. It answers one of three verdicts, because five zero counters +// and a false flag are what a clean read prints too: +// +// TableOpenOk the form is known and the table read whole +// TableOpenRefused a FORM BYTE this reader does not carry: nothing is +// decoded, nothing is counted, and no damage is reported +// TableOpenDamaged a table that cannot be read whole — fewer than eight +// bytes, a count whose entries run past the front of the +// file, a count that leaves no room for the form byte, or +// ONE ID IN TWO ENTRIES. The whole wire is malformed, +// nothing is decoded, and one event is counted. +// TableOpenBodyStopped the form and the table were good and the ROOT BODY +// could not be walked to its own terminator. What it +// decoded before that is kept, as everywhere on this wire. +enum TableOpenVerdict { TableOpenOk, TableOpenRefused, TableOpenDamaged, TableOpenBodyStopped }; + +inline TableOpenVerdict TableOpen( const uint8_t * buffer, int64_t bytes, TableIdTable & table, int64_t & body_bytes ) +{ + if ( bytes < 1 ) { return TableOpenDamaged; } + if ( buffer[0] != kTableWireForm ) { return TableOpenRefused; } + if ( bytes < 9 ) { return TableOpenDamaged; } + const uint8_t * tail = buffer + bytes - 8; + uint64_t lo = uint64_t( tail[0] ) | uint64_t( tail[1] ) << 8 | uint64_t( tail[2] ) << 16 | uint64_t( tail[3] ) << 24; + uint64_t hi = uint64_t( tail[4] ) | uint64_t( tail[5] ) << 8 | uint64_t( tail[6] ) << 16 | uint64_t( tail[7] ) << 24; + uint64_t count = lo | ( hi << 32 ); + if ( count > (uint64_t) ( bytes / 8 ) ) { return TableOpenDamaged; } + const int64_t span = (int64_t) count * 8 + 8; + if ( span + 1 > bytes ) { return TableOpenDamaged; } + table.entries = buffer + bytes - span; + table.count = (int64_t) count; + // THE ENTRIES ARE DISTINCT: a table that carries one id twice is malformed + // for the whole wire, because no wire this schema writes carries a repeat + // and it would leave one more shape of table for a hostile writer to aim + // at (docs/SPEC-TABLES.md §3). + for ( int64_t i = 1; i < table.count; i++ ) + { + const uint64_t id = table.at( uint64_t( i ) + 1 ); + for ( int64_t j = 0; j < i; j++ ) + { + if ( table.at( uint64_t( j ) + 1 ) == id ) { return TableOpenDamaged; } + } + } + body_bytes = bytes - span - 1; + return TableOpenOk; +} + +// TableBodyExtent walks a body's framing to the zero reference that ends it, +// so a reader can tell a body that ENDED EARLY — leaving bytes no field claims +// — from one that is merely damaged. ANY BYTE BETWEEN THE ROOT'S TERMINATOR +// AND THE TABLE'S FIRST ENTRY IS MALFORMED, because no field claims it and the +// two ends of the file have met (docs/SPEC-TABLES.md §3). +inline bool TableBodyEndsEarly( const uint8_t * body, int64_t bytes, const TableIdTable & table ) +{ + TableReport ignored; + TableReader r( body, bytes, &ignored, &table ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.getleb( ref ) ) { return false; } + if ( ref == 0 ) { return r.offset != bytes; } + if ( ref > (uint64_t) table.count ) { return false; } + if ( !r.has( 1 ) ) { return false; } + if ( !r.skip( r.get8() ) ) { return false; } + } +} + +// THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3): a batch of BITPACKED bodies +// under one announced vocabulary. +// +// A form 2 wire is THREE PARTS: the form byte, the body count, and the bodies +// as one continuous bit stream, zero-padded to the next byte at the end and +// nowhere else. A body is a sequence of fields, each a REFERENCE followed by a +// PAYLOAD and nothing else: no kind byte and no length, because the +// announcement carries the kind and the shape of every entry. +const uint8_t kTableWireMessageForm = 2; + +// THE COUNT IS A RANGED INTEGER OVER [1, 256], eight bits carrying M - 1. 256 +// is a WIRE CONSTANT of this form rather than a receiver's policy, because the +// count's WIDTH depends on it and two peers that disagreed on the width would +// not be reading the same wire. A batch of zero is not spellable. +static const int64_t kTableMessageBatchMax = 256; + +// The RESERVED ids of the announcement's own two fields (§5, §11), beside the +// node table's. They are the announcement's transport, they never appear in a +// body, and they take no slot in the vocabulary. +static const uint64_t kTableBuildVersionFieldId = 0xFFFFFFFFFFFFFFFEull; +static const uint64_t kTableMessageVocabularyFieldId = 0xFFFFFFFFFFFFFFFDull; + +// THE WIDEST COUNT THIS FORM SPELLS, which is the count an UNBOUNDED array +// announces (§2.9): an unbounded array states no bound, so the announcement +// states the widest one a batch could carry. It is the ceiling an array's or a +// keyed entry's announced min and max are checked against. +static const uint64_t kTableMessageListMax = 0xFFFFFFFFull; + +// THIS UNIT'S OWN REFERENCE WIDTH: the bits a writer spends on every reference +// of every body it writes, which is a compile-time constant because the +// vocabulary is. A READER spends the width the SENDER's vocabulary settles. +static const int64_t kTableMessageRefBitsHere = 7; + +// THIS UNIT'S OWN ENTRY COUNT, which is the CAPACITY a receiver declares for +// its resolved vocabulary when it talks only to peers of this schema (§3.3). +// The vocabulary is a pure function of the build version, so a peer at this +// build announces exactly this many entries; a receiver that means to meet +// OTHER builds declares more, and an announcement above whatever it declared +// is refused as vocabulary_too_large. +static const int64_t kTableMessageEntriesHere = 75; + +// The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A +// pointered body names the node table through it, and the node table is the +// ROOT body's FIRST field because a pointer index's width is settled by the +// node count it carries. +static const uint64_t kTableNodeTableFieldSlot = 54; + +// THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own +// layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, +// so a value written here and a value written by a generated packet writer are +// the same bits in the same places. + +// EIGHT BYTES OF THE STREAM AS ONE WORD, and the word is LITTLE-END-FIRST +// whatever order this host is in, because the stream's own definition puts +// bit i in byte i/8: byte 0 of the run holds the word's low eight bits. That +// is what lets one value of any width move in one unaligned load or store +// instead of one touch a byte, and the BITS ON THE WIRE do not move. +inline uint64_t table_message_byteswap64( uint64_t v ) +{ + return ( v >> 56 ) | ( ( v >> 40 ) & 0xff00ull ) | ( ( v >> 24 ) & 0xff0000ull ) | ( ( v >> 8 ) & 0xff000000ull ) + | ( ( v << 8 ) & 0xff00000000ull ) | ( ( v << 24 ) & 0xff0000000000ull ) | ( ( v << 40 ) & 0xff000000000000ull ) + | ( v << 56 ); +} + +inline uint64_t table_message_load64( const uint8_t * p ) +{ + uint64_t v = 0; + memcpy( &v, p, 8 ); +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + v = table_message_byteswap64( v ); +#endif + return v; +} + +inline void table_message_store64( uint8_t * p, uint64_t v ) +{ +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + v = table_message_byteswap64( v ); +#endif + memcpy( p, &v, 8 ); +} + +struct TableBitWriter +{ + uint8_t * buffer; + int64_t capacity; // bytes + int64_t bits; + bool overflow; + + TableBitWriter() : buffer( NULL ), capacity( 0 ), bits( 0 ), overflow( false ) {} + TableBitWriter( uint8_t * to_buffer, int64_t to_capacity ) : buffer( to_buffer ), capacity( to_capacity ), bits( 0 ), overflow( false ) {} + + // ONE WORD AT A TIME, never one bit and never one byte of arithmetic: the + // value is shifted into place in a REGISTER once, and the bytes it + // occupies are stored from that register with no read back. A sixty-four + // bit field costs one shift rather than nine masked read-modify-writes. + // The word is assembled little-end-first, so the BITS ON THE WIRE are the + // same bits in the same places, bit i in byte i/8 at position i%8 with the + // low bit first, which is what the pinned goldens hold. IT WRITES EXACTLY + // THE BYTES THE VALUE OCCUPIES and never one past them, so a caller's + // buffer beyond the batch is its own. + void put( uint64_t value, int64_t n ) + { + if ( n <= 0 ) { return; } + if ( ( bits + n + 7 ) / 8 > capacity ) { overflow = true; bits += n; return; } + if ( n < 64 ) { value &= ( uint64_t( 1 ) << n ) - 1; } // a caller's high bits never leak + const int64_t index = bits >> 3; + const int64_t bit = bits & 7; + // the byte the write STARTS in keeps the bits already written to it, + // and every byte after it is this value's own + const uint64_t head = bit != 0 ? ( uint64_t( buffer[index] ) & ( ( uint64_t( 1 ) << bit ) - 1 ) ) : 0; + const uint64_t word = head | ( value << bit ); + const int64_t need = ( bit + n + 7 ) >> 3; // 1 to 9 bytes + if ( need >= 8 ) + { + table_message_store64( buffer + index, word ); + if ( need > 8 ) { buffer[index + 8] = uint8_t( value >> ( 64 - bit ) ); } + } + else + { + for ( int64_t i = 0; i < need; i++ ) { buffer[index + i] = uint8_t( word >> ( 8 * i ) ); } + } + bits += n; + } + + // THE ALIGN IS WHAT BUYS THIS (docs/SPEC-TABLES.md §3.3): a string(N), a + // bytes(N) and a blob record align before their bytes precisely so the + // largest payload on the wire moves as ONE memcpy. Off a boundary there is + // nothing to memcpy and the bytes go through put. + void putbytes( const uint8_t * data, int64_t n ) + { + if ( n <= 0 ) { return; } + if ( ( bits & 7 ) == 0 ) + { + if ( ( bits >> 3 ) + n > capacity ) { overflow = true; bits += n * 8; return; } + memcpy( buffer + ( bits >> 3 ), data, (size_t) n ); + bits += n * 8; + return; + } + for ( int64_t i = 0; i < n; i++ ) { put( (uint64_t) data[i], 8 ); } + } + + // a string's or a bytes' payload ALIGNS before its bytes, and a batch + // aligns once at its end. Both are zero fill, spent in one call. + void align() { put( 0, ( 8 - ( bits & 7 ) ) & 7 ); } +}; + +// TableAlignBits is what an align costs from a bit position, which a measure +// spends exactly where a save does. +inline int64_t TableAlignBits( int64_t bits ) { return ( 8 - ( bits % 8 ) ) % 8; } + +struct TableBitReader +{ + const uint8_t * buffer; + int64_t bits; // the stream's extent, in bits + int64_t offset; // bits consumed + + TableBitReader() : buffer( NULL ), bits( 0 ), offset( 0 ) {} + TableBitReader( const uint8_t * from_buffer, int64_t from_bytes ) : buffer( from_buffer ), bits( from_bytes * 8 ), offset( 0 ) {} + + bool has( int64_t n ) const { return n >= 0 && offset + n <= bits; } + + // the primitive is sixty-four bits, and a width above it is refused + // here as well as at the announcement: no field on any body can ask this + // reader to move more bits than it holds + // ONE WORD OF THE BUFFER AT A TIME, the mirror of the writer's put: the + // eight bytes the value starts in load as one little-end-first word and a + // ninth byte carries the spill a value that straddles the word needs. + // Within nine bytes of the stream's end there is no room for a word load + // and the bytes come one at a time, by the same arithmetic. + bool get( uint64_t & value, int64_t n ) + { + if ( n > 64 || !has( n ) ) { return false; } + value = 0; + if ( n == 0 ) { return true; } + const int64_t index = offset >> 3; + const int64_t bit = offset & 7; + const int64_t bytes = ( bits + 7 ) >> 3; + if ( index + 9 <= bytes ) + { + uint64_t v = table_message_load64( buffer + index ) >> bit; + if ( bit != 0 && bit + n > 64 ) { v |= uint64_t( buffer[index + 8] ) << ( 64 - bit ); } + value = n == 64 ? v : ( v & ( ( uint64_t( 1 ) << n ) - 1 ) ); + offset += n; + return true; + } + int64_t got = 0; + while ( got < n ) + { + const int64_t byte = offset >> 3; + const int64_t off = offset & 7; + const int64_t room = 8 - off; + const int64_t take = ( n - got ) < room ? ( n - got ) : room; + const uint64_t chunk = ( uint64_t( buffer[byte] ) >> off ) & ( ( uint64_t( 1 ) << take ) - 1 ); + value |= chunk << got; + offset += take; + got += take; + } + return true; + } + + // the bytes of an ALIGNED payload, which is the read side of the memcpy + // the align buys (docs/SPEC-TABLES.md §3.3) + bool getbytes( uint8_t * out, int64_t n ) + { + if ( n < 0 || !has( n * 8 ) ) { return false; } + if ( ( offset & 7 ) == 0 ) + { + memcpy( out, buffer + ( offset >> 3 ), (size_t) n ); + offset += n * 8; + return true; + } + for ( int64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !get( by, 8 ) ) { return false; } + out[i] = (uint8_t) by; + } + return true; + } + + bool skip( int64_t n ) { if ( !has( n ) ) { return false; } offset += n; return true; } + + // the pad to the next byte boundary is VERIFIED ZERO, which is the packet + // wire's rule for the same reason (SPEC.md §4.3) + bool align() + { + const int64_t pad = ( 8 - ( offset & 7 ) ) & 7; + if ( pad == 0 ) { return true; } + uint64_t bits_read = 0; + return get( bits_read, pad ) && bits_read == 0; + } +}; + +// TableBitsRequired is bits_required( min, max ): the bit length of max - min, +// and zero where the two are equal, which is a value that spends no bit at all. +inline int64_t TableBitsRequired( int64_t min, int64_t max ) +{ + if ( max <= min ) { return 0; } + uint64_t span = (uint64_t) ( max - min ); + int64_t n = 0; + while ( span > 0 ) { n++; span >>= 1; } + return n; +} + +// THE ANNOUNCED ENTRY (§3.3): an id, a kind, and a SHAPE, which is the width +// and range facts a reader needs to SKIP a field exactly and to DECODE one +// whose own declaration has moved. One name may take TWO entries, at two kinds or two +// shapes, and a body names the one it means. +// +// The ELEMENT's own facts ride beside the field's because an array's element +// is the one nesting this wire has: an array of arrays is not a table-wire +// construct, so one level is every level. +// +// IT IS THE RESOLVED ENTRY AND THE CALLER SIZES AN ARRAY OF THEM, so it +// carries what a DECODE takes and nothing a decode does not: qmin, qdelta and +// qcount are what SPEC.md §4.3's rule leaves behind, and the qmax and qres +// that rule CONSUMES are locals of the parse. The widths are int16 because a +// width is bounded by the kind it came under and no kind holds more than 128 +// bits. +struct TableMessageEntry +{ + uint64_t id = 0; + int64_t min = 0; // an array's minimum count + int64_t max = 0; // an array's maximum count, a string's capacity, a keyed array's slots + int64_t base_lo = 0; // the ranged base, low half: a signed kind's sign-extends, an unsigned kind's is whole + int64_t base_hi = 0; // its high half, for a 128-bit kind + int64_t elem_max = 0; + int64_t elem_base_lo = 0; + int64_t elem_base_hi = 0; + // what SPEC.md §4.3's derivation leaves: the base, the step and the count + float qmin = 0.0f; + float qdelta = 0.0f; + uint32_t qcount = 0; + float elem_qmin = 0.0f; + float elem_qdelta = 0.0f; + uint32_t elem_qcount = 0; + // THE PAYLOAD'S WIDTH, RESOLVED: what the kind, the packing and the + // announced bits together say, computed once at AnnounceRead, and -1 + // where the payload is not a fixed-width value at all + int16_t value_bits = -1; + int16_t elem_value_bits = -1; + uint8_t kind = 0; + uint8_t packing = 0; + uint8_t elem_kind = 0; + uint8_t elem_packing = 0; +}; + +// TableMessageEntrySame reports whether two RESOLVED entries carry the same +// shape, which is every fact of the entry but its id and its kind. It is what +// the announcement's duplicate rule is asked in: two entries that agree on all +// three parts are malformed (§3.3). +inline bool TableMessageEntrySame( const TableMessageEntry & a, const TableMessageEntry & b ) +{ + return a.min == b.min && a.max == b.max && a.base_lo == b.base_lo && a.base_hi == b.base_hi + && a.elem_max == b.elem_max && a.elem_base_lo == b.elem_base_lo && a.elem_base_hi == b.elem_base_hi + && a.qmin == b.qmin && a.qdelta == b.qdelta && a.qcount == b.qcount + && a.elem_qmin == b.elem_qmin && a.elem_qdelta == b.elem_qdelta && a.elem_qcount == b.elem_qcount + && a.value_bits == b.value_bits && a.elem_value_bits == b.elem_value_bits + && a.packing == b.packing && a.elem_kind == b.elem_kind && a.elem_packing == b.elem_packing; +} + +// TableMessageKindBits is the widest RANGED value a kind can carry, its own +// storage width: a width above it is a hostile width on the announcement. +inline int64_t TableMessageKindBits( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 6: case 20: case 25: return 8; + case 3: case 7: case 21: case 26: return 16; + case 4: case 8: case 22: case 27: return 32; + case 5: case 9: case 23: case 28: return 64; + default: return 128; + } +} + +// TableMessageQuantization is SPEC.md §4.3's derivation over an announced +// triple, in float32 and by nothing else: delta, the step count and the +// width. False is a triple SPEC.md calls non-conforming, which on the +// announcement is a hostile width like any other (§3.3). +inline bool TableMessageQuantization( float qmin, float qmax, float qres, float & delta, uint32_t & count, int64_t & bits ) +{ + if ( !( qmin < qmax ) || !( qres > 0.0f ) ) { return false; } + delta = qmax - qmin; + float values = delta / qres; + if ( !( delta - delta == 0.0f ) || !( values - values == 0.0f ) ) { return false; } // Inf - Inf is NaN + if ( !( values >= 1.0f ) ) { values = 1.0f; } + else if ( values > 4294967040.0f ) { values = 4294967040.0f; } // the largest float below 2^32 + count = (uint32_t) values; + if ( (float) count < values ) { count++; } // ceil, on a value the cast holds exactly + bits = TableBitsRequired( 0, (int64_t) count ); + return true; +} + +// The two roundings on each side of the rule (SPEC.md §7.2): the product +// rounds to float32 BEFORE the add, which a compiler permitted to contract +// would otherwise fuse into one rounding and move the wire. +#if ( defined( __GNUC__ ) || defined( __clang__ ) ) && ( defined( __aarch64__ ) || defined( _M_ARM64 ) ) +#define TABLE_FLOAT_FORCE_ROUND( x ) __asm__ ( "" : "+w" ( x ) ) +#elif ( defined( __GNUC__ ) || defined( __clang__ ) ) && ( defined( __x86_64__ ) || defined( __i386__ ) ) +#define TABLE_FLOAT_FORCE_ROUND( x ) __asm__ ( "" : "+x" ( x ) ) +#else +#define TABLE_FLOAT_FORCE_ROUND( x ) do { volatile float table_float_force_round_slot = ( x ); ( x ) = table_float_force_round_slot; } while ( 0 ) +#endif + +// TableMessageQuantize is the writer's half: the index a value takes. +inline uint32_t TableMessageQuantize( float value, float qmin, float delta, uint32_t count ) +{ + float normalized = ( value - qmin ) / delta; + if ( !( normalized >= 0.0f ) ) { normalized = 0.0f; } + else if ( !( normalized <= 1.0f ) ) { normalized = 1.0f; } + float scaled = normalized * (float) count; + TABLE_FLOAT_FORCE_ROUND( scaled ); + uint32_t index = (uint32_t) ( scaled + 0.5f ); // floor of a non-negative value + if ( index > count ) { index = count; } + return index; +} + +// TableMessageDequantize is the reader's half: the float an index names. +inline float TableMessageDequantize( uint32_t index, float qmin, float delta, uint32_t count ) +{ + if ( index > count ) { index = count; } + const float normalized = index / (float) count; + float scaled = normalized * delta; + TABLE_FLOAT_FORCE_ROUND( scaled ); + return scaled + qmin; +} + +inline bool TableMessageIntegerKind( uint8_t kind ) +{ + return ( kind >= 2 && kind <= 9 ) || kind == 18 || kind == 19; +} + +inline bool TableMessageFixedKind( uint8_t kind ) { return kind >= 20 && kind <= 29; } + +inline bool TableMessageKnownKind( uint8_t kind ) +{ + return kind == 0 || ( kind >= 1 && kind <= 17 ) || ( kind >= 18 && kind <= 29 ) || ( kind >= 30 && kind <= 33 ); +} + +// A CANONICAL LEB128, which is the announcement's own integer: the +// announcement is a form 1 FILE and takes §3's rule. +inline bool TableMessageLeb( const uint8_t * in, int64_t size, int64_t & at, uint64_t & value ) +{ + value = 0; + for ( int64_t shift = 0; at < size; shift += 7 ) + { + if ( shift >= 64 ) { return false; } + const uint8_t by = in[ at++ ]; + value |= uint64_t( by & 0x7F ) << shift; + if ( ( by & 0x80 ) == 0 ) { return !( shift > 0 && by == 0 ); } + } + return false; +} + +// TableMessageShapeFacts is where one shape's facts land: the field's own, +// or its element's, which is the one nesting this wire has. +struct TableMessageShapeFacts +{ + uint8_t & packing; int64_t & value_bits; int64_t & base_lo; int64_t & base_hi; + float & qmin; float & qmax; float & qres; float & qdelta; uint32_t & qcount; + int64_t & min; int64_t & max; uint8_t & elem_kind; +}; + +inline bool TableMessageShapeRead( const uint8_t * in, int64_t size, int64_t & at, uint8_t kind, TableMessageShapeFacts f ); +inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t value_bits ); + +// TableMessageEntryRead parses ONE entry, and answers false for a HOSTILE +// SHAPE: bits above the kind's own domain, an array whose min exceeds its +// max, an element kind outside the closed set, a quantized triple SPEC.md +// calls non-conforming, or a shape running past the vocabulary's own bytes. +inline bool TableMessageEntryRead( const uint8_t * in, int64_t size, int64_t & at, TableMessageEntry & entry ) +{ + if ( at + 9 > size ) { return false; } + entry = TableMessageEntry(); + for ( int i = 0; i < 8; i++ ) { entry.id |= uint64_t( in[ at + i ] ) << ( 8 * i ); } + entry.kind = in[ at + 8 ]; + at += 9; + if ( !TableMessageKnownKind( entry.kind ) ) { return false; } + // The parse lands in LOCALS and the entry keeps what a decode reads: the + // quantized max and res are the derivation's inputs and never a field's. + uint8_t packing = 0, elem_kind = 0; + int64_t bits = 0, base_lo = 0, base_hi = 0, min = 0, max = 0; + float qmin = 0.0f, qmax = 0.0f, qres = 0.0f, qdelta = 0.0f; + uint32_t qcount = 0; + TableMessageShapeFacts own = { packing, bits, base_lo, base_hi, + qmin, qmax, qres, qdelta, qcount, + min, max, elem_kind }; + if ( !TableMessageShapeRead( in, size, at, entry.kind, own ) ) { return false; } + entry.packing = packing; + entry.value_bits = (int16_t) TableMessageValueBits( entry.kind, packing, bits ); + entry.base_lo = base_lo; + entry.base_hi = base_hi; + entry.qmin = qmin; + entry.qdelta = qdelta; + entry.qcount = qcount; + entry.min = min; + entry.max = max; + entry.elem_kind = elem_kind; + if ( entry.kind == 14 || entry.kind == 16 ) + { + uint8_t elem_packing = 0, inner_kind = 0; + int64_t elem_bits = 0, elem_base_lo = 0, elem_base_hi = 0, elem_min = 0, elem_max = 0; + float elem_qmin = 0.0f, elem_qmax = 0.0f, elem_qres = 0.0f, elem_qdelta = 0.0f; + uint32_t elem_qcount = 0; + TableMessageShapeFacts elem = { elem_packing, elem_bits, elem_base_lo, elem_base_hi, + elem_qmin, elem_qmax, elem_qres, elem_qdelta, elem_qcount, + elem_min, elem_max, inner_kind }; + if ( !TableMessageShapeRead( in, size, at, entry.elem_kind, elem ) ) { return false; } + entry.elem_packing = elem_packing; + entry.elem_value_bits = (int16_t) TableMessageValueBits( entry.elem_kind, elem_packing, elem_bits ); + entry.elem_base_lo = elem_base_lo; + entry.elem_base_hi = elem_base_hi; + entry.elem_qmin = elem_qmin; + entry.elem_qdelta = elem_qdelta; + entry.elem_qcount = elem_qcount; + entry.elem_max = elem_max; + } + return true; +} + +// TableMessageShapeRead is one shape, by the kind that names it (§3.3's shape +// table). Every number in it is a canonical LEB128 except where the row says +// otherwise: a RANGED BASE IS ENCODED BY ITS KIND'S SIGNEDNESS, zigzag for the +// signed kinds, unsigned for the unsigned kinds and sixteen bytes for the +// 128-bit and fixed-point kinds, and a QUANTIZED f32 carries min, max and res +// as float32, from which the step count and the width derive by SPEC.md +// §4.3's rule and by nothing else. +inline bool TableMessageShapeRead( const uint8_t * in, int64_t size, int64_t & at, uint8_t kind, TableMessageShapeFacts f ) +{ + uint64_t v = 0; + if ( TableMessageIntegerKind( kind ) || TableMessageFixedKind( kind ) || kind == 10 ) + { + if ( at >= size ) { return false; } + f.packing = in[ at++ ]; + if ( f.packing == 0 ) { return true; } + if ( f.packing == 1 && kind != 10 ) + { + if ( !TableMessageLeb( in, size, at, v ) || (int64_t) v > TableMessageKindBits( kind ) ) { return false; } + f.value_bits = (int64_t) v; + if ( kind == 18 || kind == 19 || TableMessageFixedKind( kind ) ) + { + if ( at + 16 > size ) { return false; } + uint64_t lo = 0, hi = 0; + for ( int i = 0; i < 8; i++ ) { lo |= uint64_t( in[ at + i ] ) << ( 8 * i ); } + for ( int i = 0; i < 8; i++ ) { hi |= uint64_t( in[ at + 8 + i ] ) << ( 8 * i ); } + f.base_lo = (int64_t) lo; f.base_hi = (int64_t) hi; + at += 16; + return true; + } + if ( !TableMessageLeb( in, size, at, v ) ) { return false; } + if ( kind >= 2 && kind <= 5 ) { f.base_lo = (int64_t) ( v >> 1 ) ^ -(int64_t) ( v & 1 ); } // zigzag + else { f.base_lo = (int64_t) v; } // the unsigned domain, whole + return true; + } + if ( f.packing == 2 && kind == 10 ) + { + if ( at + 12 > size ) { return false; } + uint32_t raw[3] = { 0, 0, 0 }; + for ( int k = 0; k < 3; k++ ) { for ( int i = 0; i < 4; i++ ) { raw[k] |= uint32_t( in[ at + 4 * k + i ] ) << ( 8 * i ); } } + at += 12; + memcpy( &f.qmin, &raw[0], 4 ); + memcpy( &f.qmax, &raw[1], 4 ); + memcpy( &f.qres, &raw[2], 4 ); + return TableMessageQuantization( f.qmin, f.qmax, f.qres, f.qdelta, f.qcount, f.value_bits ); + } + return false; // a packing outside the closed set + } + // A MAX ABOVE WHAT THE KIND CAN HOLD IS A HOSTILE WIDTH (§3.3). A string + // and a wide string are bounded by the int32 storage cap the checker + // applies to every N (SPEC §4.3, §6.1), and an array and a keyed entry by + // the 32-bit count an unbounded array announces (§2.9), which is the + // widest count this form spells. A larger bound is a shape no conforming + // declaration can produce, and a reader that carried it would do its + // length arithmetic in a range that overflows. + if ( kind == 12 || kind == 33 ) + { + if ( !TableMessageLeb( in, size, at, v ) || v > (uint64_t) INT32_MAX ) { return false; } + f.max = (int64_t) v; + return true; + } + if ( kind == 14 || kind == 16 ) + { + if ( kind == 14 ) + { + if ( !TableMessageLeb( in, size, at, v ) || v > kTableMessageListMax ) { return false; } + f.min = (int64_t) v; + } + if ( !TableMessageLeb( in, size, at, v ) || v > kTableMessageListMax ) { return false; } + if ( (int64_t) v < f.min ) { return false; } + f.max = (int64_t) v; + if ( at >= size ) { return false; } + f.elem_kind = in[ at++ ]; + if ( !TableMessageKnownKind( f.elem_kind ) ) { return false; } + // AND AN ELEMENT KIND OF 12 OR 33 IS REFUSED HERE, at the + // announcement, rather than at the skip that would meet it (§3.3): no + // declaration this language accepts is an array of string(N) or of + // wstring(N), so a shape announcing one is one rule's business and not + // two. + if ( f.elem_kind == 12 || f.elem_kind == 33 ) { return false; } + return true; + } + return true; +} + +// TableMessageValueBits is one value's width under a shape, and -1 where the +// kind's payload is not a fixed-width value at all. +inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t value_bits ) +{ + if ( kind == 1 ) { return 1; } + if ( kind == 11 ) { return 64; } + if ( kind == 10 ) { return packing == 2 ? value_bits : 32; } + if ( TableMessageIntegerKind( kind ) || TableMessageFixedKind( kind ) ) + { + if ( packing == 1 ) { return value_bits; } + switch ( kind ) + { + case 2: case 6: case 20: case 25: return 8; + case 3: case 7: case 21: case 26: return 16; + case 4: case 8: case 22: case 27: return 32; + case 5: case 9: case 23: case 28: return 64; + default: return 128; + } + } + return -1; +} + +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an +// ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under +// the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 +// over element kind 6, and a trailer of those two reserved ids. +// +// THE VOCABULARY IS A FIELD AND NOT THE TRAILER, and that buys three things: +// §3's writer rule that an id no body references is never written is restored +// unbroken, an entry can carry a KIND and a SHAPE which a trailer of bare ids +// cannot, and one NAME can appear at two shapes. +// +// The order is the COOK PROJECTION's (§20.2): each record in the order the +// projection renders it and each record's fields in the order the projection +// renders them, then each enum's variants and each union's arms. Then comes +// the tail the projection does not name: the reserved node-table id, the three +// blob type ids as bytes, string and wstring, and every table's own name id in +// the projection's sorted record order. The tail is UNCONDITIONAL, so an +// ordinary edit only ever grows it at its end and never moves a slot a +// generated field header carries as a literal. +static const int64_t kTableAnnounceBytes = 901; +static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, + 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, + 0xe4, 0x7c, 0x11, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, + 0x20, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0d, 0xec, 0x10, + 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, + 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, + 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, + 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, + 0x07, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x21, 0x06, + 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0c, 0x10, 0x38, 0x81, + 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, + 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, +}; + +// TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries +// an announcement carried, RESOLVED ONCE, under one numbering. +// +// THE RECEIVER RESOLVES ONCE (§3.3), so this holds the entries themselves and +// not the announcement's bytes: every entry is parsed at AnnounceRead, and +// every body after it dispatches through ONE ARRAY INDEX with nothing to +// re-read and nothing to decide. The announcement is free the moment +// AnnounceRead returns. +// +// THE STORAGE IS THE CALLER'S and this library never allocates. The caller +// declares an array of entries wherever it wants it, static, on a heap, in an +// arena or beside its connection, and hands it here with its CAPACITY. The +// announcement holds for the life of the connection (§3.3), so the array does +// too, and a peer holds TWO for a connection, the one it writes with and the +// one it reads with. A restart opens a fresh connection with an empty +// vocabulary and nothing is cached across connections. +// +// kTableMessageEntriesHere is the capacity a receiver that talks only to peers +// of THIS schema declares, and a receiver meeting other builds declares more. +struct TableVocabulary +{ + // THE CONFORMING DEFAULT BYTE BOUND (§3.3). The ENTRY bound has no default + // because it IS the caller's capacity: an announcement naming more entries + // than the caller made room for is refused as vocabulary_too_large before + // an entry is touched, and the byte bound is read off the vocabulary + // field's own length before that. + static const int64_t kDefaultMaxBytes = 64 * 1024; + + TableVocabulary( TableMessageEntry * storage, int64_t capacity ) + : entries( storage ), max_entries( capacity ) {} + + TableMessageEntry * entries; // THE CALLER'S, capacity max_entries + int64_t max_entries; + int64_t count = 0; + int64_t ref_bits = 0; + uint64_t build_version = 0; + bool announced = false; + // REFUSAL IS TERMINAL (§3.3): a connection whose first announcement was + // refused, for any reason, carries no vocabulary for its life, and every + // announcement after it is refused as second_announcement + bool refused = false; + int64_t max_bytes = kDefaultMaxBytes; +}; + +// TableVocabularyEntryAt is the entry a reference names, counted from 1: ONE +// ARRAY INDEX into the caller's resolved storage, no parse and no branch. +inline const TableMessageEntry & TableVocabularyEntryAt( const TableVocabulary & vocabulary, uint64_t slot ) +{ + return vocabulary.entries[ slot - 1 ]; +} + +// AnnounceRead reads an announcement into one direction's vocabulary (§3.3). +// +// The announcement IS a file, so every malformed rule of §3 already covers it. +// Over its body there are EXACTLY TWO STRICT CHECKS: the BUILD VERSION +// present, exactly once, under kind 9, eight bytes wide, and the VOCABULARY +// present, exactly once, under kind 14 over element kind 6. Everything else is +// ordinary and tolerant, so an unknown field is skipped and counted and the +// announcement can GAIN a field in a later minor without a lockstep redeploy. +// +// The FIRST announcement sets the vocabulary and it is the only one that can. +// A SECOND is refused by name: it does not replace it, does not amend it and +// changes nothing. A refused announcement sets NO VOCABULARY, and the refusal +// is TERMINAL: every announcement after it, whether or not the first set +// anything, is second_announcement, so a peer holds no retry on the +// connection and cannot buy a second resolve by having its first refused. +inline bool AnnounceReadOnce( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * to ); + +inline bool AnnounceRead( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * to = report != NULL ? report : &ignored; + if ( vocabulary.announced || vocabulary.refused ) + { + to->refused = true; + to->reason = second_announcement; + return false; + } + const bool set = AnnounceReadOnce( vocabulary, buffer, bytes, to ); + if ( !set ) { vocabulary.refused = true; } + return set; +} + +inline bool AnnounceReadOnce( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * to ) +{ + if ( bytes < 1 ) { to->malformed = true; return false; } + if ( buffer[0] != kTableWireForm ) + { + to->refused = true; + to->reason = buffer[0] == kTableWireMessageForm ? message_form_as_file : newer_form; + return false; + } + if ( bytes < 9 ) { to->malformed = true; return false; } + TableIdTable table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( buffer, bytes, table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { to->malformed = true; } + else { to->refused = true; to->reason = newer_form; } + return false; + } + if ( TableBodyEndsEarly( buffer + 1, body_bytes, table ) ) { to->malformed = true; return false; } + TableReader r( buffer + 1, body_bytes, to, &table ); + uint64_t version = 0; + const uint8_t * words = NULL; + int64_t words_bytes = 0; + int32_t seen_version = 0, seen_vocabulary = 0; + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.getleb( ref ) ) { to->malformed = true; return false; } + if ( ref == 0 ) { break; } + if ( ref > (uint64_t) table.count || !r.has( 1 ) ) { to->malformed = true; return false; } + const uint64_t id = table.at( ref ); + const uint8_t kind = r.get8(); + if ( id == kTableBuildVersionFieldId ) + { + if ( kind != 9 || !r.has( 8 ) ) { to->malformed = true; return false; } + version = r.get64(); + // THE BUILD VERSION IS KEPT THE MOMENT IT IS READ, refusal or not, so + // that a refusal on this connection NAMES IT (§3.3). It is not the + // vocabulary, and a refused announcement still sets none. + vocabulary.build_version = version; + seen_version++; + continue; + } + if ( id == kTableMessageVocabularyFieldId ) + { + // kind 14 over element kind 6, which is §3's spelling for an + // opaque run of bytes + uint64_t framed = 0; + if ( kind != 14 || !r.getleb( framed ) || !r.has( (int64_t) framed ) ) { to->malformed = true; return false; } + const int64_t begin = r.offset, end = r.offset + (int64_t) framed; + r.offset = end; + if ( begin >= end || r.buffer[ begin ] != 6 ) { to->malformed = true; return false; } + int64_t at = begin + 1; + uint64_t length = 0; + if ( !TableMessageLeb( r.buffer, end, at, length ) || at + (int64_t) length != end ) { to->malformed = true; return false; } + if ( (int64_t) length > vocabulary.max_bytes ) { to->refused = true; to->reason = vocabulary_too_large; return false; } + words = r.buffer + at; + words_bytes = (int64_t) length; + seen_vocabulary++; + continue; + } + to->unknown++; + if ( !r.skip( kind ) ) { to->malformed = true; return false; } + } + if ( seen_version != 1 || seen_vocabulary != 1 ) { to->malformed = true; return false; } + + // THE ENTRIES, RESOLVED ONCE into the caller's storage (§3.3): every width + // is checked here and never again, and no body after this parses a byte of + // an announcement. An entry count above the caller's CAPACITY is refused + // by name before the entry is touched. + int64_t at = 0, count = 0, node_table_slots = 0; + while ( at < words_bytes ) + { + if ( count >= vocabulary.max_entries ) { to->refused = true; to->reason = vocabulary_too_large; return false; } + TableMessageEntry & parsed = vocabulary.entries[ count ]; + if ( !TableMessageEntryRead( words, words_bytes, at, parsed ) ) { to->malformed = true; return false; } + // THE RESERVED IDS WHERE THEY DO NOT BELONG (§3.3): the announcement's + // own two never take a slot, and the node-table id takes exactly one, + // so a vocabulary carrying either of the first or a SECOND node-table + // id is malformed whole and sets nothing + if ( parsed.id == kTableBuildVersionFieldId || parsed.id == kTableMessageVocabularyFieldId ) { to->malformed = true; return false; } + if ( parsed.id == kTableNodeTableFieldId ) { if ( node_table_slots++ > 0 ) { to->malformed = true; return false; } } + // A TRIPLE ALREADY PLACED IS NEVER PLACED TWICE, so two entries that + // agree on the id, the kind and every fact of the shape are malformed + // (§3.3): no writer this wire has produces one, and a reader that took + // it would carry two slots naming one thing. The scan is quadratic in + // the entry count, and the entry count is bounded above at 4096, so it + // is at most eight million compares on a path that runs ONCE a + // connection and never again. + for ( int64_t seen = 0; seen < count; seen++ ) + { + const TableMessageEntry & other = vocabulary.entries[ seen ]; + if ( other.id == parsed.id && other.kind == parsed.kind && TableMessageEntrySame( other, parsed ) ) { to->malformed = true; return false; } + } + count++; + } + vocabulary.count = count; + vocabulary.ref_bits = TableBitsRequired( 0, count ); + vocabulary.build_version = version; + vocabulary.announced = true; + return true; +} + +// TableMessageReserved is one of the three ids the language holds back (§3.1, +// §3.3, §5): each is malformed anywhere but its own transport, and the rule +// OUTRANKS the wrong-sort rule below. +inline bool TableMessageReserved( uint64_t id ) +{ + // THE THREE ARE THE TOP THREE VALUES a uint64 holds, so the test is ONE + // comparison: 0xFFFFFFFFFFFFFFFD, FE and FF and nothing else is at or + // above the vocabulary's own id, and a declaration hashing to any of them + // is refused by name (§11) + return id >= kTableMessageVocabularyFieldId; +} + +// TableMessageNameEntry resolves a reference used as a VALUE, which is an +// enum's variant, a keyed array's slot key or a node record's type id, and +// which must name a kind-0 entry (§3.3). A reference of 0 where an entry is required, one +// above E, one naming a reserved id and one naming an entry that carries a +// payload are each damage: the reader RESOLVED the entry and it contradicts +// the position it was used in, so the next bit's meaning is what is in doubt. +inline bool TableMessageNameEntry( const TableVocabulary & vocabulary, uint64_t ref, TableMessageEntry & entry ) +{ + if ( ref == 0 || ref > (uint64_t) vocabulary.count ) { return false; } + entry = TableVocabularyEntryAt( vocabulary, ref ); + return !TableMessageReserved( entry.id ) && entry.kind == 0; +} + +// TableMessageArmEntry resolves a UNION's arm reference, which must name an +// entry carrying the arm's own kind and shape: a kind-0 entry frames nothing, +// and a reserved id belongs to no arm (§3.3). +inline bool TableMessageArmEntry( const TableVocabulary & vocabulary, uint64_t ref, TableMessageEntry & entry ) +{ + if ( ref == 0 || ref > (uint64_t) vocabulary.count ) { return false; } + entry = TableVocabularyEntryAt( vocabulary, ref ); + return !TableMessageReserved( entry.id ) && entry.kind != 0; +} + +// TableMessageSkipVariant steps over an ENUM's variant reference on a SKIP +// path and RESOLVES it while it is there: 0 is None and the whole payload, and +// every other reference must name a kind-0 entry, because every reference +// above E is damage and one naming an entry that carries a payload +// contradicts the position it was used in, whether or not this reader was +// going to keep the value (§3.3). +inline bool TableMessageSkipVariant( TableBitReader & r, const TableVocabulary & vocabulary ) +{ + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + TableMessageEntry named; + return TableMessageNameEntry( vocabulary, ref, named ); +} +// TableMessageSkip steps over one field's payload without decoding it, using +// the announced ENTRY alone (§3.3). It is what makes an unknown entry +// skippable on a body with no kind byte, and it is ONE function over every +// table, because a shape says everything a skipper needs. +inline bool TableMessageSkipBody( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits ); +inline bool TableMessageSkip( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ); + +// TableMessageSkipElement steps over ONE element of an array or keyed entry +// by the element's own announced shape: a nested body to its zero reference, +// a variant or a node index at its reference width, a union arm by its own +// entry, and a fixed-width value at its bits. +inline bool TableMessageSkipElement( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ) +{ + switch ( entry.elem_kind ) + { + case 13: return TableMessageSkipBody( r, vocabulary, index_bits ); + case 30: return TableMessageSkipVariant( r, vocabulary ); + case 17: return index_bits > 0 && r.skip( index_bits ); + case 15: + { + TableMessageEntry inner; + inner.kind = 15; + return TableMessageSkip( r, vocabulary, index_bits, inner ); + } + default: + { + const int64_t elem = entry.elem_value_bits; + return elem >= 0 && r.skip( elem ); + } + } +} + +// TableMessageElementRunBits is the bits ONE element of an array or a keyed +// entry occupies on the SKIP path, where nothing is resolved and a run of them +// is one multiplication, and -1 where the element's width is its own +// content's. A ZERO is a real answer, and it is why this exists: a ranged +// element whose min equals its max rides no bits at all (§3.3). +inline int64_t TableMessageElementRunBits( const TableVocabulary & vocabulary, const TableMessageEntry & entry ) +{ + int64_t elem = 0; + switch ( entry.elem_kind ) + { + // a nested body, a union arm, an enum's variant and a node index each + // RESOLVE something, and a resolve that contradicts its position is + // damage this reader must still find, so they are walked + case 13: case 15: case 30: case 17: return -1; + default: elem = entry.elem_value_bits; break; + } + if ( elem < 0 ) { return -1; } + if ( entry.kind == 16 ) { elem += vocabulary.ref_bits; } // a keyed slot's own key reference + return elem; +} + +// TableMessageSkipRun steps over n elements of one fixed width in a single +// arithmetic step. A FIXED-WIDTH ELEMENT IS ARITHMETIC (§3.3), and a loop here +// would be the one superlinear thing in this form: a zero-width element under +// a count of 2^31 is six bytes of wire. +inline bool TableMessageSkipRun( TableBitReader & r, uint64_t n, int64_t width ) +{ + if ( width < 0 ) { return false; } + if ( width == 0 ) { return true; } + if ( n > (uint64_t) ( INT64_MAX / width ) ) { return false; } + return r.skip( (int64_t) ( n * (uint64_t) width ) ); +} + +inline bool TableMessageSkip( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ) +{ + switch ( entry.kind ) + { + case 0: case 32: return true; // a name, and a payload-free arm + case 30: return TableMessageSkipVariant( r, vocabulary ); + case 13: return TableMessageSkipBody( r, vocabulary, index_bits ); + case 17: return index_bits > 0 && r.skip( index_bits ); // a node index, at the width the body's node count settled + case 15: + { + uint64_t arm = 0; + if ( !r.get( arm, vocabulary.ref_bits ) ) { return false; } + if ( arm == 0 ) { return true; } + TableMessageEntry arm_entry; + if ( !TableMessageArmEntry( vocabulary, arm, arm_entry ) ) { return false; } + return TableMessageSkip( r, vocabulary, index_bits, arm_entry ); + } + case 12: + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() ) { return false; } + return r.skip( (int64_t) n * 8 ); + } + case 33: + { + // the length, NO align, then SIXTEEN bits a code unit (§3.3) + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + return r.skip( (int64_t) n * 16 ); + } + case 31: + { + // THE ESCAPE: align, a thirty-two bit L, then L bytes, opaque. It is + // the one path a later-major writer has on this form (§3.3) + uint64_t n = 0; + if ( !r.align() || !r.get( n, 32 ) ) { return false; } + return r.skip( (int64_t) n * 8 ); + } + case 14: case 16: + { + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, width ) ) { return false; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !r.align() ) { return false; } + // A RUN OF FIXED-WIDTH ELEMENTS IS ONE MULTIPLICATION (§3.3), and + // only an element whose width is its own content's is walked + const int64_t run = TableMessageElementRunBits( vocabulary, entry ); + if ( run >= 0 ) { return TableMessageSkipRun( r, n, run ); } + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 && !r.skip( vocabulary.ref_bits ) ) { return false; } + if ( !TableMessageSkipElement( r, vocabulary, index_bits, entry ) ) { return false; } + } + return true; + } + } + const int64_t width = entry.value_bits; + return width >= 0 && r.skip( width ); +} + +inline bool TableMessageSkipBody( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + if ( !TableMessageSkip( r, vocabulary, index_bits, TableVocabularyEntryAt( vocabulary, ref ) ) ) { return false; } + } +} + +// TableMessageNodeTableOpen reads the node table's opening when a body has +// one: the reserved id's reference and the count at thirty-two raw bits. A +// body whose first reference is anything else has no node table, and the +// reader is left where it was. False is damage: a reference past E, or bits +// that run out. +inline bool TableMessageNodeTableOpen( TableBitReader & r, const TableVocabulary & vocabulary, int64_t & count ) +{ + count = 0; + const int64_t at = r.offset; + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { r.offset = at; return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + if ( TableVocabularyEntryAt( vocabulary, ref ).id != kTableNodeTableFieldId ) { r.offset = at; return true; } + uint64_t n = 0; + if ( !r.get( n, 32 ) ) { return false; } + count = (int64_t) n; + return true; +} +// AnnounceMeasure is the announcement's byte count, which is a constant of the +// unit and not a walk. +inline int64_t AnnounceMeasure() { return kTableAnnounceBytes; } + +// Announce writes the announcement into the caller's buffer and answers the +// bytes written, which is exactly AnnounceMeasure's answer, or -1 when the +// buffer is too small. It allocates nothing and walks nothing. +inline int64_t Announce( uint8_t * buffer, int64_t capacity ) +{ + if ( buffer == NULL || capacity < kTableAnnounceBytes ) { return -1; } + memcpy( buffer, kTableAnnounce, (size_t) kTableAnnounceBytes ); + return kTableAnnounceBytes; +} + +// THE PRIMITIVE IS A BATCH (§3.3): a number of bodies of ONE ROOT in one +// buffer, one count and one continuous bit stream with no alignment between +// them. A single message is the batch of one. +// +// The count rides ahead of the bodies, so a writer declares it at Begin and +// End refuses a batch that wrote a different number: a count the bodies do not +// match is not a wire this writer will hand anyone. +struct TableMessageBatch +{ + TableBitWriter w; + int64_t declared = 0; + int64_t written = 0; +}; + +inline bool TableMessageBatchBegin( TableMessageBatch & batch, uint8_t * buffer, int64_t capacity, int64_t bodies ) +{ + if ( buffer == NULL || capacity < 1 || bodies < 1 || bodies > kTableMessageBatchMax ) { return false; } + buffer[0] = kTableWireMessageForm; // the FORM BYTE is read first, always + batch.w = TableBitWriter( buffer + 1, capacity - 1 ); + batch.declared = bodies; + batch.written = 0; + batch.w.put( (uint64_t) ( bodies - 1 ), 8 ); // a ranged integer over [1, 256] + return true; +} + +// TableMessageBatchEnd zero-fills to the next byte, the one alignment a batch +// spends at its end, and answers the whole batch's byte count, or -1. +inline int64_t TableMessageBatchEnd( TableMessageBatch & batch ) +{ + if ( batch.written != batch.declared || batch.w.overflow ) { return -1; } + batch.w.align(); + if ( batch.w.overflow ) { return -1; } + return 1 + batch.w.bits / 8; +} + +// TableMessageBatchBytes is a batch's byte count from its bodies' BIT count, +// which is what every MeasureMessages answers. +inline int64_t TableMessageBatchBytes( int64_t body_bits ) +{ + if ( body_bits < 0 ) { return -1; } + return 1 + ( 8 + body_bits + 7 ) / 8; +} + +// The reading half. A batch is opened once and its bodies are then read in +// order into the storage the caller sized for them: which root a batch carries +// is the APPLICATION's and never this wire's. +struct TableMessageBatchReader +{ + TableBitReader r; + const TableVocabulary * vocabulary = NULL; + TableReport * report = NULL; + int64_t remaining = 0; + // THE SINK A CALLER THAT PASSED NO REPORT WRITES INTO IS THE READER'S OWN, + // not a static: a static is shared mutable state, and two threads reading + // two batches without reports would be writing one object. LoadMessages + // already keeps its sink locally, for the same reason. + TableReport ignored; +}; + +// TableMessageBatchOpen answers the batch's body count, or -1 with the refusal +// on the report: a form byte this reader does not carry, or a body from a peer +// that never announced. +inline int64_t TableMessageBatchOpen( TableMessageBatchReader & br, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + br.report = report != NULL ? report : &br.ignored; + br.vocabulary = &vocabulary; + if ( bytes < 1 ) { br.report->malformed = true; return -1; } + if ( buffer[0] != kTableWireMessageForm ) { br.report->refused = true; br.report->reason = newer_form; return -1; } + if ( !vocabulary.announced ) { br.report->refused = true; br.report->reason = no_vocabulary; return -1; } + br.r = TableBitReader( buffer + 1, bytes - 1 ); + uint64_t count = 0; + if ( !br.r.get( count, 8 ) ) { br.report->malformed = true; return -1; } + br.remaining = (int64_t) count + 1; + return br.remaining; +} + +// TableMessageRefuseBatch is the batch's own refusal (§3.3): M above 256 on the +// write side, or above the caller's capacity on the read side. Nothing is +// written or decoded, no counter moves, and the reason names it. +inline void TableMessageRefuseBatch( TableReport * report ) +{ + if ( report == NULL ) { return; } + report->refused = true; + report->reason = batch_too_large; +} + +// TableMessageBatchClose verifies the trailing pad, and that NOTHING FOLLOWS +// IT: the batch ends at the pad to the byte boundary, and a buffer with bytes +// left over describes no batch this reader can name (§3.3). +inline bool TableMessageBatchClose( TableMessageBatchReader & br ) +{ + if ( br.remaining != 0 || !br.r.align() || br.r.offset != br.r.bits ) { br.report->malformed = true; return false; } + return true; +} + + +// An ENUM-KEYED array's storage: E.Max slots, ONE PER NAMED VARIANT, with the +// key k at index k-1 — the storage SHIFTS LEFT and nothing is stored for None. +// +// NOTHING OUTSIDE THE ARRAY NAMES ITS SIZE: the extent is derived from E::Max +// here and nowhere else, so there is no size parameter to spell and no count a +// consumer could put one out of step with. +// +// NONE IS THE NULL KEY: it names no slot, it never rides on the wire, a stored +// key of 0 is malformed, and INDEXING BY IT IS A PROGRAM ERROR IN EVERY +// CONFIGURATION — caught by operator[], which cannot see a runtime key any +// earlier, and REFUSED UNCONDITIONALLY. A KEY PAST Max IS THE SAME ERROR for +// the same reason — it names a variant this enum does not have — so the +// accessor refuses BOTH ENDS. NDEBUG does not remove the compare: +// there is NO UB PATH here in any build. ITERATION is still the surface a +// consumer of the whole array wants: begin()/end() walk every stored slot and +// yield the KEY, 1..E.Max, so a call site writes no bound, no cast, no shift +// and no None question. +template +struct TableKeyed +{ + // the extent is the enum's, derived here and named nowhere else + static constexpr int32_t kSlots = (int32_t) E::Max; + + T slots[kSlots] = {}; + + T & operator[]( E key ) + { + RefuseKey( key ); + return slots[ (int32_t) key - 1 ]; + } + const T & operator[]( E key ) const + { + RefuseKey( key ); + return slots[ (int32_t) key - 1 ]; + } + + // THE REFUSAL, and it stands in EVERY BUILD, AT BOTH ENDS. The storage + // holds one slot per NAMED variant: nothing for None below it and nothing + // above Max, so a build that skipped this compare would index one element + // BEFORE the array or past its end — undefined behavior in the + // configuration a game ships. Either key is a program error, so the + // accessor ends the program rather than reading something. The assert + // carries the message where a debugger can read it and NDEBUG removes + // that; the fatal is what stands after it. BOTH GO THROUGH THE HOOKS — + // define schema_assert and schema_fatal and this refusal lands in your + // own handler. + // + // ONE UNSIGNED COMPARE COVERS BOTH ENDS: the storage index is key - 1, and + // None's is -1, which wraps above kSlots unsigned. The cost is one + // perfectly-predicted compare, on a path that reads config. + static void RefuseKey( E key ) + { + if ( (uint32_t) ( (int32_t) key - 1 ) >= (uint32_t) kSlots ) + { + schema_assert( false && "an enum-keyed array holds one slot per named variant: None keys none, and neither does a key past Max" ); + schema_fatal(); + } + } + + // ---- iteration: keys 1..E.Max over storage 0..E.Max-1, key beside element ---- + // + // The entry is a key and a REFERENCE, handed out BY VALUE the way any + // proxy is: for ( auto [ key, element ] : keyed ) binds element to the + // reference member, so iterating fills the array as well as reads it. + // auto & [ key, element ] does NOT compile, and that is by design — a + // non-const lvalue reference cannot bind to the proxy. Write + // auto [ ... ], or auto && [ ... ] if you prefer the reference form. + // + // THE ITERATORS CARRY NO iterator_traits TYPEDEFS. They bought std::distance + // and the forward-pass algorithms for an audience that does not call them, + // and the they need is the single most expensive include the + // generated corpus had: 536 headers and 986 KB, in a header whose whole + // remaining set is 123. begin(), end() and size() need none of it. + + struct Entry { E key; T & element; }; + struct ConstEntry { E key; const T & element; }; + + struct Iterator + { + T * slots; + int32_t index; // the STORAGE index; the key it holds is index + 1 + Entry operator*() const { return Entry{ (E) ( index + 1 ), slots[index] }; } + Iterator & operator++() { index++; return *this; } + bool operator==( const Iterator & other ) const { return index == other.index; } + bool operator!=( const Iterator & other ) const { return index != other.index; } + }; + + struct ConstIterator + { + const T * slots; + int32_t index; // the STORAGE index; the key it holds is index + 1 + ConstEntry operator*() const { return ConstEntry{ (E) ( index + 1 ), slots[index] }; } + ConstIterator & operator++() { index++; return *this; } + bool operator==( const ConstIterator & other ) const { return index == other.index; } + bool operator!=( const ConstIterator & other ) const { return index != other.index; } + }; + + Iterator begin() { return Iterator{ slots, 0 }; } + Iterator end() { return Iterator{ slots, kSlots }; } + ConstIterator begin() const { return ConstIterator{ slots, 0 }; } + ConstIterator end() const { return ConstIterator{ slots, kSlots }; } +}; + +inline float table_bits_to_float( uint32_t bits ) { float f; memcpy( &f, &bits, 4 ); return f; } +inline uint32_t table_float_to_bits( float f ) { uint32_t b; memcpy( &b, &f, 4 ); return b; } +inline double table_bits_to_double( uint64_t bits ) { double d; memcpy( &d, &bits, 8 ); return d; } +inline uint64_t table_double_to_bits( double d ) { uint64_t b; memcpy( &b, &d, 8 ); return b; } + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_PRIMITIVES + +#ifndef MAPDEMO_SCHEMA_TABLE_ARENA +#define MAPDEMO_SCHEMA_TABLE_ARENA + +namespace mapdemo { + +// ---- variable-length tables: tuning constants (docs/SPEC-TABLES.md) ---- +// +// The segment size and the count multiply to exactly 2^32: the u32 reference +// is the arena's hard ceiling, and these constants saturate it rather than +// leaving address space unreachable. Slab handout costs one atomic per slab, +// so per-node allocation costs no synchronization at all. + +static const uint32_t kTableSegmentBits = 22; // 4 MiB segments +static const uint32_t kTableSegmentSize = 1u << kTableSegmentBits; +static const uint32_t kTableSegmentMask = kTableSegmentSize - 1u; +static const uint32_t kTableMaxSegments = 1u << ( 32 - kTableSegmentBits ); // 1024 -> 4 GiB +static const uint32_t kTableSlabBytes = 64u * 1024u; // one atomic per slab +static const uint32_t kTableAlign = 8; // every node starts 8-aligned +static const uint32_t kTableAllocFailed = 0xFFFFFFFFu; + +// ---- THE CALLER'S ALLOCATOR (docs/SPEC-TABLES.md §6.5) ---- +// +// Every allocation the variable-length runtime makes goes through one of +// these — the arena's segments, the pack walk's identity map, the numbering's +// entry array, the packed region, and the tool path's node directory. There is +// no other call to the C library on this path, so a counting allocator sees +// every byte and a game's own heap can own all of it. +// +// It is the shape TableBlockAllocator already has (§19.1): two function +// pointers and a context the caller carries. What it adds is a CONTRACT ON +// alloc — the bytes come back ZEROED. Lock copies whole nodes, PADDING +// INCLUDED, so anything left uninitialized reaches a packed region; the default +// pair reaches that through calloc, which costs nothing measurable because a +// fresh segment is untouched pages either way. +struct TableAllocator +{ + void * ( *alloc )( void * context, int64_t bytes ); // ZEROED bytes, NULL on failure + void ( *free )( void * context, void * pointer ); + void * context; +}; + +// The default pair, and it is the one every entry point takes when the caller +// names none. It calls schema_allocate / schema_release, so a program with its +// own C-library replacement can move the floor without writing a struct at all. +inline void * table_default_alloc( void * context, int64_t bytes ) { (void) context; return schema_allocate( bytes ); } +inline void table_default_free( void * context, void * pointer ) { (void) context; schema_release( pointer ); } + +inline TableAllocator TableDefaultAllocator() +{ + TableAllocator allocator; + allocator.alloc = table_default_alloc; + allocator.free = table_default_free; + allocator.context = NULL; + return allocator; +} + +// ---- TableRef: a relocatable reference (never a machine pointer) ---- +// +// Two encodings, one slot, and the FORM says which is in force: +// +// in the arena — the node's arena offset (segment index in the high bits) +// in a region — the SELF-RELATIVE byte delta from this slot's own address, +// so a deref is one add, needs no base pointer, and a whole +// region relocates by memcpy with zero fix-up +// +// 0 is null in both, and a slot can never name the node that contains it, so +// zero names nothing real in either form. +// +// A REGION DELTA HAS NO REQUIRED SIGN (§6.3). A region is packed depth-first, +// so a node's FIRST reference points forward; every LATER reference to that +// same node points BACK at the one body it already has, which is exactly what +// makes one node one node in a region. Sharing and a back-reference are the +// same fact, and nothing validates a reference by its sign. +// +// IT IS EIGHT BYTES, SIGNED, so ONE REGION REACHES EVERYTHING (§6.3, §7): a +// four-byte slot bounded a region at 2 GiB, and the scale a cook exists for is +// *"100mbs or many gigabytes of data in Assets.bin"*. +struct TableRef +{ + int64_t value = 0; + bool null() const { return value == 0; } +}; + +// TableSlot is what Alloc hands back: usable as the node pointer (write +// fields through it) AND as the reference to store in a pointer field. +template struct TableSlot +{ + T * ptr = NULL; + TableRef ref; + T * operator->() const { return ptr; } + T & operator*() const { return *ptr; } + operator T *() const { return ptr; } + operator TableRef() const { return ref; } + bool null() const { return ptr == NULL; } +}; + +inline uint32_t TableAlignUp( uint32_t bytes ) { return ( bytes + kTableAlign - 1 ) & ~( kTableAlign - 1 ); } +inline int64_t TableAlignUp64( int64_t bytes ) { return ( bytes + kTableAlign - 1 ) & ~( int64_t( kTableAlign ) - 1 ); } + +// ---- a BYTE BUFFER's node (docs/SPEC-TABLES.md §2.5, §6.3) ---- +// +// A *bytes or *string slot is a TableRef like every pointer slot, and it names +// a BLOB NODE: this eight-byte header and then the bytes, at offset eight so +// the data is eight-aligned. A *string blob carries one more zero byte after +// its data, so a region hands back a C string with no copy. The node's extent +// is the header plus its bytes, rounded to the arena's alignment like every +// node's; on the wire it is a record whose body is the bytes (§3.1). +struct TableBlob +{ + uint32_t length; + uint32_t zero; +}; + +static const int64_t kTableBlobHeader = 8; // length (u32), then four zero bytes +static const int64_t kTableBlobMaxLength = 0xFFFFFFFF; // a record's length is a u32 (§3.1) + +// the node's storage: the header, the bytes, a string's terminator, rounded +// to the arena's alignment like every node +inline int64_t TableBlobStorage( int64_t length, bool terminated ) +{ + return TableAlignUp64( kTableBlobHeader + length + ( terminated ? 1 : 0 ) ); +} + +// What a read answers: a pointer INTO the region and the length, NULL and +// zero for a null slot. Off a locked region, a loaded one or an opened cook +// the pointer is one add from the slot, and nothing is copied. +struct TableBytesView +{ + const uint8_t * data; + int64_t length; +}; + +struct TableStringView +{ + const char * data; // zero-terminated + int64_t length; +}; + +// What AllocBytes and AllocString hand back: the bytes to write through, the +// length asked for, and the reference to store in the slot — the three +// answers TableSlot gives for a table node. +struct TableBytesSlot +{ + uint8_t * data = NULL; + int64_t length = 0; + TableRef ref; + bool null() const { return data == NULL; } + operator TableRef() const { return ref; } +}; + +struct TableStringSlot +{ + char * data = NULL; // room for length bytes and the terminator, already zero + int64_t length = 0; + TableRef ref; + bool null() const { return data == NULL; } + operator TableRef() const { return ref; } +}; + +// ---- the arena: segmented, slab-handed, lock-free by ownership ---- +// +// Allocation is thread-local inside a worker's slab — no atomics on the node +// path. A worker takes its next slab with ONE compare-exchange, and a new +// segment is published with one more. Nothing ever moves: a segment, once +// allocated, lives untouched until the arena is torn down, so a T* obtained +// from Alloc stays valid while other workers allocate, and an offset stays +// correct while the arena grows. +// +// The model this DELIBERATELY refuses: one buffer under a lock, grown by +// realloc. A realloc moves the buffer under workers mid-write; offsets fix +// identity but not the raw references already resolved from them, and the +// resulting corruption is invisible until much later. Segments never move, so +// that bug class cannot be written here. +// +// Slack: at most one slab tail per worker plus one slab per segment (a slab +// that will not fit is skipped rather than split), i.e. under 2% of a segment +// plus threads x 64 KiB. That is the price of never synchronizing per node. +struct TableArena +{ + std::atomic segments[ kTableMaxSegments ]; + std::atomic cursor; // (segment << kTableSegmentBits) | bytes handed out + bool locked = false; // MONOTONIC: Lock() is one-way, there is no unlock + // THE ARENA CARRIES ITS OWN, so everything downstream of a builder — + // segments, pack map, numbering, region, node directory — allocates through + // the one pair the caller named, with nothing to thread by hand. + TableAllocator allocator; +}; + +inline void TableArenaInit( TableArena & arena, TableAllocator allocator ) +{ + for ( uint32_t i = 0; i < kTableMaxSegments; i++ ) + { + arena.segments[i].store( NULL, std::memory_order_relaxed ); + } + arena.cursor.store( 0, std::memory_order_relaxed ); + arena.locked = false; + arena.allocator = allocator; +} + +inline void TableArenaShutdown( TableArena & arena ) +{ + for ( uint32_t i = 0; i < kTableMaxSegments; i++ ) + { + uint8_t * segment = arena.segments[i].exchange( NULL, std::memory_order_acq_rel ); + if ( segment != NULL ) { arena.allocator.free( arena.allocator.context, segment ); } + } + arena.cursor.store( 0, std::memory_order_relaxed ); +} + +// one L1 load plus an add: the segment table is 8 KiB and stays hot +inline uint8_t * TableArenaAt( const TableArena & arena, uint32_t offset ) +{ + return arena.segments[ offset >> kTableSegmentBits ].load( std::memory_order_relaxed ) + ( offset & kTableSegmentMask ); +} + +// TableArenaGrabSlab hands one worker its next private slab. Returns +// kTableAllocFailed when the arena's address space or the allocator is +// exhausted — a loud refusal, never a silent smaller slab. +inline uint32_t TableArenaGrabSlab( TableArena & arena ) +{ + for ( ;; ) + { + uint32_t cursor = arena.cursor.load( std::memory_order_acquire ); + uint32_t segment = cursor >> kTableSegmentBits; + uint32_t used = cursor & kTableSegmentMask; + // strictly less: a slab is never split across segments, and the tail + // is the documented slack + if ( used + kTableSlabBytes < kTableSegmentSize ) + { + if ( arena.segments[segment].load( std::memory_order_acquire ) == NULL ) + { + // THE SEGMENT COMES BACK ZEROED, which is the allocator's + // contract and not an extra pass here: Lock copies whole nodes, + // PADDING INCLUDED, so anything uninitialized reaches a packed + // region. Value-initializing a node with placement new zeroes + // its MEMBERS and not its padding, so the zeroing has to happen + // at the segment or not at all. It costs nothing measurable: a + // fresh segment is untouched pages either way, and the default + // pair's calloc has the kernel hand them over zeroed. + uint8_t * memory = (uint8_t *) arena.allocator.alloc( arena.allocator.context, (int64_t) kTableSegmentSize ); + if ( memory == NULL ) { return kTableAllocFailed; } + uint8_t * expected = NULL; + if ( !arena.segments[segment].compare_exchange_strong( expected, memory, std::memory_order_acq_rel ) ) + { + // another worker published this segment first + arena.allocator.free( arena.allocator.context, memory ); + } + } + if ( arena.cursor.compare_exchange_weak( cursor, cursor + kTableSlabBytes, std::memory_order_acq_rel ) ) + { + return ( segment << kTableSegmentBits ) | used; + } + continue; + } + uint32_t next_segment = segment + 1; + if ( next_segment >= kTableMaxSegments ) { return kTableAllocFailed; } // 4 GiB: the u32 reference's ceiling + arena.cursor.compare_exchange_weak( cursor, next_segment << kTableSegmentBits, std::memory_order_acq_rel ); + } +} + +// TableArenaGrabSpan reserves a SPAN of the arena's address space for one node +// larger than a slab — a BYTE BUFFER of any size (docs/SPEC-TABLES.md §2.5) — +// and allocates it as one contiguous block. It takes whole segment indices +// from the cursor, starting at the index after the cursor's so nothing else +// is ever handed out inside the span, and publishes the block under the first +// of them; the indices the span covers past that one stay NULL, which is +// enough, because only a node's START is ever resolved through the segment +// table and a blob's bytes follow its header inside the one allocation. The +// unused tail of the segment the cursor was in is slack, like a slab tail. +// Returns kTableAllocFailed when the address space or the allocator is +// exhausted — a loud refusal, never a smaller blob. +inline uint32_t TableArenaGrabSpan( TableArena & arena, int64_t bytes ) +{ + if ( bytes <= 0 || bytes > ( (int64_t) kTableMaxSegments - 2 ) * (int64_t) kTableSegmentSize ) { return kTableAllocFailed; } + const uint32_t spanned = (uint32_t) ( ( bytes + kTableSegmentSize - 1 ) >> kTableSegmentBits ); + for ( ;; ) + { + uint32_t cursor = arena.cursor.load( std::memory_order_acquire ); + uint32_t start = ( cursor >> kTableSegmentBits ) + 1; + if ( start + spanned >= kTableMaxSegments ) { return kTableAllocFailed; } // 4 GiB: the u32 reference's ceiling + uint32_t next = ( start + spanned ) << kTableSegmentBits; + if ( !arena.cursor.compare_exchange_weak( cursor, next, std::memory_order_acq_rel ) ) { continue; } + // the span is this worker's now: nothing else can publish under its + // first index, so a plain store suffices, and the block comes back + // ZEROED like every segment — the blob's bytes and its tail are zeros + // until written + uint8_t * memory = (uint8_t *) arena.allocator.alloc( arena.allocator.context, bytes ); + if ( memory == NULL ) { return kTableAllocFailed; } + arena.segments[start].store( memory, std::memory_order_release ); + return start << kTableSegmentBits; + } +} + +// ---- TableWorker: one thread's allocation front ---- +// +// The threading contract, stated plainly: +// * Alloc on YOUR OWN worker is safe concurrently with any other worker's. +// No locks, no atomics per node. +// * Writing fields of a node ANOTHER worker allocated is your own +// synchronization problem — this runtime does not arbitrate it. +// * Lock and Save are single-threaded: call them after the workers have +// joined. +struct TableWorker +{ + TableArena * arena = NULL; + uint32_t next = 0; + uint32_t end = 0; + + template TableSlot Alloc() + { + static_assert( alignof( T ) <= kTableAlign, "a table node's alignment must fit the arena's" ); + TableSlot slot; + if ( arena == NULL || arena->locked ) { return slot; } + uint32_t bytes = TableAlignUp( (uint32_t) sizeof( T ) ); + if ( bytes > kTableSlabBytes ) { return slot; } // a node larger than a slab: refused, never split + if ( end == 0 || next + bytes > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return slot; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + uint32_t at = next; + next += bytes; + // A NODE IS BORN IN TWO HALVES: start its lifetime in the raw + // storage, then write the declared defaults ONE MEMBER AT A TIME. + // + // It is "T", not "T{}". Value-initialising the whole aggregate says + // the same thing and costs cl O(BYTES) TO COMPILE — it expands element + // by element in its front end — while both halves here cost + // O(declarations). The slab cap below refuses a large node at RUN + // TIME and bounds nothing at compile time: the cost is paid by + // whatever T a caller instantiates this with. + // Padding is not the difference: value-initialisation zeroes MEMBERS + // and not padding either way, which is why the segment is calloc'd. + // + // TableReset is an OVERLOAD SET, one per closure member, reached from + // this template by argument-dependent lookup on T's own namespace — + // Alloc is a template and cannot spell Reset. + // + // The reset is here because ONE DEFINITION SAYS WHAT THE DECLARED + // DEFAULTS ARE, and it is Reset. Default-initialisation lands on + // the same values today, because a member with a non-zero default + // carries a member initializer that says so — but that is the class + // definition agreeing with Reset, not the arena reading it, and #320's + // fix was itself a pass that MOVED initialisation between the two. + // The arena reads the definition. + slot.ptr = new ( TableArenaAt( *arena, at ) ) T; + TableReset( *slot.ptr ); + slot.ref.value = at; + return slot; + } + + // Alloc a BYTE BUFFER's node of exactly length bytes (docs/SPEC-TABLES.md + // §2.5): the blob header and its bytes, zeroed, in this thread's slab when + // it fits and in a span of the arena's own when it does not. NULL is the + // arena locked, a length below zero or past a record's u32, or the + // allocator refusing. The offset comes back for the reference. + TableBlob * AllocBlob( int64_t length, bool terminated, uint32_t & at ) + { + at = 0; + if ( arena == NULL || arena->locked ) { return NULL; } + if ( length < 0 || length > kTableBlobMaxLength ) { return NULL; } + const int64_t bytes = TableBlobStorage( length, terminated ); + if ( bytes > (int64_t) kTableSlabBytes ) + { + at = TableArenaGrabSpan( *arena, bytes ); + if ( at == kTableAllocFailed ) { at = 0; return NULL; } + } + else + { + if ( end == 0 || next + (uint32_t) bytes > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return NULL; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + at = next; + next += (uint32_t) bytes; + } + TableBlob * blob = (TableBlob *) TableArenaAt( *arena, at ); + blob->length = (uint32_t) length; // the bytes after it are the segment's zeros + blob->zero = 0; + return blob; + } + + // RAW, ZEROED storage of the bytes asked for, at the alignment asked for: a MAP's or a LIST's builder + // head and its segments (docs/SPEC-TABLES.md §2.8, §2.9). It is not a node: it carries + // no type id, takes no index and has no Reset, so it goes through the same + // slab and span the blob path uses rather than through Alloc. + uint8_t * AllocRaw( int64_t bytes, int64_t align, uint32_t & at ) + { + at = 0; + if ( arena == NULL || arena->locked ) { return NULL; } + if ( bytes <= 0 || align > (int64_t) kTableAlign ) { return NULL; } + const int64_t rounded = TableAlignUp64( bytes ); + if ( rounded > (int64_t) kTableSlabBytes ) + { + at = TableArenaGrabSpan( *arena, rounded ); + if ( at == kTableAllocFailed ) { at = 0; return NULL; } + return TableArenaAt( *arena, at ); + } + if ( end == 0 || next + (uint32_t) rounded > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return NULL; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + at = next; + next += (uint32_t) rounded; + return TableArenaAt( *arena, at ); // the segment came back zeroed + } + // a *bytes node: the bytes to write through, and the reference to store + TableBytesSlot AllocBytes( int64_t length ) + { + TableBytesSlot slot; + uint32_t at = 0; + TableBlob * blob = AllocBlob( length, false, at ); + if ( blob == NULL ) { return slot; } + slot.data = (uint8_t *) ( blob + 1 ); + slot.length = length; + slot.ref.value = at; + return slot; + } + + // a *string node: room for length bytes and the zero byte after them + TableStringSlot AllocString( int64_t length ) + { + TableStringSlot slot; + uint32_t at = 0; + TableBlob * blob = AllocBlob( length, true, at ); + if ( blob == NULL ) { return slot; } + slot.data = (char *) ( blob + 1 ); + slot.length = length; + slot.ref.value = at; + return slot; + } +}; + +// ---- TablePackMap: the pack walk's identity map (docs/SPEC-TABLES.md §3.1, §6.2) ---- +// +// ONE ENTRY PER REACHABLE NODE, and that map IS identity: a node must know +// where it landed to be named a second time, so Lock packs a shared node ONCE +// and every later reference resolves to the one body it already has. That is +// the same first-visit numbering the wire uses, so the pack order and the node +// order are one order. +// +// COLOURING AN ENTRY WHILE ITS DESCENT IS OPEN COSTS ONE BIT, and it is what +// makes a data cycle free to refuse: a reference to an entry still open is a +// cycle, and Lock returns failure rather than recursing away. The ROOT's entry +// is open for the whole walk. +// +// The map is proportional to NODES, never to bytes, and it lives on the +// AUTHORING side, where §6.5 licenses allocation. Nothing on the reading path +// ever builds one. +struct TablePackEntry +{ + const void * key; // the node's address in the graph being packed + int64_t offset; // where that node landed in the region + uint8_t open; // its descent is still open: a reference here is a cycle +}; + +struct TablePackMap +{ + TablePackEntry * entries = NULL; + int64_t capacity = 0; // a power of two, or zero while empty + int64_t count = 0; + TableAllocator allocator; // the caller's, carried from the walk that built it +}; + +inline void TablePackMapInit( TablePackMap & map, TableAllocator allocator ) +{ + map.entries = NULL; + map.capacity = 0; + map.count = 0; + map.allocator = allocator; +} + +inline void TablePackMapShutdown( TablePackMap & map ) +{ + map.allocator.free( map.allocator.context, map.entries ); + TablePackMapInit( map, map.allocator ); +} + +// The two walks behind Lock re-derive the SAME map from the same graph — the +// numbering is never carried between them (§3.1) — so the second starts from +// an empty map and keeps the capacity the first paid for. +inline void TablePackMapReset( TablePackMap & map ) +{ + if ( map.entries != NULL ) { memset( map.entries, 0, (size_t) map.capacity * sizeof( TablePackEntry ) ); } + map.count = 0; +} + +// open addressing, linear probing, a multiply-shift hash over the address: a +// node key is a pointer and its low bits are alignment, so the low bits alone +// would collide on every node of one type +inline int64_t TablePackMapSlot( const TablePackMap & map, const void * key ) +{ + uint64_t hash = (uint64_t) (uintptr_t) key; + hash *= 0x9E3779B97F4A7C15ull; + hash ^= hash >> 29; + int64_t mask = map.capacity - 1; + int64_t at = (int64_t) ( hash & (uint64_t) mask ); + while ( map.entries[at].key != NULL && map.entries[at].key != key ) + { + at = ( at + 1 ) & mask; + } + return at; +} + +inline TablePackEntry * TablePackMapFind( TablePackMap & map, const void * key ) +{ + if ( map.capacity == 0 ) { return NULL; } + TablePackEntry * entry = &map.entries[ TablePackMapSlot( map, key ) ]; + return entry->key == key ? entry : NULL; +} + +// QUADRUPLING, not doubling, and the reason is measured: growth rehashes every +// entry, and on a graph of 131,071 nodes the doubling schedule spent 45% of +// Lock in rehashing alone. Quadrupling from 1024 buys 1.35x on that graph and +// keeps the map NODE-proportional (§6.2) — under 128 bytes a node at its +// worst, right after a grow, and about 64 on average. +inline bool TablePackMapGrow( TablePackMap & map ) +{ + TablePackMap grown; + grown.allocator = map.allocator; + grown.capacity = map.capacity != 0 ? map.capacity * 4 : 1024; + grown.entries = (TablePackEntry *) map.allocator.alloc( map.allocator.context, grown.capacity * (int64_t) sizeof( TablePackEntry ) ); + if ( grown.entries == NULL ) { return false; } + for ( int64_t i = 0; i < map.capacity; i++ ) + { + if ( map.entries[i].key == NULL ) { continue; } + grown.entries[ TablePackMapSlot( grown, map.entries[i].key ) ] = map.entries[i]; + grown.count++; + } + map.allocator.free( map.allocator.context, map.entries ); + map = grown; + return true; +} + +// REACH a node: one probe answers both questions the walk has. A true "taken" +// says this is a FIRST visit, and the entry is now the node's, coloured open +// at "offset"; otherwise the entry is the one the node already has, and its +// open bit says cycle or sharing. NULL is an allocation failure, and it is a +// refusal like any other: Lock fails rather than packing a graph it cannot +// track. +// +// It is one call and not a find followed by an insert because the walk asks +// this question twice per node — once to measure, once to pack — and every +// probe is a miss into a table larger than L2. +inline TablePackEntry * TablePackMapReach( TablePackMap & map, const void * key, int64_t offset, bool & taken, int64_t & slot ) +{ + if ( ( map.count + 1 ) * 4 >= map.capacity * 3 ) // keep the load factor under three quarters + { + if ( !TablePackMapGrow( map ) ) { return NULL; } + } + slot = TablePackMapSlot( map, key ); + TablePackEntry * entry = &map.entries[slot]; + taken = entry->key != key; // an empty slot is a first visit; the key is never NULL + if ( taken ) + { + entry->key = key; + entry->offset = offset; + entry->open = 1; + map.count++; + } + return entry; +} + +// The descent finished: the node keeps its entry — identity outlives the +// descent — and stops being a cycle. The "hint" is the slot Reach returned, and it +// is checked against the key rather than trusted, so a rehash between the two +// costs a second probe instead of correctness. +inline void TablePackMapClose( TablePackMap & map, const void * key, int64_t hint ) +{ + if ( hint >= 0 && hint < map.capacity && map.entries[hint].key == key ) + { + map.entries[hint].open = 0; + return; + } + TablePackEntry * entry = TablePackMapFind( map, key ); + if ( entry != NULL ) { entry->open = 0; } +} + +// ---- resolution contexts: which encoding a walk is reading ---- + +struct TableArenaCtx { const TableArena * arena; }; +struct TableRegionCtx {}; + +// ---- a BYTE BUFFER's resolution (docs/SPEC-TABLES.md §2.5, §6.3) ---- +// +// The same two encodings a table pointer has, resolved the same way: a +// self-relative delta in a region — one add, no base — and an arena offset +// while the builder is mutable. The blob is reached through its header, and a +// view is the header plus eight and the header's first word. Nothing here +// allocates and nothing copies: off a locked region, a loaded one or an +// opened cook the view points INTO the region. +inline const TableBlob * TableBlobAt( const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) ( (const uint8_t *) &ref + ref.value ) : NULL; +} +inline const TableBlob * TableBlobAt( const TableRegionCtx &, const TableRef & ref ) { return TableBlobAt( ref ); } +inline const TableBlob * TableBlobAt( const TableArenaCtx & ctx, const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) TableArenaAt( *ctx.arena, (uint32_t) ref.value ) : NULL; +} +inline const TableBlob * TableBlobAt( const TableArena & arena, const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) TableArenaAt( arena, (uint32_t) ref.value ) : NULL; +} + +inline TableBytesView TableBytesViewOf( const TableBlob * blob ) +{ + TableBytesView view = { NULL, 0 }; + if ( blob != NULL ) { view.data = (const uint8_t *) ( blob + 1 ); view.length = (int64_t) blob->length; } + return view; +} +inline TableStringView TableStringViewOf( const TableBlob * blob ) +{ + TableStringView view = { NULL, 0 }; + if ( blob != NULL ) { view.data = (const char *) ( blob + 1 ); view.length = (int64_t) blob->length; } + return view; +} + +// the const form's hot path: one add, no base +inline TableBytesView TableBytesAt( const TableRef & ref ) { return TableBytesViewOf( TableBlobAt( ref ) ); } +inline TableStringView TableStringAt( const TableRef & ref ) { return TableStringViewOf( TableBlobAt( ref ) ); } +// and the context forms a walk uses: a region context, an arena context, or +// the arena itself while the builder is mutable +template inline TableBytesView TableBytesAt( const Ctx & ctx, const TableRef & ref ) { return TableBytesViewOf( TableBlobAt( ctx, ref ) ); } +template inline TableStringView TableStringAt( const Ctx & ctx, const TableRef & ref ) { return TableStringViewOf( TableBlobAt( ctx, ref ) ); } + +// allocate a blob in the arena and point the slot at it; the slot holds the +// arena offset, as every slot does while the builder is mutable +inline uint8_t * TableBytesEmplace( TableWorker & worker, TableRef & slot, int64_t length ) +{ + TableBytesSlot allocated = worker.AllocBytes( length ); + slot = allocated.ref; + return allocated.data; +} +// the text is copied in when one is given; a NULL text leaves the zeros for +// the caller to fill +inline char * TableStringEmplace( TableWorker & worker, TableRef & slot, const char * text, int64_t length ) +{ + TableStringSlot allocated = worker.AllocString( length ); + slot = allocated.ref; + if ( allocated.data != NULL && text != NULL && length > 0 ) { memcpy( allocated.data, text, (size_t) length ); } + return allocated.data; +} + +// ---- the FLAT NODE TABLE (docs/SPEC-TABLES.md §3.1) ---- +// +// A pointered save writes every reachable node ONCE, into a node table, and a +// pointer field rides as an INDEX into it under kind 17. The encoding is +// flat: no pointer edge is a nesting level, so a chain's length is not a depth, +// and two references to one node are one node. +// +// THE FIELD RIDES ONCE: an L with sixty-four bits of capability frames a +// numbering of any size, so the whole numbering is one contiguous payload and a +// save's node bodies have no aggregate ceiling. + +static const uint64_t kTableNodeIndexNull = 0; // absence and null are one value +static const uint64_t kTableNodeIndexRoot = 1; // the body that hosts the table + +// The not-materialized sentinel (§6.3): a record whose type id this build could +// not name. Distinct from every real offset including the root's 0, so an index +// resolving through it yields NULL and can never fabricate the root. +static const uint64_t kTableNodeAbsent = 0xFFFFFFFFFFFFFFFFull; + +// What a node's storage answers when the FRAMING ITSELF is refused rather than +// merely unnameable: a count its L cannot carry, one above the int32 cap, or a +// blob past the size cap (docs/SPEC-TABLES.md §3.1, §6.5). An unnameable type +// id commands no storage and keeps its index. This one makes the whole measure +// answer -1 with its reason. +static const int64_t kTableNodeRefused = -2; + +// ---- the numbering, on the SAVE side ---- +// +// One entry per reachable node in FIRST-VISIT order, so entry k is node index +// k + 2. The two thunks are what let one loop write a table of mixed types: the +// numbering walk knows each target's type STATICALLY at the site it numbers it, +// so it stores the instantiation there and the loop never asks what a node is. +struct TableNumbering; + +struct TableNodeEntry +{ + const void * node; + uint64_t type_id; + // the type id's MESSAGE-FORM SLOT (docs/SPEC-TABLES.md §3.3), stored where + // the numbering walk stores the id itself and for the same reason: the + // target's type is known STATICALLY at the site that numbers it, so a + // form 2 save reads the slot out of the entry instead of looking an id up. + // Every pointer target's type id is an entry of the announcement, which is + // what makes the slot a compile-time fact of a POINTERED message too. + uint64_t type_slot; + int64_t ( * measure )( const void * ctx, const TableNumbering & numbering, TableIds & ids, const void * node ); + bool ( * save )( const void * ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const void * node ); + // the same two over the MESSAGE FORM (docs/SPEC-TABLES.md §3.3): a bitpacked + // body at a bit position, its pointer indices at the width the node count + // settled + int64_t ( * message_measure )( const void * ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const void * node ); + bool ( * message_save )( const void * ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const void * node ); +}; + +struct TableNumbering +{ + TablePackMap seen; // node -> index; the ROOT is index 1, open for the whole walk + TableNodeEntry * entries = NULL; + int64_t count = 0; + int64_t capacity = 0; +}; + +// The numbering allocates through the map's pair rather than carrying a second +// copy of it: one numbering is one walk, and a walk has one allocator. +inline void TableNumberingInit( TableNumbering & n, TableAllocator allocator ) +{ + TablePackMapInit( n.seen, allocator ); + n.entries = NULL; + n.count = 0; + n.capacity = 0; +} + +inline void TableNumberingShutdown( TableNumbering & n ) +{ + TableAllocator allocator = n.seen.allocator; + TablePackMapShutdown( n.seen ); + allocator.free( allocator.context, n.entries ); + n.entries = NULL; + n.count = 0; + n.capacity = 0; +} + +// The index a numbered node was given, for the save that writes it into a +// pointer slot. False means the two walks disagree about the graph, which is a +// refusal and never a guess. +inline bool TableNumberingIndex( const TableNumbering & n, const void * node, uint64_t & index ) +{ + if ( n.seen.capacity == 0 ) { return false; } + const TablePackEntry & entry = n.seen.entries[ TablePackMapSlot( n.seen, node ) ]; + if ( entry.key != node ) { return false; } + index = (uint64_t) entry.offset; + return true; +} + +inline bool TableNumberingAppend( TableNumbering & n, const TableNodeEntry & entry ) +{ + if ( n.count == n.capacity ) + { + // GROW BY COPY, never by realloc: the allocator hook is a PAIR, and a + // game's heap is not required to have a resize primitive at all. The + // schedule quadruples, so the copying is amortized to a constant per + // entry and the growth is the same growth it always was. + int64_t capacity = n.capacity != 0 ? n.capacity * 4 : 256; + TableAllocator allocator = n.seen.allocator; + TableNodeEntry * grown = (TableNodeEntry *) allocator.alloc( allocator.context, capacity * (int64_t) sizeof( TableNodeEntry ) ); + if ( grown == NULL ) { return false; } + if ( n.entries != NULL ) + { + memcpy( grown, n.entries, (size_t) n.count * sizeof( TableNodeEntry ) ); + allocator.free( allocator.context, n.entries ); + } + n.entries = grown; + n.capacity = capacity; + } + n.entries[n.count++] = entry; + return true; +} + +// The thunks the numbering stores. Each resolves to the closure member's own +// MeasureBody / SaveBodyFields through an overload set in the member's DECLARING +// file, reached by argument-dependent lookup at instantiation — the same bridge +// the arena's TableReset uses, and the reason a numbering may span the files of +// one unit without any file naming another's members. +template +inline int64_t TableNodeMeasureThunk( const void * ctx, const TableNumbering & numbering, TableIds & ids, const void * node ) +{ + return TableNodeMeasure( *(const Ctx *) ctx, numbering, ids, *(const T *) node ); +} + +template +inline bool TableNodeSaveThunk( const void * ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const void * node ) +{ + return TableNodeSave( *(const Ctx *) ctx, numbering, w, ids, *(const T *) node ); +} + +template +inline int64_t TableNodeMessageMeasureThunk( const void * ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const void * node ) +{ + return TableNodeMessageMeasure( *(const Ctx *) ctx, numbering, index_bits, at, *(const T *) node ); +} + +template +inline bool TableNodeMessageSaveThunk( const void * ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const void * node ) +{ + return TableNodeMessageSave( *(const Ctx *) ctx, numbering, index_bits, w, *(const T *) node ); +} +// ---- a BYTE BUFFER's record (docs/SPEC-TABLES.md §2.5, §3.1) ---- +// +// A blob rides as a node record under one of two RESERVED type ids — the fold +// a table's name takes, over the keywords "bytes" and "string", which no table +// can be named — with the bytes as its body and nothing framed inside. These +// two thunks are what the numbering stores for a blob, as it stores a +// member's codec for a table: the length, and the bytes verbatim. +static const uint64_t kTableBytesTypeId = 0x2f2ec0474f1c4fe4ull; // fnv1a64( "bytes" ) +static const uint64_t kTableStringTypeId = 0x704be0d8faaffc58ull; // fnv1a64( "string" ) + +template +inline int64_t TableBlobMeasureThunk( const void *, const TableNumbering &, TableIds &, const void * node ) +{ + return (int64_t) ( (const TableBlob *) node )->length; +} + +template +inline bool TableBlobSaveThunk( const void *, const TableNumbering &, TableWriter & w, TableIds &, const void * node ) +{ + const TableBlob * blob = (const TableBlob *) node; + w.raw( (const void *) ( blob + 1 ), (int64_t) blob->length ); + return true; +} + +// and the same two on the MESSAGE FORM (§3.3): a blob record is its length at +// thirty-two raw bits, an ALIGN, then the bytes verbatim +template +inline int64_t TableBlobMessageMeasureThunk( const void *, const TableNumbering &, int64_t, int64_t at, const void * node ) +{ + const int64_t length = (int64_t) ( (const TableBlob *) node )->length; + return 32 + TableAlignBits( at + 32 ) + length * 8; +} + +template +inline bool TableBlobMessageSaveThunk( const void *, const TableNumbering &, int64_t, TableBitWriter & w, const void * node ) +{ + const TableBlob * blob = (const TableBlob *) node; + w.put( (uint64_t) blob->length, 32 ); + w.align(); + w.putbytes( (const uint8_t *) ( blob + 1 ), (int64_t) blob->length ); + return !w.overflow; +} +// TableNodeTableMeasure and TableNodeTableSave are the framing, and they are +// ONE fill rule written twice — measure derives it from the graph and save +// derives the same one, which is what makes measure == save hold across a +// pointer graph (§3.1). +// +// The field rides ONCE, under the reserved id, kind 12: the payload opens with +// the count and then carries the records back to back, each a type id +// REFERENCE, a length and a body. The reserved id is interned BEFORE the +// records, and a record's type id before its body, which is the first-use order +// the trailer is written in (§3). +template +inline int64_t TableNodeTablePayload( const Ctx & ctx, TableIds & ids, const TableNumbering & n ) +{ + int64_t payload = TableLebBytes( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + payload += TableLebBytes( ids.ref( n.entries[k].type_id ) ); + const int64_t body = n.entries[k].measure( (const void *) &ctx, n, ids, n.entries[k].node ); + if ( body < 0 ) { return -1; } + payload += TableLebBytes( (uint64_t) body ) + body; + } + return payload; +} + +template +inline int64_t TableNodeTableMeasure( const Ctx & ctx, TableIds & ids, const TableNumbering & n ) +{ + if ( n.count == 0 ) { return 0; } // a root that reaches no nodes writes none of them + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayload( ctx, ids, n ); + if ( payload < 0 ) { return -1; } + return TableLebBytes( ref ) + 1 + TableLebBytes( (uint64_t) payload ) + payload; +} + +template +inline bool TableNodeTableSave( const Ctx & ctx, TableWriter & w, TableIds & ids, const TableNumbering & n ) +{ + if ( n.count == 0 ) { return true; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayload( ctx, ids, n ); + if ( payload < 0 ) { return false; } + w.putleb( ref ); + w.put8( 12 ); // kind 12 is the opaque byte payload: a reader that cannot name the id skips by L + w.putleb( (uint64_t) payload ); + w.putleb( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.putleb( ids.ref( n.entries[k].type_id ) ); + const int64_t body = n.entries[k].measure( (const void *) &ctx, n, ids, n.entries[k].node ); + if ( body < 0 ) { return false; } + w.putleb( (uint64_t) body ); + if ( !n.entries[k].save( (const void *) &ctx, n, w, ids, n.entries[k].node ) ) { return false; } + } + return true; +} + +// ---- the numbering, on the LOAD side: a region's NODE DIRECTORY (§6.3) ---- +// +// The wire's numbering made resident: one entry per numbered node, in index +// order, position i describing node index i + 1 — so position 0 is the ROOT at +// offset 0. It is ATTRIBUTION, and attribution is separable: nothing that reads +// a structure touches it, a deref is one add on a self-relative offset, and a +// caller may release it once Load returns. +struct TableNodeDirEntry +{ + uint64_t offset; + uint64_t type_id; +}; + +// the node's extent cursor, defined with the extent runtime (docs/SPEC-TABLES.md +// §2.8, §2.9); the node map names it only through a pointer. +struct TableExtentCarve; + +// TableNodeMap is what a pointer slot resolves through while a body decodes. +struct TableNodeMap +{ + uint8_t * base = NULL; + const TableNodeDirEntry * entries = NULL; + int64_t count = 0; // the ROOT's entry included, so it is records + 1 + bool good = false; // the node table read whole; a numbering that failed resolves nothing + // WHERE THE NODES LIVE, and therefore what a resolved slot holds: a region + // takes the SELF-RELATIVE delta so a deref is one add, and the tool's + // builder path takes the node's ARENA OFFSET (§6.3). + bool arena = false; + // WHERE A MAP'S ENTRIES AND A LIST'S ELEMENTS LAND while this node's body + // decodes (docs/SPEC-TABLES.md §2.8, §2.9): the node's own extent on the + // region path and the builder's arena on the tool's. It is MUTABLE + // because the cursor belongs to ONE node's decode and the dispatch that + // owns that node holds the map by const reference, exactly as it did + // before either construct existed. The decoder's signature does not + // move for a construct it may not carry. + mutable TableExtentCarve * carve = NULL; + // and the TOOL's path's allocation front, set once: there the arrays + // are the builder's arena's rather than a node's extent. + TableWorker * worker = NULL; + // THE TOOL PATH'S REFUSAL (docs/SPEC-TABLES.md §2.9): a count above the + // int32 cap met while a body decoded. LoadBuilder answers NULL for it + // and moves no counter; mutable for the reason the cursor is. + mutable bool refused = false; +}; + +// TableNodeResolve places one node index in a pointer slot, and every failure +// is one of §4's events with the pointer left null. The declared TARGET type id +// is checked at every index, the root's included: the root carries no record +// and therefore no wire type id, so the READER'S OWN root type is what the +// claim is checked against. +inline void TableNodeResolve( const TableNodeMap & map, TableRef & slot, uint64_t index, uint64_t target, TableReport * report ) +{ + slot.value = 0; + if ( index == kTableNodeIndexNull || !map.good ) { return; } + if ( index - 1 >= (uint64_t) map.count ) + { + report->malformed = true; // an index above node_count + 1 + return; + } + const TableNodeDirEntry & entry = map.entries[index - 1]; + if ( entry.offset == kTableNodeAbsent ) + { + // a node whose type id this build could not name KEEPS ITS INDEX, and + // every pointer naming it reads null. The unknown was counted once, at + // the node, not once per pointer. + return; + } + if ( entry.type_id != target ) + { + report->kind_mismatch++; + return; + } + slot.value = map.arena ? (int64_t) entry.offset + : (int64_t) ( ( map.base + entry.offset ) - (const uint8_t *) &slot ); +} + +// ---- the record SCAN, and it is the whole of load's bound (§3.1) ---- +// +// Reading follows no reference. The scan walks the root body's top-level fields, +// finds the ONE under the reserved id, and reads records out of its payload in +// order — the field rides once, so nothing is copied to make a body contiguous +// and the generated body decoder never learns the transport exists. +struct TableNodeScan +{ + TableReader fields; // over the ROOT body, skipping past everything else + const uint8_t * payload; // the node-table field's payload + int64_t payload_size; + int64_t payload_offset; + bool opened; // the root body has been walked for the field + uint64_t declared; + int64_t records; + bool present; // the root body carries a node table at all + bool malformed; + const TableIdTable * ids; +}; + +inline TableNodeScan TableNodeScanBegin( const uint8_t * body, int64_t size, TableReport * report, const TableIdTable * ids ) +{ + TableNodeScan s = { TableReader( body, size, report, ids ), NULL, 0, 0, false, 0, 0, false, false, ids }; + return s; +} + +// find the node-table field, or answer false when the root body has none. A +// body carrying an id more than once is legal input and THE LAST OCCURRENCE +// WINS (docs/SPEC-TABLES.md §3), so the walk runs to the terminator and keeps +// the last rather than stopping at the first. +inline bool TableNodeScanOpen( TableNodeScan & s ) +{ + if ( s.opened ) { return false; } + s.opened = true; + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.fields.getleb( ref ) ) { break; } + if ( ref == 0 ) { break; } // the terminator + if ( s.ids == NULL || ref > (uint64_t) s.ids->count ) { break; } + const uint64_t id = s.ids->at( ref ); + if ( !s.fields.has( 1 ) ) { break; } + const uint8_t kind = s.fields.get8(); + if ( id == kTableNodeTableFieldId ) + { + s.present = true; + if ( kind != 12 ) { s.malformed = true; return false; } + uint64_t length = 0; + if ( !s.fields.getleb( length ) || !s.fields.room( length ) ) { s.malformed = true; return false; } + s.payload = s.fields.buffer + s.fields.offset; + s.payload_size = (int64_t) length; + s.fields.offset += (int64_t) length; + continue; + } + if ( !s.fields.skip( kind ) ) { break; } + } + if ( s.payload == NULL ) { return false; } + TableReader head( s.payload, s.payload_size, s.fields.report, s.ids ); + if ( !head.getleb( s.declared ) ) { s.malformed = true; return false; } + s.payload_offset = head.offset; + return true; +} + +// the next record, or false at the end of the table — s.malformed says whether +// the end was the end or the framing giving out +inline bool TableNodeScanNext( TableNodeScan & s, uint64_t & type_id, const uint8_t * & body, int64_t & length ) +{ + if ( !s.opened && !TableNodeScanOpen( s ) ) { return false; } + if ( s.payload == NULL || s.payload_offset >= s.payload_size ) { return false; } + TableReader rec( s.payload, s.payload_size, s.fields.report, s.ids ); + rec.offset = s.payload_offset; + uint64_t ref = 0; + if ( !rec.getleb( ref ) || ref == 0 || s.ids == NULL || ref > (uint64_t) s.ids->count ) + { + s.malformed = true; // a type id reference of 0, or one past the table + return false; + } + type_id = s.ids->at( ref ); + uint64_t declared_length = 0; + if ( !rec.getleb( declared_length ) ) + { + s.malformed = true; // a record whose length is damaged + return false; + } + if ( declared_length > (uint64_t) ( s.payload_size - rec.offset ) ) + { + s.malformed = true; // a record whose length runs past its field + return false; + } + body = s.payload + rec.offset; + length = (int64_t) declared_length; + s.payload_offset = rec.offset + length; + s.records++; + return true; +} + +// The record scan is AUTHORITATIVE: node_count is data from the wire, and a +// count that disagrees with the scan is malformed. Nothing is sized from it +// before the scan has confirmed it. +inline bool TableNodeScanWhole( TableNodeScan & s ) +{ + if ( s.malformed ) { return false; } + if ( !s.present ) { return true; } // no node table at all is not a broken one + return s.declared == (uint64_t) s.records; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_ARENA + +#ifndef MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES +#define MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES + +namespace mapdemo { + +// ---- the NODE TABLE on the message wire (docs/SPEC-TABLES.md §3.1, §3.3) ---- +// +// THE NODE TABLE, WHEN A BODY HAS ONE, IS THE FIRST FIELD OF THE ROOT BODY: the +// reserved id as a reference, the node count at THIRTY-TWO RAW BITS, then the +// records back to back, each a type id reference and a body: a table's fields +// end at their own zero reference, and a blob's body is a length, an align and +// its bytes. A root +// that reaches no node elides the field, like every other empty thing. +// +// Measure derives the numbering from the graph and save derives the same one, +// and the two thunks stored at numbering time are what let one loop write a +// table of mixed types. +template +inline int64_t TableMessageNodeTableMeasure( const Ctx & ctx, const TableNumbering & n, int64_t index_bits, int64_t at ) +{ + if ( n.count == 0 ) { return 0; } // a root that reaches no nodes writes none of them + int64_t bits = kTableMessageRefBitsHere + 32; + for ( int64_t k = 0; k < n.count; k++ ) + { + bits += kTableMessageRefBitsHere; + const int64_t body = n.entries[k].message_measure( (const void *) &ctx, n, index_bits, at + bits, n.entries[k].node ); + if ( body < 0 ) { return -1; } + bits += body; + } + return bits; +} + +template +inline bool TableMessageNodeTableSave( const Ctx & ctx, const TableNumbering & n, int64_t index_bits, TableBitWriter & w ) +{ + if ( n.count == 0 ) { return true; } + w.put( kTableNodeTableFieldSlot, kTableMessageRefBitsHere ); + w.put( (uint64_t) n.count, 32 ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.put( n.entries[k].type_slot, kTableMessageRefBitsHere ); + if ( !n.entries[k].message_save( (const void *) &ctx, n, index_bits, w, n.entries[k].node ) ) { return false; } + } + return !w.overflow; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES + +#ifndef MAPDEMO_SCHEMA_TABLE_RETAIN +#define MAPDEMO_SCHEMA_TABLE_RETAIN + +namespace mapdemo { + +// ---- RETAIN-UNKNOWN (docs/SPEC-TABLES.md §6.6) ---- +// +// A REGION ROUND TRIP AND ONLY THAT: LoadRetain is Load's path into a region +// and SaveRetain saves from that same region. The builder path carries no +// retention, because a builder has no node directory to anchor a record on and +// re-derives its numbering from the reader's declaration order. +// +// Nothing here allocates. The record bytes and the retained-id list are the +// caller's storage, declared with their capacities, and a record that does not +// fit whole is dropped with one retain_lost. + +// THE PATH NAMES THE BODY, and it is the REGION's own address (§6.6). Step one +// is the node's index in the region's node directory, 1 for the root body and +// k for the node at directory position k - 1. Every further step is the PAIR: +// the field ordinal in the body the step descends from, in the READER's own +// declaration order, and the element index inside that field: zero for a +// scalar body, the element's index for an array of any of the four kinds, the +// ARM's OWN ORDINAL for a union, and the key's slot for a map. +static const int32_t kTableRetainDepthMax = 5; + +struct TableRetainStep +{ + uint32_t ordinal; + uint32_t index; +}; + +// at is the node's own address, which is what the SAVE side matches on: a +// record carries the directory INDEX and the directory answers the address in +// one add, so neither side ever searches a numbering. +struct TableRetainPath +{ + const void * at; + uint32_t node; + int32_t depth; + TableRetainStep steps[ kTableRetainDepthMax ]; +}; + +inline TableRetainPath TableRetainPathRoot( const void * at, uint32_t node ) +{ + TableRetainPath path; + path.at = at; + path.node = node; + path.depth = 0; + return path; +} + +// A STEP IS COMPUTED LOCALLY, at the moment the walk descends (§6.6), and it +// is taken by VALUE so that a descent is an expression: both sides walk the +// same declaration order, so neither numbers a tree and neither pops. +inline TableRetainPath TableRetainStepInto( const TableRetainPath & path, uint32_t ordinal, uint32_t index ) +{ + TableRetainPath out = path; + if ( out.depth < kTableRetainDepthMax ) + { + out.steps[ out.depth ].ordinal = ordinal; + out.steps[ out.depth ].index = index; + } + out.depth++; + return out; +} + +// THE CALLER'S TWO STORES (§6.6): the record bytes and the retained ids, each +// a pointer, a capacity and what has been used of it. A retention buffer +// belongs to ONE loaded region, and the next LoadRetain into it resets both. +struct TableRetain +{ + // AN ENTRY IS THE ID AND ITS SLOT IN THE TRAILER BEING WRITTEN. The two + // stores are numbered into ONE trailer in merged first-use order, so an + // index into this list is not the number a second reference wants and the + // slot rides beside the id. The layout is this port's own. + struct Id + { + uint64_t id; + int32_t slot; + }; + + uint8_t * bytes = NULL; + int64_t capacity = 0; + int64_t used = 0; + Id * ids = NULL; + int32_t id_capacity = 0; + int32_t id_used = 0; + int32_t count = 0; // records held + + // the REGION this buffer belongs to: a record carries a directory index + // and the save resolves it here, so nothing searches and nothing allocates + const uint8_t * base = NULL; + const TableNodeDirEntry * directory = NULL; + int64_t directory_count = 0; +}; + +// A RETAINED RECORD IS READER-PRIVATE (§6.6). It is not a wire form: no form +// byte, no version, no declared byte order, and nothing ever writes one to +// disk or hands one to another process. What it must CARRY is the body it +// belongs to, and the field's identity and bytes with every reference +// resolved. This layout is one sound way to carry them and nothing compares +// two ports' buffers. +// +// u32 record bytes, this header included +// u32 node the path's first step +// u32 depth the step pairs that follow +// u32 payload bytes +// u64 field id +// u8 kind +// u8 placed the save's own mark, cleared before every save +// depth x { u32 ordinal, u32 index } +// payload the field's payload with every reference resolved +static const int64_t kTableRetainRecordHeader = 26; + +inline uint32_t TableRetainRead32( const uint8_t * p ) +{ + return uint32_t( p[0] ) | uint32_t( p[1] ) << 8 | uint32_t( p[2] ) << 16 | uint32_t( p[3] ) << 24; +} + +inline uint64_t TableRetainRead64( const uint8_t * p ) +{ + return uint64_t( TableRetainRead32( p ) ) | ( uint64_t( TableRetainRead32( p + 4 ) ) << 32 ); +} + +inline void TableRetainWrite32( uint8_t * p, uint32_t v ) +{ + p[0] = uint8_t( v ); p[1] = uint8_t( v >> 8 ); p[2] = uint8_t( v >> 16 ); p[3] = uint8_t( v >> 24 ); +} + +inline void TableRetainWrite64( uint8_t * p, uint64_t v ) +{ + TableRetainWrite32( p, uint32_t( v ) ); + TableRetainWrite32( p + 4, uint32_t( v >> 32 ) ); +} + +inline int64_t TableRetainRecordBytes( const uint8_t * record ) { return (int64_t) TableRetainRead32( record ); } +inline uint32_t TableRetainRecordNode( const uint8_t * record ) { return TableRetainRead32( record + 4 ); } +inline int32_t TableRetainRecordDepth( const uint8_t * record ) { return (int32_t) TableRetainRead32( record + 8 ); } +inline int64_t TableRetainRecordPayloadBytes( const uint8_t * record ) { return (int64_t) TableRetainRead32( record + 12 ); } +inline uint64_t TableRetainRecordId( const uint8_t * record ) { return TableRetainRead64( record + 16 ); } +inline uint8_t TableRetainRecordKind( const uint8_t * record ) { return record[24]; } +inline bool TableRetainRecordPlaced( const uint8_t * record ) { return record[25] != 0; } +inline const uint8_t * TableRetainRecordSteps( const uint8_t * record ) { return record + kTableRetainRecordHeader; } +inline const uint8_t * TableRetainRecordPayload( const uint8_t * record ) +{ + return record + kTableRetainRecordHeader + 8 * (int64_t) TableRetainRecordDepth( record ); +} +inline uint8_t * TableRetainRecordPayload( uint8_t * record ) +{ + return record + kTableRetainRecordHeader + 8 * (int64_t) TableRetainRecordDepth( record ); +} + +// THE RECORD'S OWN BODY, resolved through the directory the buffer holds. A +// node index names one node for the life of the region, so this is one add. +inline const void * TableRetainRecordAt( const TableRetain & retain, const uint8_t * record ) +{ + const uint32_t node = TableRetainRecordNode( record ); + if ( retain.directory == NULL || node == 0 || (int64_t) node > retain.directory_count ) { return NULL; } + return (const void *) ( retain.base + retain.directory[ node - 1 ].offset ); +} + +// Does this record belong to the body the walk is standing in? The node first, +// which rejects almost everything in one compare, then the step pairs. +inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * record, const TableRetainPath & path ) +{ + if ( TableRetainRecordDepth( record ) != path.depth ) { return false; } + if ( TableRetainRecordAt( retain, record ) != path.at ) { return false; } + const uint8_t * steps = TableRetainRecordSteps( record ); + for ( int32_t i = 0; i < path.depth; i++ ) + { + if ( TableRetainRead32( steps + 8 * i ) != path.steps[i].ordinal ) { return false; } + if ( TableRetainRead32( steps + 8 * i + 4 ) != path.steps[i].index ) { return false; } + } + return true; +} + +// EVERY ID THIS BUILD CAN NAME, ascending: the set TableIds's capacity is +// derived from. An id inside a retained record takes its trailer entry from +// the GENERATED table when it is here and from the CALLER's list otherwise, so +// no retained id ever enters the generated table and no id is written twice. +static const int32_t kTableRetainKnownIds = 76; +static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, + 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, + 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, + 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, + 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, +}; + +inline bool TableRetainNameable( uint64_t id ) +{ + int32_t low = 0, high = kTableRetainKnownIds - 1; + while ( low <= high ) + { + const int32_t mid = low + ( high - low ) / 2; + if ( kTableRetainKnown[mid] == id ) { return true; } + if ( kTableRetainKnown[mid] < id ) { low = mid + 1; } else { high = mid - 1; } + } + return false; +} + +// THE TWO STORES, NUMBERED INTO ONE TRAILER in merged first-use order (§6.6). +// It answers the surface TableIds answers, ref, count, truncate and +// overflow, so the retain family's codec is the plain one with its names +// changed, and +// the GENERATED TABLE IS UNTOUCHED: its capacity, its overflow rule and its +// -1 stand exactly as they are for every save. +struct TableRetainIds +{ + TableIds known; + int32_t known_slot[ TableIds::kCapacity ]; + TableRetain * retain; + int32_t count; + bool overflow; + bool lost; // a retained id past the caller's capacity: the record is dropped + + TableRetainIds( TableRetain * to_retain ) : retain( to_retain ), count( 0 ), overflow( false ), lost( false ) {} + + // an id this build CAN name, which is every id the generated codec writes + uint64_t ref( uint64_t id ) + { + const int32_t before = known.count; + const uint64_t k = known.ref( id ); + if ( known.overflow ) { overflow = true; return 1; } + if ( known.count != before ) { known_slot[ (int32_t) k - 1 ] = ++count; } + return (uint64_t) known_slot[ (int32_t) k - 1 ]; + } + + // an id from INSIDE a retained record. A retained id takes its entry from + // the caller's list, and one past the capacity sets lost: the record is + // dropped, nothing else about the save changes, and the save is never + // refused (§6.6). + uint64_t record_ref( uint64_t id ) + { + if ( TableRetainNameable( id ) ) { return ref( id ); } + if ( retain == NULL ) { lost = true; return 0; } + for ( int32_t i = 0; i < retain->id_used; i++ ) + { + if ( retain->ids[i].id == id ) { return (uint64_t) retain->ids[i].slot; } + } + if ( retain->id_used >= retain->id_capacity ) { lost = true; return 0; } + retain->ids[ retain->id_used ].id = id; + retain->ids[ retain->id_used ].slot = ++count; + retain->id_used++; + return (uint64_t) count; + } + + // undo every entry taken since mark, in either store. Both are appended in + // slot order, so an entry removed is the last one of its store. + void truncate( int32_t mark ) + { + while ( known.count > 0 && known_slot[ known.count - 1 ] > mark ) { known.truncate( known.count - 1 ); } + while ( retain != NULL && retain->id_used > 0 && retain->ids[ retain->id_used - 1 ].slot > mark ) { retain->id_used--; } + count = mark; + } +}; + +// THE FILE STILL CARRIES ONE ID TABLE (§3): the split is the writer's storage +// rather than the wire's, and the trailer is one merge of two slot-ordered +// stores. +inline int64_t TableRetainIdsBytes( const TableRetainIds & ids ) { return int64_t( ids.count ) * 8 + 8; } + +// A TWO-WAY MERGE over the stores' own slot order, and not a scan for each +// slot. Every entry either store holds took its slot from the same counter, so +// the two runs interleave to exactly the slots 1 to count and the merge has no +// case for a slot neither store took. +inline void TableRetainIdsWrite( TableWriter & w, const TableRetainIds & ids ) +{ + const int32_t retained = ids.retain != NULL ? ids.retain->id_used : 0; + int32_t i = 0, j = 0; + while ( i < ids.known.count || j < retained ) + { + if ( j >= retained || ( i < ids.known.count && ids.known_slot[i] < ids.retain->ids[j].slot ) ) + { + w.put64( ids.known.ids[i] ); + i++; + continue; + } + w.put64( ids.retain->ids[j].id ); + j++; + } + w.put64( uint64_t( ids.count ) ); +} + +// ---- THE RESOLVING WALK (§6.6) ---- +// +// A reference names a SLOT of the file's id table, so a verbatim copy +// re-emitted into a file whose table is ordered differently would point at +// other names in silence. A retained record therefore holds the field with +// every reference replaced by the sixty-four-bit id it names, and every length +// that frames a rewritten reference recomputed. +// +// THE WALK IS AN INTERPRETATION, AND ITS VERDICT IS STATED: it reads kind +// bytes, lengths and references and nothing else. No value is decoded, no +// bound is checked, no branch is taken on a payload byte, and anything it +// cannot frame DROPS THE RECORD, counts one retain_lost, and never raises +// malformed on the plain read. +// +// THE WALK IS ONE PASS EACH WAY, and its cost is linear in the record's own +// bytes. Every length that frames a content in the resolved form is a fixed +// slot rather than a canonical LEB128, so the capture reserves it, writes the +// content, and fills the slot in behind it. A spelling that had to know the +// resolved size before writing it would have to walk each content twice, once +// at every level, and the file chooses the nesting. +// +// A retained record's inner nesting is the WRITER's and not this build's, so +// it is the one depth on this path a file can drive. The cap counts NESTED +// BODIES, and a record past it is dropped on the same rule as any other shape +// the walk cannot take. Time no longer rests on it: it is a small stated +// constant and nothing more. +static const int32_t kTableRetainWalkDepthMax = 64; + +// the three RESERVED ids (§3.1, §3.3). One inside a retained record's payload +// would be re-emitted into a nested body, where it is malformed, so meeting +// one drops the record. +inline bool TableRetainReservedId( uint64_t id ) +{ + return id == kTableNodeTableFieldId || id == kTableBuildVersionFieldId || id == kTableMessageVocabularyFieldId; +} + +struct TableRetainIn +{ + const uint8_t * in; + int64_t size; + int64_t at; + const TableIdTable * ids; + uint8_t * out; // NULL: measuring, and nothing is written + int64_t out_at; +}; + +inline void TableRetainInRaw( TableRetainIn & s, const uint8_t * from, int64_t bytes ) +{ + if ( s.out != NULL ) { memcpy( s.out + s.out_at, from, (size_t) bytes ); } + s.out_at += bytes; +} + +inline void TableRetainInLeb( TableRetainIn & s, uint64_t v ) +{ + uint8_t b[10]; + int64_t n = 0; + while ( v >= 0x80 ) { b[n++] = uint8_t( v ) | 0x80; v >>= 7; } + b[n++] = uint8_t( v ); + TableRetainInRaw( s, b, n ); +} + +inline void TableRetainInId( TableRetainIn & s, uint64_t id ) +{ + uint8_t b[8]; + TableRetainWrite64( b, id ); + TableRetainInRaw( s, b, 8 ); +} + +inline bool TableRetainInLebRead( TableRetainIn & s, uint64_t & value ) +{ + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( s.at >= s.size ) { return false; } + const uint8_t b = s.in[ s.at++ ]; + if ( i == 9 && b > 1 ) { return false; } + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) { return i == 0 || b != 0; } + shift += 7; + } + return false; +} + +// one REFERENCE resolved to the id it names. A zero reference is the wire's +// own "no id", the enum's None and the union's empty arm, and rides as the +// id zero. A reference above the entry count, a reference at an id-table entry +// of zero, and a reference at a reserved id are each damage the plain read +// never looked at, and each drops the record. +inline bool TableRetainInRef( TableRetainIn & s, bool zero_allowed ) +{ + uint64_t ref = 0; + if ( !TableRetainInLebRead( s, ref ) ) { return false; } + if ( ref == 0 ) + { + if ( !zero_allowed ) { return false; } + TableRetainInId( s, 0 ); + return true; + } + if ( s.ids == NULL || ref > (uint64_t) s.ids->count ) { return false; } + const uint64_t id = s.ids->at( ref ); + if ( id == 0 || TableRetainReservedId( id ) ) { return false; } + TableRetainInId( s, id ); + return true; +} + +inline int64_t TableRetainInPayload( TableRetainIn & s, uint8_t kind, int32_t depth ); +inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ); + +// ONE FRAMED LENGTH in the resolved form: a pair of fixed u32. The first is +// the RESOLVED byte count of the content it frames, reserved here and written +// once the content is out. The second is the SAVE's scratch, left zero by the +// capture and filled by the walk that emits. +// +// The record is the reader's own storage and nothing outside this family ever +// reads it, so a length may be written after the bytes it measures. That is +// the whole of what makes the walk one pass. +static const int64_t kTableRetainSlotBytes = 8; + +inline int64_t TableRetainInSlot( TableRetainIn & s ) +{ + const int64_t at = s.out_at; + const uint8_t zero[ kTableRetainSlotBytes ] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + TableRetainInRaw( s, zero, kTableRetainSlotBytes ); + return at; +} + +inline void TableRetainInPatch( TableRetainIn & s, int64_t slot, int64_t resolved ) +{ + if ( s.out != NULL ) { TableRetainWrite32( s.out + slot, (uint32_t) resolved ); } +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it. +// The measuring pass takes the same path and reserves the same fixed width, so +// the size it answers is the size the writing pass lays down. +inline int64_t TableRetainInFramed( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s ); + const int64_t resolved = TableRetainInContent( s, kind, length, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s, slot, resolved ); + return resolved; +} + +inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + if ( length < 0 || s.at + length > s.size ) { return -1; } + const int64_t end = s.at + length; + const int64_t began = s.out_at; + switch ( kind ) + { + case 13: // a table BODY: fields, then the zero reference + { + for ( ;; ) + { + uint64_t ref = 0; + const int64_t mark = s.at; + if ( !TableRetainInLebRead( s, ref ) ) { return -1; } + // THE TERMINATOR IS A REFERENCE, and a reference in the + // resolved form is a fixed eight-byte id: the zero that ends a + // body rides at the width every other one does. + if ( ref == 0 ) { TableRetainInId( s, 0 ); break; } + s.at = mark; + if ( !TableRetainInRef( s, false ) ) { return -1; } + if ( s.at >= end ) { return -1; } + const uint8_t field_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &field_kind, 1 ); + if ( TableRetainInPayload( s, field_kind, depth ) < 0 ) { return -1; } + if ( s.at > end ) { return -1; } + } + break; + } + case 14: // an ARRAY body: the element kind, the count, then the elements + { + if ( s.at >= end ) { return -1; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainInLebRead( s, n ) ) { return -1; } + TableRetainInLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( TableRetainInPayload( s, elem_kind, depth ) < 0 ) { return -1; } + if ( s.at > end ) { return -1; } + } + break; + } + case 16: // an ENUM-KEYED body: N triples of a KEY REFERENCE, an L and the element + { + if ( s.at >= end ) { return -1; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainInLebRead( s, n ) ) { return -1; } + TableRetainInLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + // A KEYED BODY'S KEYS RESOLVE AT EVERY ELEMENT KIND (§6.6, §3.2) + if ( !TableRetainInRef( s, false ) ) { return -1; } + uint64_t slot_bytes = 0; + if ( !TableRetainInLebRead( s, slot_bytes ) ) { return -1; } + if ( slot_bytes > (uint64_t) ( end - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, elem_kind, (int64_t) slot_bytes, depth + 1 ) < 0 ) { return -1; } + } + break; + } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + default: + // every other content is bytes: a string, wide text, an escape, a + // payload-free kind, a scalar under a keyed slot's own length + TableRetainInRaw( s, s.in + s.at, length ); + s.at += length; + break; + } + if ( s.at != end ) { return -1; } + return s.out_at - began; +} + +// the depth a payload carries is its enclosing body's: only a framed CONTENT +// is a level, and TableRetainInContent is the one place the cap is read. +inline int64_t TableRetainInPayload( TableRetainIn & s, uint8_t kind, int32_t depth ) +{ + const int64_t began = s.out_at; + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: // the fixed-width kinds, by width + case 3: case 7: case 21: case 26: + case 4: case 8: case 10: case 22: case 27: + case 5: case 9: case 11: case 23: case 28: + case 18: case 19: case 24: case 29: + { + int64_t width = 1; + switch ( kind ) + { + case 3: case 7: case 21: case 26: width = 2; break; + case 4: case 8: case 10: case 22: case 27: width = 4; break; + case 5: case 9: case 11: case 23: case 28: width = 8; break; + case 18: case 19: case 24: case 29: width = 16; break; + default: width = 1; break; + } + if ( s.at + width > s.size ) { return -1; } + TableRetainInRaw( s, s.in + s.at, width ); + s.at += width; + break; + } + case 12: case 31: case 32: case 33: // L, then L bytes, nothing framed inside + { + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + TableRetainInLeb( s, length ); + TableRetainInRaw( s, s.in + s.at, (int64_t) length ); + s.at += (int64_t) length; + break; + } + case 13: case 14: case 16: // L, then a body the walk resolves + { + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, kind, (int64_t) length, depth + 1 ) < 0 ) { return -1; } + break; + } + case 15: // a UNION: the arm id reference, and when it is not zero its kind, L and payload + { + const int64_t mark = s.at; + uint64_t arm = 0; + if ( !TableRetainInLebRead( s, arm ) ) { return -1; } + s.at = mark; + if ( !TableRetainInRef( s, true ) ) { return -1; } + if ( arm == 0 ) { break; } + if ( s.at >= s.size ) { return -1; } + const uint8_t arm_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &arm_kind, 1 ); + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, arm_kind, (int64_t) length, depth + 1 ) < 0 ) { return -1; } + break; + } + case 30: // an ENUM's variant reference, zero for None + { + if ( !TableRetainInRef( s, true ) ) { return -1; } + break; + } + case 17: return -1; // A NODE INDEX (§3.1): the whole record goes with it + default: return -1; // a kind this walk cannot frame + } + return s.out_at - began; +} + +// ---- CAPTURE: the load side (§6.6) ---- +// +// The field is skipped by its framing exactly as it always was and counted +// unknown exactly as it always was, so a full buffer degrades to the default +// behavior one field at a time. False is what r.skip( kind ) answers false +// for, and nothing else: retention can lose a field, it can never turn a good +// read into a bad one. +inline bool TableRetainCapture( TableRetain * retain, TableReader & r, const TableRetainPath & path, + uint64_t field_id, uint8_t kind ) +{ + const int64_t start = r.offset; + if ( !r.skip( kind ) ) { return false; } + if ( retain == NULL ) { return true; } + const int64_t wire_bytes = r.offset - start; + + TableRetainIn probe; + probe.in = r.buffer + start; + probe.size = wire_bytes; + probe.at = 0; + probe.ids = r.ids; + probe.out = NULL; + probe.out_at = 0; + const int64_t payload = TableRetainInPayload( probe, kind, 0 ); + if ( payload < 0 || probe.at != wire_bytes ) { r.report->retain_lost++; return true; } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL: the buffer never holds a + // truncated field, and the read continues (§6.6) + r.report->retain_lost++; + return true; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, field_id ); + record[24] = kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableRetainIn write; + write.in = r.buffer + start; + write.size = wire_bytes; + write.at = 0; + write.ids = r.ids; + write.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out_at = 0; + if ( TableRetainInPayload( write, kind, 0 ) < 0 ) { r.report->retain_lost++; return true; } + retain->used += need; + retain->count++; + r.report->retained++; + return true; +} + +// LoadRetain RESETS BOTH STORES and writes into neither list (§6.6): a +// retained record carries its field's identity in the record itself, with +// every reference resolved. +inline void TableRetainReset( TableRetain * retain, const TableNodeMap & nodes, const uint8_t * region ) +{ + if ( retain == NULL ) { return; } + retain->used = 0; + retain->id_used = 0; + retain->count = 0; + retain->base = region; + retain->directory = nodes.entries; + retain->directory_count = nodes.count; +} + +// ---- RECORD LIFETIME (docs/SPEC-TABLES.md §6.6) ---- +// +// A RETAINED RECORD BELONGS TO THE BODY OCCURRENCE THAT CARRIED IT, AND DIES +// WITH IT. Legal input can carry a known child body twice, and the later +// occurrence resets the child and wins whole (§3, §4): the records retained +// under the earlier occurrence go with the values it held. The discard moves +// NEITHER counter. The writer superseded the data, so nothing was lost that +// the load could have kept, and retained counted the record when its bytes +// were kept and does not fall when they are let go. +// +// The occurrences are four, and each is a body the wire lets a writer put down +// again: a repeated TABLE field, by value or under ?, a UNION whose arm is +// written again, a MAP's duplicate key, and a KEYED-ARRAY slot written again. +// The FIELD form covers the three where the field itself is read again, arm +// switches and shrinking arrays included; the BODY form covers a duplicate key +// inside one occurrence of a map, where the field is read once and the entry +// twice. +inline bool TableRetainUnder( const uint8_t * record, const void * at, const TableRetain & retain, + const TableRetainPath & path, bool field, uint32_t ordinal ) +{ + if ( TableRetainRecordAt( retain, record ) != at ) { return false; } + const int32_t depth = TableRetainRecordDepth( record ); + if ( field ) + { + if ( depth <= path.depth ) { return false; } + } + else if ( depth < path.depth ) { return false; } + const uint8_t * steps = TableRetainRecordSteps( record ); + for ( int32_t i = 0; i < path.depth; i++ ) + { + if ( TableRetainRead32( steps + 8 * i ) != path.steps[i].ordinal ) { return false; } + if ( TableRetainRead32( steps + 8 * i + 4 ) != path.steps[i].index ) { return false; } + } + if ( field && TableRetainRead32( steps + 8 * path.depth ) != ordinal ) { return false; } + return true; +} + +inline void TableRetainDiscard( TableRetain * retain, const TableRetainPath & path, bool field, uint32_t ordinal ) +{ + if ( retain == NULL || retain->count == 0 ) { return; } + int64_t read = 0, write = 0; + int32_t kept = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + const int64_t bytes = TableRetainRecordBytes( retain->bytes + read ); + if ( !TableRetainUnder( retain->bytes + read, path.at, *retain, path, field, ordinal ) ) + { + if ( write != read ) { memmove( retain->bytes + write, retain->bytes + read, (size_t) bytes ); } + write += bytes; + kept++; + } + read += bytes; + } + retain->used = write; + retain->count = kept; +} + +inline void TableRetainDiscardBody( TableRetain * retain, const TableRetainPath & path ) +{ + TableRetainDiscard( retain, path, false, 0 ); +} + +inline void TableRetainDiscardField( TableRetain * retain, const TableRetainPath & path, uint32_t ordinal ) +{ + TableRetainDiscard( retain, path, true, ordinal ); +} + +// ---- EMIT: the save side (§6.6) ---- +// +// The record read back the other way: every resolved id becomes the reference +// the trailer being written gives it, and every length is recomputed against +// the references' new widths. The walk is the capture's mirror and the same +// damage rules apply, except that damage cannot be met: these bytes are the +// reader's own. +// +// A WIRE LENGTH IS CANONICAL LEB128 AND RIDES BEFORE ITS CONTENT, so this side +// cannot fill a slot in behind the bytes the way the capture does. It takes +// one POST-ORDER pass instead: measuring computes each content's wire size and +// leaves it in that content's own scratch slot, and the emit reads the size +// there rather than walking for it. Measuring runs immediately before the +// emit, on the same record and the same id table, which is what makes the two +// readings one walk. +struct TableRetainOut +{ + uint8_t * in; // the record: only a framed length's scratch half is written + int64_t size; + int64_t at; + TableRetainIds * ids; + TableWriter * w; // NULL: measuring, and the scratch slots are being filled + int64_t bytes; +}; + +inline void TableRetainOutRaw( TableRetainOut & s, const uint8_t * from, int64_t bytes ) +{ + if ( s.w != NULL ) { s.w->raw( from, bytes ); } + s.bytes += bytes; +} + +inline void TableRetainOutLeb( TableRetainOut & s, uint64_t v ) +{ + if ( s.w != NULL ) { s.w->putleb( v ); } + s.bytes += TableLebBytes( v ); +} + +inline bool TableRetainOutLebRead( TableRetainOut & s, uint64_t & value ) +{ + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( s.at >= s.size ) { return false; } + const uint8_t b = s.in[ s.at++ ]; + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) { return true; } + shift += 7; + } + return false; +} + +inline bool TableRetainOutRef( TableRetainOut & s ) +{ + if ( s.at + 8 > s.size ) { return false; } + const uint64_t id = TableRetainRead64( s.in + s.at ); + s.at += 8; + if ( id == 0 ) { TableRetainOutLeb( s, 0 ); return true; } // the wire's own no-id + const uint64_t ref = s.ids->record_ref( id ); + if ( s.ids->lost || s.ids->overflow ) { return false; } + TableRetainOutLeb( s, ref ); + return true; +} + +inline bool TableRetainOutPayload( TableRetainOut & s, uint8_t kind, int32_t depth ); +inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t length, int32_t depth ); + +// ONE FRAMED CONTENT, read out of the record's fixed slot and written with the +// canonical LEB128 length this wire wants. The ids it names are interned on +// the way past, which is what makes measure and save one walk in two readings, +// exactly as every other body on this wire is. +// +// MEASURING walks the content, then leaves the wire size it found in the +// scratch half of the slot. EMITTING reads that size, writes it, and CHECKS +// the content against it: a size no measure of this save left there is a +// record refused rather than a length that does not frame what follows. +inline bool TableRetainOutFramed( TableRetainOut & s, uint8_t kind, int32_t depth ) +{ + if ( s.at + kTableRetainSlotBytes > s.size ) { return false; } + uint8_t * const slot = s.in + s.at; + const int64_t resolved = (int64_t) TableRetainRead32( slot ); + s.at += kTableRetainSlotBytes; + if ( s.w == NULL ) + { + const int64_t began = s.bytes; + if ( !TableRetainOutContent( s, kind, resolved, depth ) ) { return false; } + const int64_t wire = s.bytes - began; + if ( wire > 0xFFFFFFFFll ) { return false; } + TableRetainWrite32( slot + 4, (uint32_t) wire ); + s.bytes += TableLebBytes( (uint64_t) wire ); + return true; + } + const int64_t wire = (int64_t) TableRetainRead32( slot + 4 ); + TableRetainOutLeb( s, (uint64_t) wire ); + const int64_t began = s.bytes; + if ( !TableRetainOutContent( s, kind, resolved, depth ) ) { return false; } + return s.bytes - began == wire; +} + +inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t length, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return false; } + if ( length < 0 || s.at + length > s.size ) { return false; } + const int64_t end = s.at + length; + switch ( kind ) + { + case 13: + { + for ( ;; ) + { + if ( s.at + 8 > end ) { return false; } + const uint64_t id = TableRetainRead64( s.in + s.at ); + if ( id == 0 ) { s.at += 8; TableRetainOutLeb( s, 0 ); break; } + if ( !TableRetainOutRef( s ) ) { return false; } + if ( s.at >= end ) { return false; } + const uint8_t field_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &field_kind, 1 ); + if ( !TableRetainOutPayload( s, field_kind, depth ) ) { return false; } + } + break; + } + case 14: + { + if ( s.at >= end ) { return false; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainOutLebRead( s, n ) ) { return false; } + TableRetainOutLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( !TableRetainOutPayload( s, elem_kind, depth ) ) { return false; } + } + break; + } + case 16: + { + if ( s.at >= end ) { return false; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainOutLebRead( s, n ) ) { return false; } + TableRetainOutLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( !TableRetainOutRef( s ) ) { return false; } + if ( !TableRetainOutFramed( s, elem_kind, depth + 1 ) ) { return false; } + } + break; + } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; + default: + TableRetainOutRaw( s, s.in + s.at, length ); + s.at += length; + break; + } + return s.at == end; +} + +// the depth a payload carries is its enclosing body's, exactly as on the +// capture side: only a framed CONTENT is a level. +inline bool TableRetainOutPayload( TableRetainOut & s, uint8_t kind, int32_t depth ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: + case 3: case 7: case 21: case 26: + case 4: case 8: case 10: case 22: case 27: + case 5: case 9: case 11: case 23: case 28: + case 18: case 19: case 24: case 29: + { + int64_t width = 1; + switch ( kind ) + { + case 3: case 7: case 21: case 26: width = 2; break; + case 4: case 8: case 10: case 22: case 27: width = 4; break; + case 5: case 9: case 11: case 23: case 28: width = 8; break; + case 18: case 19: case 24: case 29: width = 16; break; + default: width = 1; break; + } + if ( s.at + width > s.size ) { return false; } + TableRetainOutRaw( s, s.in + s.at, width ); + s.at += width; + break; + } + case 12: case 31: case 32: case 33: + { + uint64_t length = 0; + if ( !TableRetainOutLebRead( s, length ) ) { return false; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return false; } + TableRetainOutLeb( s, length ); + TableRetainOutRaw( s, s.in + s.at, (int64_t) length ); + s.at += (int64_t) length; + break; + } + case 13: case 14: case 16: + { + if ( !TableRetainOutFramed( s, kind, depth + 1 ) ) { return false; } + break; + } + case 15: + { + if ( s.at + 8 > s.size ) { return false; } + const uint64_t arm = TableRetainRead64( s.in + s.at ); + if ( !TableRetainOutRef( s ) ) { return false; } + if ( arm == 0 ) { break; } + if ( s.at >= s.size ) { return false; } + const uint8_t arm_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &arm_kind, 1 ); + if ( !TableRetainOutFramed( s, arm_kind, depth + 1 ) ) { return false; } + break; + } + case 30: + { + if ( !TableRetainOutRef( s ) ) { return false; } + break; + } + default: return false; + } + return true; +} + +// ---- THE RETAINED TAIL: where the records go back (§6.6) ---- +// +// AT THE END OF THEIR OWN BODY, IN THE ORDER RETAINED. Position carries +// nothing on this wire, so appending is chosen for three properties: it is a +// write with no splice, the retained order is preserved, and the result is +// IDEMPOTENT after the first save. +// +// A RETAINED ID PAST THE CAPACITY COUNTS ONE retain_lost AND ITS RECORD IS +// DROPPED, and the save is never refused. MeasureRetain and SaveRetain drop +// the same records under the same walk, so the measure sees the same overflow +// and its answer is the size the save writes. + +// one record's WIRE bytes under the trailer being written, and -1 for a record +// this save cannot place: an id the caller's list had no room for, or a +// resolved form the walk cannot read back. The ids it names are interned on +// the way past, which is what makes measure and save one rule read twice. +// +// THIS IS THE MEASURING PASS, and it leaves every framed content's wire size +// in that content's own scratch slot. The record is the caller's buffer and +// the pass writes nothing else into it. +inline int64_t TableRetainRecordWire( uint8_t * record, TableRetainIds & ids, uint64_t & ref ) +{ + const int32_t mark = ids.count; + ids.lost = false; + ref = ids.record_ref( TableRetainRecordId( record ) ); + if ( !ids.lost && !ids.overflow ) + { + TableRetainOut s; + s.in = TableRetainRecordPayload( record ); + s.size = TableRetainRecordPayloadBytes( record ); + s.at = 0; + s.ids = &ids; + s.w = NULL; + s.bytes = 0; + if ( TableRetainOutPayload( s, TableRetainRecordKind( record ), 0 ) && s.at == s.size ) + { + return TableLebBytes( ref ) + 1 + s.bytes; + } + } + // the record is not written at all, and nothing else about the save + // changes: a full id list degrades to the default behavior one record at a + // time, and the entries this attempt took are given back + ids.truncate( mark ); + ids.lost = false; + return -1; +} + +inline int64_t TableRetainTailMeasure( TableRetain * retain, TableRetainIds & ids, const TableRetainPath & path ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return 0; } + int64_t bytes = 0; + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordHere( *retain, record, path ) ) { continue; } + uint64_t ref = 0; + const int64_t wire = TableRetainRecordWire( record, ids, ref ); + if ( wire < 0 ) { continue; } + bytes += wire; + } + return bytes; +} + +inline bool TableRetainTailSave( TableRetain * retain, TableRetainIds & ids, TableWriter & w, const TableRetainPath & path ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return true; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordHere( *retain, record, path ) ) { continue; } + uint64_t ref = 0; + if ( TableRetainRecordWire( record, ids, ref ) < 0 ) { continue; } + w.putleb( ref ); + w.put8( TableRetainRecordKind( record ) ); + TableRetainOut s; + s.in = TableRetainRecordPayload( record ); + s.size = TableRetainRecordPayloadBytes( record ); + s.at = 0; + s.ids = &ids; + s.w = &w; + s.bytes = 0; + if ( !TableRetainOutPayload( s, TableRetainRecordKind( record ), 0 ) ) { return false; } + record[25] = 1; // PLACED: the one mark the save leaves on the buffer + } + return !w.overflow; +} + +// THE SAVE'S OWN SHARE OF retain_lost, counted ONCE and read after the save +// (§6.6): every record the walk did not place. A record whose path no longer +// names a body, one the caller's id list had no room for, and one the walk +// could not read back are one number here, because the check a caller reads is +// one number. A record is marked as it is written, so this cannot double-count +// a body measured twice. +inline void TableRetainClearPlaced( TableRetain * retain ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + retain->bytes[ at + 25 ] = 0; + at += TableRetainRecordBytes( retain->bytes + at ); + } +} + +inline void TableRetainCountLost( const TableRetain * retain, TableReport * report ) +{ + if ( retain == NULL || retain->bytes == NULL || report == NULL ) { return; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + const uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordPlaced( record ) ) { report->retain_lost++; } + } +} + +// THE NODE TABLE under retention (§3.1, §6.6): the same fill rule the plain +// pair derives, with the retain family's ids and each record's own body +// reached through a dispatch the CALL supplies rather than a second pair of +// thunks on the numbering. A store per node on the PLAIN save path would be a +// cost this feature is not allowed to have. +template +inline int64_t TableNodeTablePayloadRetain( const Ctx & ctx, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure ) +{ + int64_t payload = TableLebBytes( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + payload += TableLebBytes( ids.ref( n.entries[k].type_id ) ); + const int64_t body = measure( ctx, n, ids, n.entries[k].type_id, n.entries[k].node, retain ); + if ( body < 0 ) { return -1; } + payload += TableLebBytes( (uint64_t) body ) + body; + } + return payload; +} + +template +inline int64_t TableNodeTableMeasureRetain( const Ctx & ctx, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure ) +{ + if ( n.count == 0 ) { return 0; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayloadRetain( ctx, ids, n, retain, measure ); + if ( payload < 0 ) { return -1; } + return TableLebBytes( ref ) + 1 + TableLebBytes( (uint64_t) payload ) + payload; +} + +template +inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure, Save save ) +{ + if ( n.count == 0 ) { return true; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayloadRetain( ctx, ids, n, retain, measure ); + if ( payload < 0 ) { return false; } + w.putleb( ref ); + w.put8( 12 ); // kind 12 is the opaque byte payload, exactly as the plain save writes it + w.putleb( (uint64_t) payload ); + w.putleb( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.putleb( ids.ref( n.entries[k].type_id ) ); + const int64_t body = measure( ctx, n, ids, n.entries[k].type_id, n.entries[k].node, retain ); + if ( body < 0 ) { return false; } + w.putleb( (uint64_t) body ); + if ( !save( ctx, n, w, ids, n.entries[k].type_id, n.entries[k].node, retain ) ) { return false; } + } + return true; +} +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_RETAIN + +#ifndef MAPDEMO_SCHEMA_TABLE_EXTENT +#define MAPDEMO_SCHEMA_TABLE_EXTENT + +namespace mapdemo { + +// ---- the NODE EXTENT: where a map's entries and a list's elements live (§2.8, §2.9) ---- + +// TableExtentCarve is a node's extent cursor, PRE-ORDER: a container's whole +// array first, then, element by element in the container's own order, the +// arrays of any list or map an element holds by value. The cursor is the node +// map's, because the generated decoder is threaded with that and not with a +// region. +struct TableExtentCarve +{ + uint8_t * at = NULL; // the region path: the node's extent, unspent + int64_t left = 0; + TableWorker * worker = NULL; // the TOOL's path: the arrays come from the arena +}; + +// AN UNREACHED SLOT MUST HOLD NO LIST OR MAP WITH ELEMENTS IN IT (§2.8, §2.9, +// §7.6). An empty one takes no bytes, so a record whose extent measures ZERO is +// a record whose every by-value list and map is empty. A measure that REFUSED +// answers non-zero here too, and refusing on it is the same answer one level up. +inline bool TableExtentUnreachedEmpty( int64_t extent ) { return extent == 0; } + +// ---- LoadMeasure's framing walk (§6.5) ---- +// +// The measure reads no field value: it walks each record's field headers, +// skipping every payload by its framing, to reach each N at every depth. A +// false is a REFUSAL, and it carries its reason (§6.5). +typedef bool ( * TableWireExtentFn )( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ); + +// the framing walk over an ARRAY OF TABLES held by value: its elements' own +// lists and maps are part of this node's extent too +inline bool TableWireExtentElements( const uint8_t * body, int64_t length, int64_t & at, TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } + if ( r.get8() != 13 ) { return true; } + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +// and over an ENUM-KEYED array, whose triples carry a key REFERENCE before each +// length-prefixed element (docs/SPEC-TABLES.md §3.2) +inline bool TableWireExtentKeyed( const uint8_t * body, int64_t length, int64_t & at, TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } + if ( r.get8() != 13 ) { return true; } + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t key = 0; + if ( !r.getleb( key ) ) { return true; } + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_EXTENT + +#ifndef MAPDEMO_SCHEMA_TABLE_MAP +#define MAPDEMO_SCHEMA_TABLE_MAP + +namespace mapdemo { + +// ---- a MAP: a sorted entry array, and the lookup over it (§2.8) ---- +// +// On the wire, in a region and in a cook a map is an array of one generated +// ENTRY table held in ascending key order. What this adds is Find — a binary +// search over that array where it lies — and a builder that inserts, replaces +// and erases by key. Nothing here is stored: a region and a cook carry the +// array and the count, and not one byte about a hash or a probe. + +// entries carved from ONE call to the allocator pair; a new segment is +// appended when the current one fills, and nothing ever moves (§6.4) +static const int32_t kTableMapSegmentEntries = 32; + +// TableDeclRef names a type in an unevaluated context and is never defined — +// what 's declval is for, without the include the generated corpus +// refuses to pay for (the iterator_traits note, §13.9). +template T & TableDeclRef(); + +// THE ORDER IS TOTAL, AND IT IS THE SAME IN NINE LANGUAGES (§2.8). Integers +// compare by VALUE, signed for the signed kinds and unsigned for the unsigned. +// Strings compare by BYTES, unsigned, a shorter string that is a prefix of a +// longer one first: memcmp over the common length, then the lengths. Never a +// locale, never a code point, never a case fold. +inline int TableKeyOrder( uint64_t a, uint64_t b ) { return a < b ? -1 : ( a > b ? 1 : 0 ); } +inline int TableKeyOrder( int64_t a, int64_t b ) { return a < b ? -1 : ( a > b ? 1 : 0 ); } +inline int TableKeyOrder( const char * a, int32_t a_length, const char * b, int32_t b_length ) +{ + const int32_t common = a_length < b_length ? a_length : b_length; + if ( common > 0 ) + { + const int order = memcmp( (const void *) a, (const void *) b, (size_t) common ); + if ( order != 0 ) { return order < 0 ? -1 : 1; } + } + return a_length < b_length ? -1 : ( a_length > b_length ? 1 : 0 ); +} + +// the length of a NUL-terminated key at a call site, bounded by the storage it +// has to fit: a key one byte longer than the bound is refused, never truncated +inline int32_t TableKeyLength( const char * key, int32_t bound ) +{ + if ( key == NULL ) { return 0; } + for ( int32_t i = 0; i <= bound; i++ ) { if ( key[i] == 0 ) { return i; } } + return bound + 1; // longer than the bound: the caller refuses it +} + +// A KEY IS DATA AND A LENGTH, and the length is CARRIED, never recomputed +// (§2.8, §3). A string(N) key holds any byte a wire or a text can spell, +// U+0000 included, so a lookup that measures to the first NUL answers that "a" +// and "a", 0, "b" are the same key: the first entry is found, RESET, and +// relabeled with the second key, which deletes an entry the report never +// mentions. Every internal lookup and every insertion takes this pair, and the +// public const char * surface builds one and is a wrapper over it. +struct TableMapKeyRef +{ + const char * data; + int32_t length; +}; + +// ---- the storage: SIXTEEN BYTES in the holder's record (§2.8, §7.2) ---- +// +// An int64 self-relative reference to the entry array and an int32 count, then +// padding to eight. The reference is a TableRef like a pointer's: in the arena +// it names the builder's HEAD, in a region it is the delta from the slot to +// the first entry, and 0 is the empty map in both. +template struct TableMap +{ + TableRef entries; + int32_t count = 0; // the LIVE count, in both forms + int32_t padding = 0; // named, so the record has no unwritten byte in it + + // ---- the CONST form: a locked region, a loaded one, an opened cook ---- + // + // One surface over one encoding (§6.3). A region reference resolves from + // the slot's own address, so every one of these is a member and needs no + // base and no context. + const Entry * Entries() const + { + return entries.value != 0 ? (const Entry *) ( (const uint8_t *) &entries + entries.value ) : NULL; + } + int32_t size() const { return count; } + + // FIND: floor( log2 n ) + 1 key compares, in place, no allocation. NULL + // when absent, and on a map[K]*T the RESOLVED pointer, which is what a + // pointer field's accessor answers. + template const Entry * FindEntry( Key key ) const + { + const Entry * base = Entries(); + int32_t low = 0, high = count; + while ( low < high ) + { + const int32_t mid = low + ( high - low ) / 2; + const int order = TableEntryOrder( base[mid], key ); + if ( order == 0 ) { return base + mid; } + if ( order < 0 ) { low = mid + 1; } else { high = mid; } + } + return NULL; + } + // the return type is DEDUCED, so it is worked out when a call site + // instantiates Find and not when the holder's record declares the slot — + // which is what lets the entry's own overloads be declared after it + template auto Find( Key key ) const + { + return TableEntryFound( FindEntry( key ) ); + } + + // ---- iteration: ASCENDING key order, the key beside the value ---- + // + // A proxy BY VALUE, the keyed array's shape (§2.4): for ( auto [ key, + // value ] : map ). It carries no iterator_traits, for the reason + // TableKeyed's does not (§13.9). + struct ConstEntry + { + decltype( TableEntryKey( TableDeclRef() ) ) key; + decltype( TableEntryFound( (const Entry *) NULL ) ) value; + }; + + struct ConstIterator + { + const Entry * at; + ConstEntry operator*() const { return ConstEntry{ TableEntryKey( *at ), TableEntryFound( at ) }; } + ConstIterator & operator++() { at++; return *this; } + bool operator==( const ConstIterator & other ) const { return at == other.at; } + bool operator!=( const ConstIterator & other ) const { return at != other.at; } + }; + + ConstIterator begin() const { return ConstIterator{ Entries() }; } + ConstIterator end() const { return ConstIterator{ Entries() + count }; } +}; + +// ---- the BUILDER's side: a head, and segments that never move (§2.8, §6.4) ---- +// +// The head is a small node in the arena holding the segment chain, the live +// count and the dead count, allocated when the first entry is inserted. Each +// segment is a fixed number of entries carved from one call to the allocator +// pair. An entry's address is stable for the arena's life, so a value handed +// back by an insert stays valid while other entries arrive. +struct TableMapHead +{ + TableRef first; // the arena offset of the first segment + TableRef last; // and of the one an insert appends into + int32_t live; + int32_t dead; +}; + +template struct TableMapSegment +{ + TableRef next; + int32_t used; // entries carved from this segment + int32_t padding; + uint32_t dead[ ( kTableMapSegmentEntries + 31 ) / 32 ]; // Erase marks one bit, never the entry + Entry entries[ kTableMapSegmentEntries ]; +}; + +inline bool TableMapSegmentDead( const uint32_t * dead, int32_t index ) +{ + return ( dead[ index / 32 ] & ( 1u << ( index % 32 ) ) ) != 0; +} + +// ---- the ORDERED CURSOR the four writing walks read (§2.8) ---- +// +// Measure, Save, Lock and Cook each write a map's entries in ascending key +// order with no key twice, deriving the order from the builder's entries as +// each walk derives the numbering (§3.1). Nothing passes between them, so +// measure == save over a map is a real check on two sorts agreeing. +// +// A REGION is already sorted, so its cursor is the array in place and +// allocates nothing. The BUILDER's is the sort: an array of entry pointers +// allocated through the pair and released before the walk returns, because +// sorting the segments themselves would move entries whose addresses a caller +// holds. +template struct TableMapCursor +{ + const Entry * const * order = NULL; // the builder's form: sorted pointers + const Entry * entries = NULL; // the region's form: the array in place + int32_t count = 0; + TableAllocator allocator; + bool ok = false; + const Entry * operator[]( int32_t index ) const + { + return order != NULL ? order[index] : entries + index; + } +}; + +// heapsort: O( n log n ) once per map, no recursion, no allocation past the +// pointer array the caller already paid for +template inline void TableMapSort( const Entry ** order, int32_t count ) +{ + for ( int32_t start = count / 2 - 1; start >= 0; start-- ) + { + int32_t root = start; + for ( ;; ) + { + int32_t child = 2 * root + 1; + if ( child >= count ) { break; } + if ( child + 1 < count && TableEntryOrder( *order[child], *order[child + 1] ) < 0 ) { child++; } + if ( TableEntryOrder( *order[root], *order[child] ) >= 0 ) { break; } + const Entry * swap = order[root]; order[root] = order[child]; order[child] = swap; + root = child; + } + } + for ( int32_t end = count - 1; end > 0; end-- ) + { + const Entry * swap = order[0]; order[0] = order[end]; order[end] = swap; + int32_t root = 0; + for ( ;; ) + { + int32_t child = 2 * root + 1; + if ( child >= end ) { break; } + if ( child + 1 < end && TableEntryOrder( *order[child], *order[child + 1] ) < 0 ) { child++; } + if ( TableEntryOrder( *order[root], *order[child] ) >= 0 ) { break; } + const Entry * hold = order[root]; order[root] = order[child]; order[child] = hold; + root = child; + } + } +} + +// the REGION form: the array is already sorted, so the cursor is the array +template +inline TableMapCursor TableMapOrder( const TableRegionCtx &, const TableMap & map ) +{ + TableMapCursor cursor; + cursor.entries = map.Entries(); + cursor.count = map.count; + cursor.ok = true; + return cursor; +} + +// the BUILDER's form: gather the LIVE entries out of the segment chain in +// insertion order, then sort. A dead entry costs nothing on any wire (§2.8). +template +inline TableMapCursor TableMapOrder( const TableArena & arena, const TableMap & map ) +{ + TableMapCursor cursor; + cursor.allocator = arena.allocator; + cursor.count = map.count; + if ( map.entries.value == 0 || map.count <= 0 ) { cursor.ok = map.count == 0; cursor.count = 0; return cursor; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + if ( head->live != map.count ) { return cursor; } // the slot and the head disagree: refused, never guessed + const Entry ** order = (const Entry **) arena.allocator.alloc( arena.allocator.context, (int64_t) map.count * (int64_t) sizeof( const Entry * ) ); + if ( order == NULL ) { return cursor; } + int32_t at = 0; + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 && at < map.count ) + { + const TableMapSegment * segment = (const TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used && at < map.count; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + order[at++] = segment->entries + i; + } + segment_ref = segment->next; + } + if ( at != map.count ) + { + arena.allocator.free( arena.allocator.context, order ); + return cursor; + } + TableMapSort( order, map.count ); + cursor.order = order; + cursor.ok = true; + return cursor; +} + +template +inline TableMapCursor TableMapOrder( const TableArenaCtx & ctx, const TableMap & map ) +{ + return TableMapOrder( *ctx.arena, map ); +} + +template inline void TableMapRelease( TableMapCursor & cursor ) +{ + if ( cursor.order != NULL ) { cursor.allocator.free( cursor.allocator.context, (void *) cursor.order ); } + cursor.order = NULL; +} + +// ---- the builder's five (§2.8) ---- +// +// Insert APPENDS after one LINEAR SCAN of the live entries for the key it may +// replace, Find is that same scan, and Erase is the scan and one bit. The +// builder builds NO INDEX, and that is a rule: the sort happens once, at Lock, +// Save or Cook, and every lookup that matters runs over the sorted region. + +// the head, allocated when the first entry is inserted +template +inline TableMapHead * TableMapReach( TableWorker & worker, TableMap & map ) +{ + if ( worker.arena == NULL || worker.arena->locked ) { return NULL; } + if ( map.entries.value != 0 ) { return (TableMapHead *) TableArenaAt( *worker.arena, (uint32_t) map.entries.value ); } + uint32_t at = 0; + TableMapHead * head = (TableMapHead *) worker.AllocRaw( (int64_t) sizeof( TableMapHead ), (int64_t) alignof( TableMapHead ), at ); + if ( head == NULL ) { return NULL; } + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + map.entries.value = (int64_t) at; + return head; +} + +// one entry's storage, appended: the current segment when it has room, a new +// one carved from one call to the pair when it does not +template +inline Entry * TableMapAppend( TableWorker & worker, TableMapHead * head, TableMap & map ) +{ + TableMapSegment * segment = NULL; + if ( head->last.value != 0 ) + { + segment = (TableMapSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + if ( segment->used >= kTableMapSegmentEntries ) { segment = NULL; } + } + if ( segment == NULL ) + { + uint32_t at = 0; + segment = (TableMapSegment *) worker.AllocRaw( (int64_t) sizeof( TableMapSegment ), (int64_t) alignof( TableMapSegment ), at ); + if ( segment == NULL ) { return NULL; } // the arena could not carve another segment + segment->next.value = 0; + segment->used = 0; + segment->padding = 0; + for ( int32_t i = 0; i < (int32_t) ( sizeof( segment->dead ) / sizeof( segment->dead[0] ) ); i++ ) { segment->dead[i] = 0; } + if ( head->last.value != 0 ) + { + TableMapSegment * previous = (TableMapSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + previous->next.value = (int64_t) at; + } + else + { + head->first.value = (int64_t) at; + } + head->last.value = (int64_t) at; + } + Entry * entry = segment->entries + segment->used; + segment->used++; + head->live++; + map.count++; + return entry; +} + +// the LINEAR SCAN: the live entries in insertion order, O( n ) key compares +template +inline Entry * TableMapScan( const TableArena & arena, const TableMap & map, Key key ) +{ + if ( map.entries.value == 0 ) { return NULL; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( TableEntryOrder( segment->entries[i], key ) == 0 ) { return segment->entries + i; } + } + segment_ref = segment->next; + } + return NULL; +} + +// ERASE marks the entry DEAD, one bit in the segment's slot and not in the +// entry table, and decrements the live count. Its storage is reclaimed at +// RESET and never reused mid-build, because reusing a slot would make "an +// entry's address is stable" false for exactly one case. +template +inline bool TableMapErase( TableArena & arena, TableMap & map, Key key ) +{ + if ( map.entries.value == 0 ) { return false; } + TableMapHead * head = (TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( TableEntryOrder( segment->entries[i], key ) != 0 ) { continue; } + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + map.count--; + return true; + } + segment_ref = segment->next; + } + return false; +} + +// ---- iterate on the BUILDER: INSERTION order, live entries only (§2.8) ---- +template struct TableMapEach +{ + const TableArena * arena; + TableRef first; + + struct Iterator + { + const TableArena * arena; + TableMapSegment * segment; + int32_t index; + + void Skip() + { + for ( ;; ) + { + if ( segment == NULL ) { return; } + if ( index >= segment->used ) + { + segment = segment->next.value != 0 ? (TableMapSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + index = 0; + continue; + } + if ( TableMapSegmentDead( segment->dead, index ) ) { index++; continue; } + return; + } + } + auto operator*() const { return TableEntryEach( segment->entries + index ); } + Iterator & operator++() { index++; Skip(); return *this; } + bool operator==( const Iterator & other ) const { return segment == other.segment && index == other.index; } + bool operator!=( const Iterator & other ) const { return !( *this == other ); } + }; + + Iterator begin() const + { + Iterator it = { arena, first.value != 0 ? (TableMapSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL, 0 }; + it.Skip(); + return it; + } + Iterator end() const { Iterator it = { arena, NULL, 0 }; return it; } +}; + +template +inline TableMapEach TableMapEachOf( const TableArena & arena, const TableMap & map ) +{ + TableMapEach each = { &arena, TableRef() }; + if ( map.entries.value != 0 ) + { + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + each.first = head->first; + } + return each; +} + +// ---- the LOAD side: where a decoded entry lands (§2.8) ---- +// +// THE READER TRUSTS NOTHING and spends one compare per entry. Every load path +// applies the same rules and produces one report (§4), so the region load of +// §6.5 and LoadBuilder never disagree about a wire. These two shapes are what +// makes that true with one generated decoder: a REGION carves the entry array +// out of the holder node's own extent, and the TOOL's path appends into the +// builder's arena, and the decoder above them cannot tell which it has. + +// The node's extent cursor is TableExtentCarve, the extent runtime's (§2.8, +// §2.9): a map's whole entry array is carved first, then, entry by entry in +// key order, the arrays of any list or map an entry's value holds by value. + +// TableMapFill is one map field being decoded: where the next entry lands, and +// the entry that last LANDED, which is what the ascending check compares +// against. +template struct TableMapFill +{ + TableMap * map = NULL; + Entry * array = NULL; // the region path: the carved array + int32_t capacity = 0; + TableWorker * worker = NULL; // the TOOL's path + bool ok = false; +}; + +template +inline TableMapFill TableMapFillBegin( const TableNodeMap & nodes, TableMap & map, uint32_t n ) +{ + TableMapFill fill; + fill.map = ↦ + map.entries.value = 0; + map.count = 0; + if ( nodes.carve == NULL ) { return fill; } + if ( nodes.carve->worker != NULL ) + { + fill.worker = nodes.carve->worker; // the tool's path: the arena carves + fill.ok = true; + return fill; + } + const int64_t align = (int64_t) alignof( Entry ); + uint8_t * base = (uint8_t *) ( ( (uintptr_t) nodes.carve->at + (uintptr_t) ( align - 1 ) ) & ~( (uintptr_t) ( align - 1 ) ) ); + const int64_t bytes = (int64_t) n * (int64_t) sizeof( Entry ); + const int64_t pad = (int64_t) ( base - nodes.carve->at ); + if ( pad + bytes > nodes.carve->left ) { return fill; } // the measure and the load disagree: refused + nodes.carve->at = base + bytes; + nodes.carve->left -= pad + bytes; + fill.array = (Entry *) base; + fill.capacity = (int32_t) n; + map.entries.value = (int64_t) ( base - (const uint8_t *) &map.entries ); + fill.ok = true; + return fill; +} + +// the entry that last LANDED — NULL before the first +template inline Entry * TableMapFillLast( TableMapFill & fill ) +{ + if ( fill.map->count <= 0 ) { return NULL; } + if ( fill.array != NULL ) { return fill.array + ( fill.map->count - 1 ); } + return TableMapLive( *fill.worker->arena, *fill.map, fill.map->count - 1 ); +} + +// the next slot, at the entry type's declared defaults +template inline Entry * TableMapFillNext( TableMapFill & fill ) +{ + if ( fill.array != NULL ) + { + if ( fill.map->count >= fill.capacity ) { return NULL; } + Entry * entry = fill.array + fill.map->count; + TableReset( *entry ); + fill.map->count++; + return entry; + } + TableMapHead * head = TableMapReach( *fill.worker, *fill.map ); + if ( head == NULL ) { return NULL; } + Entry * entry = TableMapAppend( *fill.worker, head, *fill.map ); + if ( entry != NULL ) { TableReset( *entry ); } + return entry; +} + +// A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): at the first entry whose key +// kind disagrees with the reader's declaration the map resets to EMPTY, one +// kind_mismatch is counted for the map, and its remaining bytes are skipped. +template inline void TableMapFillReset( TableMapFill & fill ) +{ + if ( fill.array != NULL ) + { + fill.map->entries.value = 0; + fill.map->count = 0; + return; + } + if ( fill.map->entries.value != 0 ) + { + TableMapHead * head = (TableMapHead *) TableArenaAt( *fill.worker->arena, (uint32_t) fill.map->entries.value ); + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + } + fill.map->count = 0; +} + +// an EMPTY map's reference is null in both encodings, so a load that placed +// nothing leaves the slot exactly as a Reset does +template inline void TableMapFillEnd( TableMapFill & fill ) +{ + if ( fill.array != NULL && fill.map->count == 0 ) { fill.map->entries.value = 0; } +} + +// the k-th LIVE entry of a builder map, in insertion order — what the tool +// path's ascending check compares against +template +inline Entry * TableMapLive( const TableArena & arena, const TableMap & map, int32_t index ) +{ + if ( map.entries.value == 0 ) { return NULL; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + int32_t at = 0; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( at == index ) { return segment->entries + i; } + at++; + } + segment_ref = segment->next; + } + return NULL; +} + +// ---- LoadMeasure's term, from the FRAMING alone (§2.8, §6.5) ---- +// +// LoadMeasure's term for a map is N x sizeof( Entry ) rounded to +// alignof( Entry ), AT EVERY DEPTH. N is framing and not a value, so this +// reads no field: it walks the map's own header and, where an entry's value +// holds a map or a list of its own, the entries' headers under it. The caller +// owns the allocation precisely so it can refuse a number it did not expect. +// Every -1 carries its REASON (§6.5): the int32 cap first, because a count +// past it cannot fit any body, and then the body's own L, the one rule a +// list's term answers by. +// A MAP ENTRY'S SMALLEST WIRE FOOTPRINT that commands one storage unit is its +// own L and the body's terminator, and under this form's variable lengths that +// footprint is TWO BYTES (docs/SPEC-TABLES.md §4.2). It is what bounds the N a +// map's L can carry, and therefore what a LoadMeasure may be asked for. +static const int64_t kTableMapEntryFloor = 2; + +inline bool TableMapWireExtent( const uint8_t * body, int64_t length, int64_t & at, + int64_t entry_size, int64_t entry_align, TableWireExtentFn inner, + const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } // no array header: nothing rides + if ( r.get8() != 13 ) { return true; } // not an array of tables: §4's ordinary kind mismatch + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + if ( n > (uint64_t) INT32_MAX ) { reason = count_over_extent_cap; return false; } + const int64_t rest = length - r.offset; + if ( n > (uint64_t) ( rest / kTableMapEntryFloor ) ) { reason = count_over_length; return false; } // an N the map's L cannot carry + at = ( at + entry_align - 1 ) & ~( entry_align - 1 ); + at += (int64_t) n * entry_size; + if ( inner == NULL ) { return true; } // no map below an entry: one depth is the whole term + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } // framing damage: the load reports it + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +// ---- the TEXT form's placement (docs/SPEC-TABLES.md §2.8, §16) ---- +// +// The text is a plain JSON object keyed by the KEY, and the generic walk fills +// it through the ENTRY'S OWN descriptor — so all it needs from here is one +// entry at one key, handed back at its defaults. It is the builder's Insert +// with the ENTRY returned rather than its value, because the walk writes the +// value through a field row and not through a typed pointer. +// +// THE ONE INSERTION PRIMITIVE. Lookup, reset, allocation and the KEY COPY are +// all here, so no caller mutates an entry this did not create and no caller +// relabels one it found. A key is copied only when an entry is created, which +// is what makes a duplicate key leave the identity it matched untouched. NULL +// is one thing and one thing only: the arena refused. +template +inline Entry * TableMapPlace( TableWorker & worker, TableMap & map, Key key ) +{ + if ( worker.arena == NULL ) { return NULL; } + Entry * found = TableMapScan( *worker.arena, map, key ); + if ( found != NULL ) + { + TableResetMapValue( *found ); // a repeated key is LAST-WINS, whole + return found; + } + TableMapHead * head = TableMapReach( worker, map ); + if ( head == NULL ) { return NULL; } + Entry * entry = TableMapAppend( worker, head, map ); + if ( entry == NULL ) { return NULL; } + TableReset( *entry ); + TableEntrySetKey( *entry, key ); + return entry; +} + +// ---- the OPTIONAL RUNTIME INDEX (§2.8) ---- +// +// Open addressing with LINEAR PROBING over the sorted array, built AT LOAD for +// a map large enough that log n compares over a cold array cost more than one +// hash and a probe. IT IS NEVER STORED: the caller measures it, owns its +// storage, builds it in one pass and releases it whenever. +// +// ITS HASH AND ITS LOAD FACTOR ARE NOT A CROSS-PORT CONTRACT, and that is a +// rule. What a port is held to is the CONTRACT of the lookup: the same value +// the sorted array's Find returns for the same key, and no allocation past the +// storage the caller handed in. +struct TableMapIndex +{ + int32_t * slots = NULL; // entry indices, +1; 0 is an empty slot + int32_t capacity = 0; + bool good = false; +}; + +// this runtime's own, and no port reproduces it: fnv1a64 over the key's bytes +inline uint64_t TableMapHash( const void * bytes, int32_t length ) +{ + uint64_t hash = 0xCBF29CE484222325ull; + const uint8_t * at = (const uint8_t *) bytes; + for ( int32_t i = 0; i < length; i++ ) { hash ^= (uint64_t) at[i]; hash *= 0x100000001B3ull; } + return hash; +} +inline uint64_t TableMapHash( uint64_t key ) { return TableMapHash( (const void *) &key, (int32_t) sizeof( key ) ); } + +// this runtime's own load factor, and no port reproduces it either: the next +// power of two at or above twice the count, so a probe run stays short +inline int32_t TableMapIndexSlots( int32_t count ) +{ + int32_t slots = 8; + while ( slots < count * 2 ) { slots *= 2; } + return slots; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_MAP + +#ifndef MAPDEMO_SCHEMA_TABLE_LIST +#define MAPDEMO_SCHEMA_TABLE_LIST + +namespace mapdemo { + +// ---- an UNBOUNDED ARRAY: a counted array whose count the data decides (§2.9) ---- +// +// On the wire, in a region and in a cook a list is the kind 14 body a [..N]T +// writes, its elements by-value records inside the holder's node extent. What +// this adds is the slot, a builder that appends into segments that never +// move, and a const surface that indexes and iterates in place. There is no +// sort, no key and no lookup: the order is INSERTION order, and it is +// identity the way position is identity in a fixed array. + +// elements carved from ONE call to the allocator pair. A new segment is +// appended when the current one fills, and nothing ever moves (§6.4) +static const int32_t kTableListSegmentElements = 32; + +// THE ELEMENT STORAGE: T itself, and a TableRef slot for a []*T, whose +// elements are references exactly as a pointer field's slot is (§2.1) +template struct TableListStorage { typedef T Element; }; +template struct TableListStorage { typedef TableRef Element; }; + +// WHAT THE CONST FORM ANSWERS: the element by reference, and on a []*T the +// RESOLVED pointer, one add on the self-relative delta, NULL for a null slot, +// exactly as At answers it (§6.2, §6.3) +template struct TableListConst +{ + typedef const T & Result; + static Result At( const T * element ) { return *element; } +}; +template struct TableListConst +{ + typedef const T * Result; + static Result At( const TableRef * element ) + { + return element->value != 0 ? (const T *) ( (const uint8_t *) element + element->value ) : NULL; + } +}; + +// ---- the storage: SIXTEEN BYTES in the holder's record (§2.9, §7.2) ---- +// +// An int64 self-relative reference to the element array and an int32 count, +// then padding to eight. The reference is a TableRef like a pointer's: in the +// arena it names the builder's HEAD, in a region it is the delta from the slot +// to the first element, and 0 is the empty list in both. It is the map's slot +// exactly, because it is the same two facts. +template struct TableList +{ + typedef typename TableListStorage::Element Element; + + TableRef elements; + int32_t count = 0; // the LIVE count, in both forms + int32_t padding = 0; // named, so the record has no unwritten byte in it + + // ---- the CONST form: a locked region, a loaded one, an opened cook ---- + // + // One surface over one encoding (§6.3). A region reference resolves from + // the slot's own address, so every one of these is a member and needs no + // base and no context. + const Element * Elements() const + { + return elements.value != 0 ? (const Element *) ( (const uint8_t *) &elements + elements.value ) : NULL; + } + int32_t size() const { return count; } + + // INDEXING IS BOUNDS-CHECKED IN EVERY BUILD (§2.4, §2.9): the extent is a + // number that CAME FROM A FILE, so an index past it is not a mistake a + // release build gets to make cheaply. There is no undefined-behavior path + // here in any configuration. The assert carries the message where a + // debugger can read it and NDEBUG removes that. The fatal is what stands + // after it. Both go through the hooks: define schema_assert and + // schema_fatal and this refusal lands in your own handler. + void RefuseIndex( int32_t index ) const + { + if ( (uint32_t) index >= (uint32_t) count ) + { + schema_assert( false && "an unbounded array is indexed inside its count, which came from a file" ); + schema_fatal(); + } + } + typename TableListConst::Result operator[]( int32_t index ) const + { + RefuseIndex( index ); + return TableListConst::At( Elements() + index ); + } + + // ---- iteration: INDEX order, the element and no key ---- + // + // It carries no iterator_traits, for the reason TableKeyed's does not + // (§13.9). + struct ConstIterator + { + const Element * at; + typename TableListConst::Result operator*() const { return TableListConst::At( at ); } + ConstIterator & operator++() { at++; return *this; } + bool operator==( const ConstIterator & other ) const { return at == other.at; } + bool operator!=( const ConstIterator & other ) const { return at != other.at; } + }; + + ConstIterator begin() const { return ConstIterator{ Elements() }; } + ConstIterator end() const { return ConstIterator{ Elements() + count }; } +}; + +// ---- the BUILDER's side: a head, and segments that never move (§2.9, §6.4) ---- +// +// The head is a small node in the arena holding the segment chain, the live +// count and the dead count, allocated when the first element is added. Each +// segment is a fixed number of elements carved from one call to the allocator +// pair. An element's address is stable for the arena's life, so a T * handed +// back by Add stays valid while other elements arrive. +struct TableListHead +{ + TableRef first; // the arena offset of the first segment + TableRef last; // and of the one an Add appends into + int32_t live; + int32_t dead; +}; + +template struct TableListSegment +{ + TableRef next; + int32_t used; // elements carved from this segment + int32_t padding; + uint32_t dead[ ( kTableListSegmentElements + 31 ) / 32 ]; // Erase marks one bit, never the element + Element elements[ kTableListSegmentElements ]; +}; + +inline bool TableListSegmentDead( const uint32_t * dead, int32_t index ) +{ + return ( dead[ index / 32 ] & ( 1u << ( index % 32 ) ) ) != 0; +} + +// the head, allocated when the first element is added +template +inline TableListHead * TableListReach( TableWorker & worker, TableList & list ) +{ + if ( worker.arena == NULL || worker.arena->locked ) { return NULL; } + if ( list.elements.value != 0 ) { return (TableListHead *) TableArenaAt( *worker.arena, (uint32_t) list.elements.value ); } + uint32_t at = 0; + TableListHead * head = (TableListHead *) worker.AllocRaw( (int64_t) sizeof( TableListHead ), (int64_t) alignof( TableListHead ), at ); + if ( head == NULL ) { return NULL; } + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + list.elements.value = (int64_t) at; + return head; +} + +// one element's storage, appended: the current segment when it has room, a +// new one carved from one call to the pair when it does not. NULL means NOT +// ADDED: an arena that cannot carve another segment, or a count at the int32 +// cap (§2.2, §2.9). +template +inline typename TableList::Element * TableListAppend( TableWorker & worker, TableListHead * head, TableList & list ) +{ + typedef typename TableList::Element Element; + if ( list.count >= INT32_MAX ) { return NULL; } // the int32 storage cap + TableListSegment * segment = NULL; + if ( head->last.value != 0 ) + { + segment = (TableListSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + if ( segment->used >= kTableListSegmentElements ) { segment = NULL; } + } + if ( segment == NULL ) + { + uint32_t at = 0; + segment = (TableListSegment *) worker.AllocRaw( (int64_t) sizeof( TableListSegment ), (int64_t) alignof( TableListSegment ), at ); + if ( segment == NULL ) { return NULL; } // the arena could not carve another segment + segment->next.value = 0; + segment->used = 0; + segment->padding = 0; + for ( int32_t i = 0; i < (int32_t) ( sizeof( segment->dead ) / sizeof( segment->dead[0] ) ); i++ ) { segment->dead[i] = 0; } + if ( head->last.value != 0 ) + { + TableListSegment * previous = (TableListSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + previous->next.value = (int64_t) at; + } + else + { + head->first.value = (int64_t) at; + } + head->last.value = (int64_t) at; + } + Element * element = segment->elements + segment->used; + segment->used++; + head->live++; + list.count++; + return element; +} + +// ADD, whole: the head, the append, and the element at its declared defaults +// (§2.9). The text form's placement is this same call, because a list has no +// key to place under (§16). +template +inline typename TableList::Element * TableListPlace( TableWorker & worker, TableList & list ) +{ + typedef typename TableList::Element Element; + TableListHead * head = TableListReach( worker, list ); + if ( head == NULL ) { return NULL; } + Element * element = TableListAppend( worker, head, list ); + if ( element == NULL ) { return NULL; } + new ( element ) Element(); // value-init: the declared defaults, and null for a slot + return element; +} + +// ERASE, ADDRESSED BY THE POINTER (§2.9): the element Add handed back is the +// handle, because a list has no key and the address is the one thing the +// builder promises never moves (§6.4). It marks the element DEAD, one bit in +// the segment's slot and not in the element storage, and decrements the live +// count. False when the pointer is not this list's. Its storage is reclaimed +// at RESET and never reused mid-build, the map's rule for the map's reason. +template +inline bool TableListErase( TableArena & arena, TableList & list, const typename TableList::Element * element ) +{ + typedef typename TableList::Element Element; + if ( list.elements.value == 0 || element == NULL ) { return false; } + TableListHead * head = (TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableListSegment * segment = (TableListSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + if ( element >= segment->elements && element < segment->elements + segment->used ) + { + const int32_t i = (int32_t) ( element - segment->elements ); + if ( TableListSegmentDead( segment->dead, i ) ) { return false; } // already erased + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + list.count--; + return true; + } + segment_ref = segment->next; + } + return false; +} + +// ---- iterate on the BUILDER: INDEX order, live elements only (§2.9) ---- +template struct TableListEach +{ + typedef typename TableList::Element Element; + const TableArena * arena; + TableRef first; + + struct Iterator + { + const TableArena * arena; + TableListSegment * segment; + int32_t index; + + void Skip() + { + for ( ;; ) + { + if ( segment == NULL ) { return; } + if ( index >= segment->used ) + { + segment = segment->next.value != 0 ? (TableListSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + index = 0; + continue; + } + if ( TableListSegmentDead( segment->dead, index ) ) { index++; continue; } + return; + } + } + Element * operator*() const { return segment->elements + index; } + Iterator & operator++() { index++; Skip(); return *this; } + bool operator==( const Iterator & other ) const { return segment == other.segment && index == other.index; } + bool operator!=( const Iterator & other ) const { return !( *this == other ); } + }; + + Iterator begin() const + { + Iterator it = { arena, first.value != 0 ? (TableListSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL, 0 }; + it.Skip(); + return it; + } + Iterator end() const { Iterator it = { arena, NULL, 0 }; return it; } +}; + +template +inline TableListEach TableListEachOf( const TableArena & arena, const TableList & list ) +{ + TableListEach each = { &arena, TableRef() }; + if ( list.elements.value != 0 ) + { + const TableListHead * head = (const TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + each.first = head->first; + } + return each; +} + +// ---- the INDEX-ORDER CURSOR the four writing walks read (§2.9) ---- +// +// Measure, Save, Lock and Cook each visit a list's live elements in the order +// they were added, and they allocate nothing to do it: a region's cursor is +// the array in place, and the builder's walks the segment chain. Indexing the +// builder's form is SEQUENTIAL by construction, every walk steps i, i + 1, +// i + 2, so the cursor remembers where the last access landed and moves one +// live slot per step. An access behind the memo restarts from the first +// segment, which no walk here does. +template struct TableListCursor +{ + const Element * elements = NULL; // the region's form: the array in place + const TableArena * arena = NULL; // the builder's form: the segments + TableRef first; + int32_t count = 0; + bool ok = false; + // the memo: the segment and slot the last access landed on, and the live + // index that slot holds + mutable const TableListSegment * segment = NULL; + mutable int32_t within = -1; + mutable int32_t logical = -1; + + const Element * At( int32_t index ) const + { + if ( elements != NULL ) { return elements + index; } + if ( segment == NULL || index < logical ) + { + segment = first.value != 0 ? (const TableListSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL; + within = -1; + logical = -1; + } + while ( logical < index ) + { + for ( ;; ) + { + within++; + while ( segment != NULL && within >= segment->used ) + { + segment = segment->next.value != 0 ? (const TableListSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + within = 0; + } + if ( segment == NULL ) { return NULL; } // the slot and the head disagree + if ( !TableListSegmentDead( segment->dead, within ) ) { break; } + } + logical++; + } + return segment->elements + within; + } + const Element & operator[]( int32_t index ) const { return *At( index ); } +}; + +// the REGION form: the array is the cursor +template +inline TableListCursor::Element> TableListElements( const TableRegionCtx &, const TableList & list ) +{ + TableListCursor::Element> cursor; + cursor.elements = list.Elements(); + cursor.count = list.count; + cursor.ok = true; + return cursor; +} + +// the BUILDER's form: the live elements out of the segment chain, in the +// order they were added. A dead element costs nothing on any wire (§2.9). +template +inline TableListCursor::Element> TableListElements( const TableArena & arena, const TableList & list ) +{ + TableListCursor::Element> cursor; + cursor.arena = &arena; + cursor.count = list.count; + if ( list.elements.value == 0 || list.count <= 0 ) { cursor.ok = list.count == 0; cursor.count = 0; return cursor; } + const TableListHead * head = (const TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + if ( head->live != list.count ) { return cursor; } // the slot and the head disagree: refused, never guessed + cursor.first = head->first; + cursor.ok = true; + return cursor; +} + +template +inline TableListCursor::Element> TableListElements( const TableArenaCtx & ctx, const TableList & list ) +{ + return TableListElements( *ctx.arena, list ); +} + +// ---- the LOAD side: where a decoded element lands (§2.9) ---- +// +// The same two shapes the map's fill takes, because the decoder above them +// cannot tell which it has: a REGION carves the element array out of the +// holder node's own extent, PRE-ORDER, and the TOOL's path appends into the +// builder's arena. +template struct TableListFill +{ + typedef typename TableList::Element Element; + TableList * list = NULL; + Element * array = NULL; // the region path: the carved array + int32_t capacity = 0; + TableWorker * worker = NULL; // the TOOL's path + bool ok = false; + bool refused = false; // a count above the int32 cap on the tool's path: LoadBuilder answers NULL +}; + +template +inline TableListFill TableListFillBegin( const TableNodeMap & nodes, TableList & list, uint64_t n ) +{ + typedef typename TableList::Element Element; + TableListFill fill; + fill.list = &list; + list.elements.value = 0; + list.count = 0; + if ( nodes.carve == NULL ) { return fill; } + if ( n > (uint64_t) INT32_MAX ) + { + // A COUNT ABOVE THE int32 STORAGE CAP (§2.2, §2.9): into a region it was + // refused by LoadMeasure before this ran, and into a builder it is the + // refusal LoadBuilder answers NULL for, moving no counter + fill.refused = nodes.carve->worker != NULL; + return fill; + } + if ( nodes.carve->worker != NULL ) + { + fill.worker = nodes.carve->worker; // the tool's path: the arena carves + fill.ok = true; + return fill; + } + const int64_t align = (int64_t) alignof( Element ); + uint8_t * base = (uint8_t *) ( ( (uintptr_t) nodes.carve->at + (uintptr_t) ( align - 1 ) ) & ~( (uintptr_t) ( align - 1 ) ) ); + const int64_t bytes = (int64_t) n * (int64_t) sizeof( Element ); + const int64_t pad = (int64_t) ( base - nodes.carve->at ); + if ( pad + bytes > nodes.carve->left ) { return fill; } // the measure and the load disagree: refused + nodes.carve->at = base + bytes; + nodes.carve->left -= pad + bytes; + fill.array = (Element *) base; + fill.capacity = (int32_t) n; + list.elements.value = (int64_t) ( base - (const uint8_t *) &list.elements ); + fill.ok = true; + return fill; +} + +// the next slot, at the element's declared defaults. NULL when the arena +// could not carve, which the decoder reports as framing damage +template inline typename TableList::Element * TableListFillNext( TableListFill & fill ) +{ + typedef typename TableList::Element Element; + if ( fill.array != NULL ) + { + if ( fill.list->count >= fill.capacity ) { return NULL; } + Element * element = fill.array + fill.list->count; + new ( element ) Element(); + fill.list->count++; + return element; + } + return TableListPlace( *fill.worker, *fill.list ); +} + +// A SLOT WHOSE ELEMENT NEVER LANDED is given back (§2.9, §4): the array keeps +// what it decoded, and an element whose own framing gave out before one byte +// of it decoded was not decoded. The region's form uncounts it, and the builder's +// marks it dead, which is what the storage rule allows mid-build. +template inline void TableListFillDrop( TableListFill & fill ) +{ + typedef typename TableList::Element Element; + if ( fill.array != NULL ) + { + if ( fill.list->count > 0 ) { fill.list->count--; } + return; + } + if ( fill.list->elements.value == 0 ) { return; } + TableListHead * head = (TableListHead *) TableArenaAt( *fill.worker->arena, (uint32_t) fill.list->elements.value ); + if ( head->last.value == 0 ) { return; } + TableListSegment * segment = (TableListSegment *) TableArenaAt( *fill.worker->arena, (uint32_t) head->last.value ); + if ( segment->used <= 0 ) { return; } + const int32_t i = segment->used - 1; + if ( TableListSegmentDead( segment->dead, i ) ) { return; } + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + fill.list->count--; +} + +// an EMPTY list's reference is null in both encodings, so a load that placed +// nothing leaves the slot exactly as a Reset does +template inline void TableListFillEnd( TableListFill & fill ) +{ + if ( fill.array != NULL && fill.list->count == 0 ) { fill.list->elements.value = 0; } +} + +// ---- LoadMeasure's term, from the FRAMING alone (§2.9, §6.5) ---- +// +// N x sizeof( T ) rounded to alignof( T ), AT EVERY DEPTH. N is framing and +// not a value, so this reads no field: it walks the list's own header and, +// where a table element holds a list or a map of its own, the elements' +// headers under it. Every -1 carries its REASON (§6.5): the int32 cap first, +// because a count past it cannot fit any body, and then the body's own L. +inline bool TableListWireExtent( const uint8_t * body, int64_t length, int64_t & at, + int64_t elem_size, int64_t elem_align, uint8_t elem_kind, int64_t elem_floor, + TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } // no array header: nothing rides + if ( r.get8() != elem_kind ) { return true; } // another element kind: §4's ordinary kind mismatch, the field reads empty + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + if ( n > (uint64_t) INT32_MAX ) { reason = count_over_extent_cap; return false; } + const int64_t rest = length - r.offset; + if ( n > (uint64_t) ( rest / elem_floor ) ) { reason = count_over_length; return false; } // an N the list's L cannot carry + at = ( at + elem_align - 1 ) & ~( elem_align - 1 ); + at += (int64_t) n * elem_size; + if ( inner == NULL ) { return true; } // nothing below an element: one depth is the whole term + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } // framing damage: the load reports it + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_LIST + +#ifndef MAPDEMO_SCHEMA_BUILD_VERSION +#define MAPDEMO_SCHEMA_BUILD_VERSION + +namespace mapdemo { + +// THE BUILD VERSION (docs/SPEC-TABLES.md §20): one digest over every fact the bytes +// this build produces depend on — the type wire's protocol id, every record's +// layout as the compiler's own C ABI model computes it, and the facts that +// decide what a load PUTS in those slots. It is the number a cook's header +// carries and the number Open compares, and the number a block's prologue +// carries and BlockOpen compares: a build version answers "which build?" and +// not "which form?", and what separates the two forms is their MAGIC. +// +// There are TWO ids in the design and they are not interchangeable: the +// PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is +// what everything cooked or blocked is keyed by. A table edit moves this and +// never the protocol id; a type edit moves both. +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_BUILD_VERSION + +#ifndef MAPDEMO_SCHEMA_TABLE_COOK +#define MAPDEMO_SCHEMA_TABLE_COOK + +namespace mapdemo { + +// ---- the cooked form (docs/SPEC-TABLES.md §7) ---- +// +// A cooked file is a HEADER, a DATA part and an ATTRIBUTION part, in that +// order. Every word of the header is a u64 written in the byte order the cook +// was produced in, and the header is 64 bytes: +// +// 0 magic 0x4b4f4f434d484353, read BYTEWISE before anything else +// 8 build_version the unit's id (docs/SPEC-TABLES.md §20) +// 16 byte_order 1 little, 2 big — the order that WROTE the file +// 24 data_length the region's bytes, rounded up to alignment +// 32 attribution_length the directory's bytes, or 0 +// 40 alignment the region's alignment, never below eight +// 48 reserved zero +// 56 reserved zero +// +// The DATA part is Lock's region written verbatim (§7.2) — the root at its +// base — and it is what a runtime points at. The ATTRIBUTION part is the node +// directory (§6.3), and NOTHING THAT READS THE STRUCTURE TOUCHES IT: it is +// written beside the data for schema cook-check, so a build that ships no +// tooling need not carry it at all. +static const int64_t kTableCookHeaderBytes = 64; + +// THE MAGIC'S VALUE, and a consumer written from the page needs the constant +// rather than a description of one. It is "SCHMCOOK" read as ASCII in the byte +// order a little-endian store produces — the same shape the block form's +// SCHMABLK takes, so a hex dump of a little-endian cook is legible and the two +// accelerators sit in one vocabulary. +// +// IT IS STORED IN THE PRODUCER'S ORDER, which is what makes it the byte-order +// check as well as the form check: a consumer reads back this build's +// constant, or that constant byte-reversed — which identifies a cook of the +// OTHER order — or something that is not a cook. All three answers but the +// first refuse, and a cook and a BLOCK are separated here too, because a +// form's identity belongs in its magic rather than in a second digest. +static const uint64_t TableCookMagic = 0x4b4f4f434d484353ull; + +// THIS BUILD's byte order, as the header's own word carries it. The magic is +// what REFUSES a foreign order; this word is what RECORDS which order wrote +// the file, so a refusal names the order rather than inferring it and a tool +// dumping a cook reads the fact. A file whose magic matched and whose order +// word did not is corrupt, and there is no reading that recovers it. +// +// The BUILD VERSION cannot do either job: §20.1 digests byteorder as a +// GENERATION input, little for every target schema generates for today, so +// two builds of one schema for two orders emit the same id. +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +static const uint64_t TableCookByteOrder = 2; // big +#else +static const uint64_t TableCookByteOrder = 1; // little +#endif + +// The greatest region alignment a cooked file may name. The DATA part begins +// at align_up( 64, alignment ), which is 64 for every unit this language can +// declare — the largest alignment it has is sixteen — so a word past this cap +// describes a file no build of this schema wrote (docs/SPEC-TABLES.md §7.1). +static const uint64_t TableCookMaxAlign = 64; + +// The header read, BYTEWISE. memcpy is the portable spelling of "these eight +// bytes, in this machine's order"; every compiler this repo builds under folds +// it to one load, and it is the only read in the whole of Open that is not a +// comparison. +inline uint64_t table_cook_read64( const uint8_t * p ) +{ + uint64_t v; + memcpy( &v, p, sizeof( v ) ); + return v; +} + +// TableCookOpen: THE WHOLE CHECK, in one place, because §7 states the +// enumeration once and every generated Open is that one enumeration plus +// its own root's two layout facts. +// +// THE CHECK, in order: the magic read bytewise, the byte order it establishes, +// the build version against this build's own, both RESERVED words zero, the +// region alignment the header names, the two part lengths against the length +// the caller passed — a truncated file and a file with trailing bytes are the +// same refusal — the root's own storage inside the data part, and the +// alignment of the base. +// +// AND THAT IS ALL OF IT. On a match the bytes ARE what this build wrote, in +// this build's layout and this build's byte order, so there is nothing to +// validate and nothing to fix up: the caller gets the root. Nothing per node +// happens here, which is what makes open O(1) in the file's size; a walk of +// any shape would forfeit that, and validating an untrusted file is schema +// cook-check's job and a person's decision (§7.4). +// +// EVERY NUMBER BELOW COMES OUT OF THE FILE, so the arithmetic is unsigned and +// each term is BOUNDED BEFORE IT IS ADDED: a forged length near 2^64 must +// refuse, and an addition that wrapped would be the defect the comparison +// after it was supposed to catch. Nothing past length is read on any path, +// including every refusing one. +// A REFUSAL NAMES ITSELF, beside the null (docs/SPEC-TABLES.md §7): the reason +// is written on the refusal path only, so a match costs nothing and a caller +// that passed no out-parameter pays nothing. +inline const uint8_t * TableCookRefuse( TableRefuseReason * reason, TableRefuseReason why ) +{ + if ( reason != NULL ) { *reason = why; } + return NULL; +} + +inline uint64_t table_cook_byteswap64( uint64_t v ) +{ + return ( v >> 56 ) | ( ( v >> 40 ) & 0xff00ull ) | ( ( v >> 24 ) & 0xff0000ull ) | ( ( v >> 8 ) & 0xff000000ull ) + | ( ( v << 8 ) & 0xff00000000ull ) | ( ( v << 24 ) & 0xff0000000000ull ) | ( ( v << 40 ) & 0xff000000000000ull ) + | ( v << 56 ); +} + +inline const uint8_t * TableCookOpen( const void * bytes, uint64_t length, uint64_t root_size, uint64_t root_align, TableRefuseReason * reason ) +{ + // a null buffer is the CALLER's defect, as an unaligned base is; a buffer + // shorter than the header has no header to read and is truncated + if ( bytes == NULL ) { return TableCookRefuse( reason, unaligned_base ); } + if ( length < (uint64_t) kTableCookHeaderBytes ) { return TableCookRefuse( reason, truncated ); } + const uint8_t * raw = (const uint8_t *) bytes; + // the MAGIC, bytewise and first: it is what establishes the byte order + // every other header word is read in, so nothing else may be read before + // it. A byte-reversed constant is a cook of the other order and refuses + // here, which is why the order never reaches a fix-up pass; anything else + // is not a cook at all, a BLOCK's magic included. + const uint64_t magic = table_cook_read64( raw ); + if ( magic != TableCookMagic ) + { + return TableCookRefuse( reason, magic == table_cook_byteswap64( TableCookMagic ) ? foreign_order : not_a_cook ); + } + // a byte-order word that contradicts its own magic describes no cook in + // EITHER order, so it shares the magic's own value (§7.1) + if ( table_cook_read64( raw + 16 ) != TableCookByteOrder ) { return TableCookRefuse( reason, not_a_cook ); } + if ( table_cook_read64( raw + 8 ) != BuildVersion ) { return TableCookRefuse( reason, wrong_build_version ); } + // the RESERVED words: a non-zero one means a writer used a form this build + // does not understand, and Open refuses rather than ignoring it. + if ( table_cook_read64( raw + 48 ) != 0 ) { return TableCookRefuse( reason, reserved_not_zero ); } + if ( table_cook_read64( raw + 56 ) != 0 ) { return TableCookRefuse( reason, reserved_not_zero ); } + const uint64_t data_length = table_cook_read64( raw + 24 ); + const uint64_t attribution_length = table_cook_read64( raw + 32 ); + const uint64_t alignment = table_cook_read64( raw + 40 ); + // THE ALIGNMENT WORD IS DATA, and it is the one header field the rest of + // the check does arithmetic WITH rather than only comparison against. A + // region's alignment is a power of two, never below eight (the floor that + // puts the attribution part on an eight-byte boundary without a second + // padding rule) and never past the cap above; a word that is none of those + // rounds nothing and aligns nothing, so it is refused before it is used, + // which is why bad_alignment precedes both truncated clauses (§7). + if ( alignment < 8 || alignment > TableCookMaxAlign ) { return TableCookRefuse( reason, bad_alignment ); } + if ( ( alignment & ( alignment - 1 ) ) != 0 ) { return TableCookRefuse( reason, bad_alignment ); } + // and it must be an alignment THE ROOT CAN SIT AT, since the root is at + // the region's base: both are powers of two, so "at least the root's" + // is one division. + if ( ( alignment % root_align ) != 0 ) { return TableCookRefuse( reason, bad_alignment ); } + // The DATA part begins at align_up( 64, alignment ). It is DERIVED and not + // a header field, because a fact a reader computes is a fact two writers + // cannot disagree about. + const uint64_t data_offset = ( (uint64_t) kTableCookHeaderBytes + alignment - 1 ) & ~( alignment - 1 ); + if ( length < data_offset ) { return TableCookRefuse( reason, truncated ); } + // the two part lengths against the length the caller passed. The whole + // file is data_offset + data_length + attribution_length, and a length + // that is not EXACTLY that refuses — truncation and trailing bytes are one + // refusal, and both terms are subtracted rather than added so no sum can + // carry. + if ( data_length > length - data_offset ) { return TableCookRefuse( reason, truncated ); } + if ( attribution_length != length - data_offset - data_length ) { return TableCookRefuse( reason, truncated ); } + // the ROOT sits at the region's base, so the region has to hold it: a + // shorter data part describes a root partly outside the file, which is the + // one way a match-and-point reader could hand back storage it never + // received. It is the second clause on truncated (§7). + if ( data_length < root_size ) { return TableCookRefuse( reason, truncated ); } + const uint8_t * base = raw + data_offset; + // the alignment of the BASE, LAST, because it is the only clause that reads + // nothing out of the file. The header pads the data part to the region's + // alignment, so a base an allocator or mmap gave you is already aligned — + // mmap gives page alignment for free — and a base that is not is a caller's + // buffer this form cannot be read out of: the caller's defect, not the file's. + if ( ( (uintptr_t) base % (uintptr_t) alignment ) != 0 ) { return TableCookRefuse( reason, unaligned_base ); } + return base; +} + +// ---- the cooked form, the WRITE side (docs/SPEC-TABLES.md §7.6) ---- +// +// THE BYTE ORDER IS THE TARGET'S, NOT THE HOST'S. A cook is produced in the +// byte order of the build that will read it (§7), so the fixing happens here — +// offline, once, on the writing side — and never at Open. Passing +// TableByteOrder::Big on a little-endian machine produces a big-endian build's +// file, and nothing about the writing host reaches the bytes. +enum class TableByteOrder +{ + Little = 1, // the header's byte_order word, and the order every scalar is written in + Big = 2, +}; + +// One store, width as an argument. Every call site passes a literal width, so +// the loop folds to a store (and a byte swap on the foreign order); a name per +// width would claim four §11 names to save nothing. +inline void table_cook_put( uint8_t * at, uint64_t value, int32_t width, TableByteOrder order ) +{ + if ( order == TableByteOrder::Little ) + { + for ( int32_t i = 0; i < width; i++ ) { at[i] = (uint8_t) ( value >> ( 8 * i ) ); } + } + else + { + for ( int32_t i = 0; i < width; i++ ) { at[i] = (uint8_t) ( value >> ( 8 * ( width - 1 - i ) ) ); } + } +} + +// A 128-bit store as two lanes: sixteen bytes, the low lane first in the +// little order and the high lane first — each lane big-endian — in the big +// order, exactly as a u64 is one lane of eight (docs/SPEC-TABLES.md §7.2). +inline void table_cook_put128( uint8_t * at, uint64_t lo, uint64_t hi, TableByteOrder order ) +{ + if ( order == TableByteOrder::Little ) { table_cook_put( at, lo, 8, order ); table_cook_put( at + 8, hi, 8, order ); } + else { table_cook_put( at, hi, 8, order ); table_cook_put( at + 8, lo, 8, order ); } +} + +// A buffer piece: the USED bytes and nothing else. The tail is already zero — +// the whole extent was zeroed before any field was written — so this copies the +// used prefix and leaves the rest, which is what makes a string's unused tail a +// consequence of one memset rather than a rule per buffer. A used length past +// the buffer, or below zero, is a value no reader could have produced and it is +// clamped rather than trusted: this writes inside the caller's buffer on every +// input. +inline void table_cook_bytes( uint8_t * at, const void * source, int64_t used, int64_t capacity ) +{ + if ( used <= 0 ) { return; } + const int64_t n = used < capacity ? used : capacity; + memcpy( at, source, (size_t) n ); +} + +// A WIDE TEXT buffer piece (docs/SPEC-TABLES.md §7.2): the USED code units, +// each a TWO-BYTE SCALAR in the cook's byte order. A record is written piece +// by piece and never memcpy'd, and a swap has to know where every scalar +// begins — a char16_t is one, so the units go one store each rather than as +// bytes. The tail is already zero, as the narrow twin's is, so the terminating +// zero unit at index used costs nothing here. +inline void table_cook_units( uint8_t * at, const char16_t * source, int64_t used, int64_t capacity, TableByteOrder order ) +{ + if ( used <= 0 ) { return; } + const int64_t n = used < capacity ? used : capacity; + for ( int64_t i = 0; i < n; i++ ) { table_cook_put( at + i * 2, (uint64_t) (uint16_t) source[i], 2, order ); } +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_COOK + +#ifndef MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE +#define MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE + +namespace mapdemo { + +// ---- the cooked form's WRITE side for a POINTERED root (docs/SPEC-TABLES.md §7.6) ---- +// +// A pointered root's cook is the region of §7.2: every node the numbering +// reached (§3.1), once, at its own type's alignment, in index order, the root +// at offset zero. This is that region while it is being laid out and written — +// the tool's own Layout and Write, in one struct. +// +// The OFFSETS are one per node, the root's zero at position 0 and node index k +// at position k - 1, which is the directory's own order (§6.3); they are the +// one allocation the write makes beyond the numbering, and they go through the +// same pair. A measure needs no offsets and leaves the pointer NULL. +struct TableCookRegion +{ + const TableNumbering * numbering = NULL; // node -> index, from the walk that placed it + int64_t * offsets = NULL; // index - 1 -> the node's region offset; NULL while measuring + int64_t count = 0; // nodes, the root included + int64_t bytes = 0; // the data part's length, rounded to align + int64_t align = 0; // the region's alignment: the nodes' greatest, never below eight + uint8_t * base = NULL; // where the data part is being written; NULL while measuring +}; + +// A reference slot: the SELF-RELATIVE delta from the slot's own address to the +// node's start (§6.3), and zero for null. The node is found by the address the +// numbering keyed it under, which is the same address the walk resolved through +// the same context — so a reference the numbering does not carry is a slot the +// walk never reached (a counted array's slot past its count, an absent +// optional's value) holding a node the region will not hold, and it is refused +// rather than written as a delta to nowhere. +inline bool table_cook_ref( const TableCookRegion & region, uint8_t * at, const void * pointee, TableByteOrder order ) +{ + if ( pointee == NULL ) { table_cook_put( at, 0, 8, order ); return true; } + uint64_t index = 0; + if ( !TableNumberingIndex( *region.numbering, pointee, index ) ) { return false; } + if ( index == 0 || index > (uint64_t) region.count ) { return false; } + const int64_t delta = region.offsets[index - 1] - (int64_t) ( at - region.base ); + table_cook_put( at, (uint64_t) delta, 8, order ); + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE + +namespace mapdemo { + +// table CrewsMembersEntry — TABLE-wire storage: relocatable, bounded, defaults in the +// member initializers (docs/SPEC-TABLES.md) +struct CrewsMembersEntry { + uint32_t key = 0; + TableRef value[2]; // [..2]*Item — used count beside it; every slot null until assigned + int32_t value_count = 0; +}; + +// table Crews — TABLE-wire storage: relocatable, bounded, defaults in the +// member initializers (docs/SPEC-TABLES.md) +struct Crews { + TableMap members; // map[uint32]*Item — the sorted entry array, empty until an insert + int32_t after = 0; +}; + +// ---- prefill: the declared defaults, in place (docs/SPEC-TABLES.md) ---- + +inline void CrewsMembersEntryReset( CrewsMembersEntry & value ); +inline void CrewsReset( Crews & value ); + +inline void CrewsMembersEntryReset( CrewsMembersEntry & value ) +{ + value.key = 0; + for ( int32_t i = 0; i < 2; i++ ) { value.value[i].value = 0; } // [2]*Item — every slot null + value.value_count = 0; +} + +inline void CrewsReset( Crews & value ) +{ + value.members.entries.value = 0; // map[uint32]*Item: empty + value.members.count = 0; + value.members.padding = 0; + value.after = 0; +} + +template inline int64_t CrewsMembersEntryMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const CrewsMembersEntry & value ); +template inline bool CrewsMembersEntrySaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const CrewsMembersEntry & value ); +inline bool CrewsMembersEntryLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, CrewsMembersEntry & value ); +template inline int64_t CrewsMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Crews & value ); +template inline bool CrewsSaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Crews & value ); +inline bool CrewsLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Crews & value ); +inline bool CrewsMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ); + +// CrewsMembersEntryMessageKeyRead: the key of one entry on the message wire, before the +// slot is chosen (docs/SPEC-TABLES.md §2.8, §3.3), and the bit the entry's +// body ends at. Field order inside a body is not contractual, so this scans +// the whole body by its announced shapes rather than assuming a position. +struct CrewsMembersEntryMessageKeyRead +{ + uint32_t key; + int64_t end; // the bit after the entry's own zero reference + bool found; // the body carried the key's id + bool kind_bad; // it carried it under another kind: the MAP's event + bool widened; // under a kind the declaration WIDENS (§4): decoded exactly, the MAP counts one + bool over; // longer than this reader's bound: the ENTRY is dropped + bool malformed; // the entry's framing gave out +}; + +inline CrewsMembersEntryMessageKeyRead CrewsMembersEntryMessageReadKey( TableBitReader r, const TableVocabulary & vocabulary, int64_t index_bits ) +{ + CrewsMembersEntryMessageKeyRead out = { 0, 0, false, false, false, false, false }; + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { out.malformed = true; return out; } + if ( ref == 0 ) { out.end = r.offset; return out; } // the terminator: no key field is the key's DEFAULT + if ( ref > (uint64_t) vocabulary.count ) { out.malformed = true; return out; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { out.malformed = true; return out; } + if ( entry.id == 0x3dc94a19365b10ecull ) // `key`, the ordinary hash of an ordinary name + { + const bool kind_bad = entry.kind != 8 && !TableKindWidens( entry.kind, 8 ); // THE KEY KIND IS THE READER'S DECLARATION + out.kind_bad = kind_bad; + out.found = !kind_bad; + out.widened = entry.kind != 8; + if ( kind_bad ) + { + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { out.malformed = true; return out; } + continue; + } + { + uint64_t raw = 0; + const int64_t width = entry.value_bits; + if ( width < 0 || !r.get( raw, width ) ) { out.malformed = true; return out; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + out.key = (uint32_t) decoded_wide; + } + continue; // the LAST occurrence is the one §3 keeps + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { out.malformed = true; return out; } + } +} + +// ---- the arena's reset hook (docs/SPEC-TABLES.md §6) ---- +// +// TableWorker::Alloc is a template and cannot name a member's Reset, so +// the arena reaches it through this overload set by argument-dependent +// lookup. It is how a node born in raw arena storage comes to hold the +// declared defaults without value-initialising the whole aggregate. + +inline void TableReset( CrewsMembersEntry & value ) { CrewsMembersEntryReset( value ); } +inline void TableReset( Crews & value ) { CrewsReset( value ); } + +// ---- pointer targets: allocation and resolution (docs/SPEC-TABLES.md §2) ---- +// +// A reference resolves differently in the two forms, and the CONTEXT says +// which: in the arena it is an offset; in a region it is a self-relative +// delta, so the const deref below is one add and needs no base pointer. + +// ---- codecs: measure/save/load per closure member ---- + +template inline int64_t CrewsMembersEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const CrewsMembersEntry & value ); +template inline bool CrewsMembersEntrySaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const CrewsMembersEntry & value ); +template inline bool CrewsMembersEntrySaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const CrewsMembersEntry & value ); +inline bool CrewsMembersEntryLoadBody( TableReader & r, const TableNodeMap & nodes, CrewsMembersEntry & value ); +template inline int64_t CrewsMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Crews & value ); +template inline bool CrewsSaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Crews & value ); +template inline bool CrewsSaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Crews & value ); +inline bool CrewsLoadBody( TableReader & r, const TableNodeMap & nodes, Crews & value ); + +// ---- pointer-graph walkers: number (measure/save), pack (Lock) ---- + +template inline bool CrewsMembersEntryNumber( const Ctx & ctx, TableNumbering & numbering, const CrewsMembersEntry & value ); +template inline int64_t CrewsMembersEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const CrewsMembersEntry & value ); +template inline bool CrewsMembersEntryPack( const Ctx & ctx, TablePackMap & seen, const CrewsMembersEntry & src, CrewsMembersEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); +template inline bool CrewsNumber( const Ctx & ctx, TableNumbering & numbering, const Crews & value ); +template inline int64_t CrewsPackMeasure( const Ctx & ctx, TablePackMap & seen, const Crews & value ); +template inline bool CrewsPack( const Ctx & ctx, TablePackMap & seen, const Crews & src, Crews & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +// ---- the numbering's bridge to each member's codec (docs/SPEC-TABLES.md §3.1) ---- + +template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const CrewsMembersEntry & value ) { return CrewsMembersEntryMeasureBody( ctx, numbering, ids, value ); } +template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const CrewsMembersEntry & value ) { return CrewsMembersEntrySaveBody( ctx, numbering, w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const CrewsMembersEntry & value ) { return CrewsMembersEntryMeasureMessageBody( ctx, numbering, index_bits, at, value ); } +template inline bool TableNodeMessageSave( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const CrewsMembersEntry & value ) { return CrewsMembersEntrySaveMessageBody( ctx, numbering, index_bits, w, value ); } +template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Crews & value ) { return CrewsMeasureBody( ctx, numbering, ids, value ); } +template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Crews & value ) { return CrewsSaveBody( ctx, numbering, w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Crews & value ) { return CrewsMeasureMessageBody( ctx, numbering, index_bits, at, value ); } +template inline bool TableNodeMessageSave( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Crews & value ) { return CrewsSaveMessageBody( ctx, numbering, index_bits, w, value ); } + +// ---- CrewsMembersEntry: the order, the key and the value (docs/SPEC-TABLES.md §2.8) ---- +// +// The four overloads the map runtime's templates reach by argument-dependent +// lookup. Nothing outside this file names them. +static_assert( alignof( CrewsMembersEntry ) <= kTableAlign, "a map entry's alignment must fit the arena's" ); + +inline int TableEntryOrder( const CrewsMembersEntry & a, const CrewsMembersEntry & b ) +{ + return TableKeyOrder( (uint64_t) a.key, (uint64_t) b.key ); // integers compare by VALUE, unsigned here +} +inline int TableEntryOrder( const CrewsMembersEntry & entry, uint32_t key ) +{ + return TableKeyOrder( (uint64_t) entry.key, (uint64_t) key ); +} +inline uint32_t TableEntryKey( const CrewsMembersEntry & entry ) { return entry.key; } +inline void TableEntrySetKey( CrewsMembersEntry & entry, uint32_t key ) { entry.key = key; } +// THIS VALUE'S STORAGE IS A PAIR (§2.8, §4.2, §7.2): `value` beside +// `value_count`, so the handle is the ENTRY and not one member of it. +// Fill both; `key` belongs to the map's own order. +inline const CrewsMembersEntry * TableEntryFound( const CrewsMembersEntry * entry ) { return entry; } +inline CrewsMembersEntry * TableEntryValue( CrewsMembersEntry * entry ) { return entry; } +struct CrewsMembersEntryEach { uint32_t key; decltype( TableEntryValue( (CrewsMembersEntry *) NULL ) ) value; }; +inline CrewsMembersEntryEach TableEntryEach( CrewsMembersEntry * entry ) { return CrewsMembersEntryEach{ TableEntryKey( *entry ), TableEntryValue( entry ) }; } +inline void TableResetMapValue( CrewsMembersEntry & value ) +{ + for ( int32_t i = 0; i < 2; i++ ) { value.value[i].value = 0; } // [2]*Item — every slot null + value.value_count = 0; +} + +// CrewsMembersEntryReadKey: the key, before the slot is chosen (docs/SPEC-TABLES.md §2.8). +// Field order inside a body is not contractual (§3), so this scans rather +// than assuming a position — and this implementation writes the key first, +// so on any wire it wrote the scan ends at the first header. +struct CrewsMembersEntryKeyRead +{ + uint32_t key; + bool found; // the body carried the key's id + bool kind_bad; // it carried it under another kind: the MAP's event + bool widened; // under a kind the declaration WIDENS (§4): decoded exactly, the MAP counts one + bool over; // longer than this reader's bound: the ENTRY is dropped + bool malformed; // the entry's framing gave out +}; + +inline CrewsMembersEntryKeyRead CrewsMembersEntryReadKey( const uint8_t * body, int64_t length, const TableIdTable * ids ) +{ + CrewsMembersEntryKeyRead out = { 0, false, false, false, false, false }; + TableReport scratch; // the scan's own framing damage is the MAP's, raised by the caller + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { out.malformed = true; return out; } + if ( field_ref == 0 ) { return out; } // the terminator: no key field is the key's DEFAULT + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { out.malformed = true; return out; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { out.malformed = true; return out; } + uint8_t field_kind = r.get8(); + if ( field_id == 0x3dc94a19365b10ecull ) // `key`, the ordinary hash of an ordinary name + { + if ( field_kind != 8 && TableKindWidens( field_kind, 8 ) ) + { + out.widened = true; + out.found = true; + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, field_kind, widened_v ) ) { out.malformed = true; return out; } + out.key = (uint32_t) widened_v; + continue; // the LAST occurrence is the one §3 keeps + } + out.kind_bad = field_kind != 8; // THE KEY KIND IS THE READER'S DECLARATION + out.found = !out.kind_bad; + if ( !out.kind_bad ) + { + if ( !r.has( 4 ) ) { out.malformed = true; return out; } + out.key = (uint32_t) r.get32(); + continue; // the LAST occurrence is the one §3 keeps + } + } + if ( !r.skip( field_kind ) ) { out.malformed = true; return out; } + } +} + +// ---- retain-unknown: the second family (docs/SPEC-TABLES.md §6.6) ---- +// +// The same walks, with the PATH threaded and the unknown arm capturing. The +// three above are untouched and cost nothing for these being here: a caller +// that does not ask instantiates none of them. + +template inline int64_t CrewsMembersEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool CrewsMembersEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool CrewsMembersEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool CrewsMembersEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline int64_t CrewsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool CrewsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool CrewsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ); +inline bool CrewsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Crews & value, TableRetain * retain, const TableRetainPath & path ); + +template +inline int64_t CrewsMembersEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const CrewsMembersEntry & value ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + if ( value.value_count < 0 || value.value_count > 2 ) { return -1; } // storage invariant + if ( value.value_count > 0 ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( value.value_count ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < value.value_count; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return -1; } + body_value += TableLebBytes( slot_index ); + } + } + bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); // value: [..2]*Item + } + return bytes; +} + +template +inline bool CrewsMembersEntrySaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const CrewsMembersEntry & value ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 8 ); // key + w.put32( uint32_t( value.key ) ); + } + if ( value.value_count < 0 || value.value_count > 2 ) { return false; } // storage invariant + if ( value.value_count > 0 ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( value.value_count ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < value.value_count; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + body_value += TableLebBytes( slot_index ); + } + } + w.putleb( ref_value ); w.put8( 14 ); w.putleb( (uint64_t) body_value ); // value + w.put8( 17 ); w.putleb( (uint64_t) ( value.value_count ) ); + for ( int32_t elem_i = 0; elem_i < value.value_count; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + w.putleb( slot_index ); + } + } + } + return !w.overflow; +} + +template +inline bool CrewsMembersEntrySaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const CrewsMembersEntry & value ) +{ + if ( !CrewsMembersEntrySaveBodyFields( ctx, numbering, w, ids, value ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool CrewsMembersEntryLoadBody( TableReader & r, const TableNodeMap & nodes, CrewsMembersEntry & value ) +{ + CrewsMembersEntryReset( value ); // prefill declared defaults in place, then overlay + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x3dc94a19365b10ecull: // key + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.key = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.key = decoded_v; + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER — the element kind byte and the + // count, so fewer than two bytes — is INERT (§4): the field keeps the + // value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + // A DAMAGED COUNT stops the elements and nothing else: the field + // RODE, so an optional is still PRESENT (§2.3) — only a foreign + // ELEMENT KIND says the payload is not this array's at all. + if ( !counted_ok ) { r.report->malformed = true; } + else if ( elem_kind != 17 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + uint64_t keep = count; + if ( keep > 2 ) { keep = 2; r.report->clamped++; } + // elements are BOUNDED by the field body: a count the length + // cannot cover keeps the decoded prefix, flags malformed, and + // the parent continues at the next field — following fields' + // bytes are never fabricated into elements + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint64_t decoded = 0; + for ( uint64_t i = 0; i < keep; i++ ) + { + { + uint64_t node_index = 0; + if ( !sub.getleb( node_index ) ) { r.report->malformed = true; break; } + TableNodeResolve( nodes, value.value[(int32_t) i], node_index, 0x52cfa1d198476806ull, r.report ); // *Item + } + decoded = i + 1; + } + value.value_count = (int32_t) decoded; + } + } + r.offset = body_end; // excess elements and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's cost, in BITS (docs/SPEC-TABLES.md §3.3). `at` is the +// body's own bit position in the batch, because a `string(N)` ALIGNS before +// its bytes and an align costs what the position says it costs. +template +inline int64_t CrewsMembersEntryMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const CrewsMembersEntry & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + int64_t bits = 0; + if ( value.key != 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + } + if ( value.value_count < 0 || value.value_count > 2 ) { return -1; } // storage invariant + if ( value.value_count > 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 2; + bits += (int64_t) ( value.value_count ) * index_bits; + } + bits += kTableMessageRefBitsHere; // the ZERO REFERENCE that ends the body + (void) at; + return bits; +} + +// The BITPACKED body: the fields, then the ZERO REFERENCE that ends it. No +// kind byte rides at all, and no length frames a nested body, because a +// body is self-delimiting: it is written where the file form put an L. +template +inline bool CrewsMembersEntrySaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const CrewsMembersEntry & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + if ( value.key != 0 ) + { + w.put( 9, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.key ), 32 ); + } + if ( value.value_count < 0 || value.value_count > 2 ) { return false; } // storage invariant + if ( value.value_count > 0 ) + { + w.put( 21, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.value_count ) - 0, 2 ); + for ( int32_t i = 0; i < value.value_count; i++ ) + { + const Item * pointee_value = ItemAt( ctx, value.value[i] ); // *Item + uint64_t index_value = 0; + if ( pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) pointee_value, index_value ) ) { return false; } + w.put( index_value, index_bits ); + } + } + w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +inline bool CrewsMembersEntryLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, CrewsMembersEntry & value ) +{ + (void) nodes; (void) index_bits; + CrewsMembersEntryReset( value ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 2 ) { kept = 2; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + TableRef scratch; + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( in_bounds ? value.value[i] : scratch ), node_index_2, 0x52cfa1d198476806ull, report ); // *Item + } + } + value.value_count = kept; + } + break; + } + default: + report->unknown++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + } +} + +template +inline int64_t CrewsMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Crews & value ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // members: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_members = TableMapOrder( ctx, value.members ); + if ( !order_members.ok ) { return -1; } // the sort could not run + if ( order_members.count > 0 ) + { + const uint64_t ref_members = ids.ref( 0x79d594675e391090ull ); + int64_t body_members = 1 + TableLebBytes( (uint64_t) order_members.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_members = CrewsMembersEntryMeasureBody( ctx, numbering, ids, *order_members[i] ); + if ( elem_members < 0 ) { TableMapRelease( order_members ); return -1; } + body_members += TableLebBytes( (uint64_t) ( elem_members ) ) + ( elem_members ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_members ) + 1 + TableLebBytes( (uint64_t) ( body_members ) ) + ( body_members ); + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + return bytes; +} + +template +inline bool CrewsSaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Crews & value ) +{ + { + TableMapCursor order_members = TableMapOrder( ctx, value.members ); // members + if ( !order_members.ok ) { return false; } + if ( order_members.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_members = ids.ref( 0x79d594675e391090ull ); + int64_t body_members = 1 + TableLebBytes( (uint64_t) order_members.count ); + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_members = CrewsMembersEntryMeasureBody( ctx, numbering, ids, *order_members[i] ); + if ( elem_members < 0 ) { TableMapRelease( order_members ); return false; } + body_members += TableLebBytes( (uint64_t) ( elem_members ) ) + ( elem_members ); + } + w.putleb( ref_members ); w.put8( 14 ); w.putleb( (uint64_t) body_members ); + w.put8( 13 ); w.putleb( (uint64_t) order_members.count ); + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_len_members = CrewsMembersEntryMeasureBody( ctx, numbering, ids, *order_members[i] ); + if ( elem_len_members < 0 ) { TableMapRelease( order_members ); return false; } + w.putleb( (uint64_t) elem_len_members ); + if ( !CrewsMembersEntrySaveBody( ctx, numbering, w, ids, *order_members[i] ) ) { TableMapRelease( order_members ); return false; } + } + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + return !w.overflow; +} + +template +inline bool CrewsSaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Crews & value ) +{ + if ( !CrewsSaveBodyFields( ctx, numbering, w, ids, value ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool CrewsLoadBody( TableReader & r, const TableNodeMap & nodes, Crews & value ) +{ + CrewsReset( value ); // prefill declared defaults in place, then overlay + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x79d594675e391090ull: // members + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + TableMapFill fill = TableMapFillBegin( nodes, value.members, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + CrewsMembersEntryKeyRead read = CrewsMembersEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + CrewsMembersEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + CrewsMembersEntryLoadBody( elem, nodes, *slot ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's cost, in BITS (docs/SPEC-TABLES.md §3.3). `at` is the +// body's own bit position in the batch, because a `string(N)` ALIGNS before +// its bytes and an align costs what the position says it costs. +template +inline int64_t CrewsMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Crews & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + int64_t bits = 0; + { + TableMapCursor order_members = TableMapOrder( ctx, value.members ); // members + if ( !order_members.ok ) { return -1; } // the sort could not run + if ( order_members.count > 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; // the count the data decides + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_members = CrewsMembersEntryMeasureMessageBody( ctx, numbering, index_bits, at + bits, *order_members[i] ); + if ( elem_members < 0 ) { TableMapRelease( order_members ); return -1; } + bits += elem_members; // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + } + bits += kTableMessageRefBitsHere; // the ZERO REFERENCE that ends the body + (void) at; + return bits; +} + +// The BITPACKED body: the fields, then the ZERO REFERENCE that ends it. No +// kind byte rides at all, and no length frames a nested body, because a +// body is self-delimiting: it is written where the file form put an L. +template +inline bool CrewsSaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Crews & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + { + TableMapCursor order_members = TableMapOrder( ctx, value.members ); // members + if ( !order_members.ok ) { return false; } // the sort could not run + if ( order_members.count > 0 ) + { + w.put( 20, kTableMessageRefBitsHere ); + w.put( (uint64_t) order_members.count, 32 ); // the count the data decides + for ( int32_t i = 0; i < order_members.count; i++ ) + { + if ( !CrewsMembersEntrySaveMessageBody( ctx, numbering, index_bits, w, *order_members[i] ) ) { TableMapRelease( order_members ); return false; } + } + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) + { + w.put( 18, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.after ), 32 ); + } + w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +// CrewsMessageExtent: the extent Crews's maps command on the message wire, from +// the FRAMING alone (docs/SPEC-TABLES.md §2.8, §3.3, §6.5). +inline bool CrewsMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { return false; } + if ( entry.id == 0x79d594675e391090ull && entry.kind == 14 && entry.elem_kind == 13 ) // members + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { return false; } + n += (uint64_t) entry.min; + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( CrewsMembersEntry ) + at += (int64_t) n * (int64_t) sizeof( CrewsMembersEntry ); // the whole array FIRST + for ( uint64_t i = 0; i < n; i++ ) // then, entry by entry in key order + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { return false; } + } + continue; + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { return false; } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +inline bool CrewsLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Crews & value ) +{ + (void) nodes; (void) index_bits; + CrewsReset( value ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x79d594675e391090ull: // members + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + TableMapFill fill = TableMapFillBegin( nodes, value.members, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const CrewsMembersEntryMessageKeyRead read = CrewsMembersEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + CrewsMembersEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !CrewsMembersEntryLoadMessageBody( r, vocabulary, report, nodes, index_bits, *slot ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + } +} + +// CrewsMembersEntryWireExtent: the extent CrewsMembersEntry's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool CrewsMembersEntryWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + (void) body; (void) length; (void) at; (void) ids; (void) reason; // no list or map below this record + return true; +} + +// CrewsMembersEntryExtentAt: the node extent CrewsMembersEntry's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as CrewsMembersEntryExtentPack advances it (§2.8, §2.9). +template +inline bool CrewsMembersEntryExtentAt( const Ctx & ctx, const CrewsMembersEntry & value, int64_t & at ) +{ + (void) ctx; (void) value; (void) at; // no list or map below this record + return true; +} + +template +inline int64_t CrewsMembersEntryExtent( const Ctx & ctx, const CrewsMembersEntry & value ) +{ + (void) ctx; (void) value; // no list or map below this record + return 0; +} + +// CrewsMembersEntryExtentPack: carve CrewsMembersEntry's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset CrewsMembersEntryExtentAt advances (§2.8, §2.9). +template +inline bool CrewsMembersEntryExtentPack( const Ctx & ctx, const CrewsMembersEntry & src, CrewsMembersEntry & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + (void) ctx; (void) src; (void) dst; (void) extent; (void) at; (void) capacity; // no list or map below this record + return true; +} + +// CrewsWireExtent: the extent Crews's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool CrewsWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; // the scan's framing damage is the LOAD's to report + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { return true; } + if ( field_ref == 0 ) { return true; } + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { return true; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { return true; } + uint8_t field_kind = r.get8(); + if ( field_id == 0x79d594675e391090ull && field_kind == 14 ) // members + { + uint64_t map_len = 0; + if ( !r.getleb( map_len ) || !r.room( map_len ) ) { return true; } + const uint8_t * map_body = r.buffer + r.offset; + r.offset += (int64_t) map_len; + if ( !TableMapWireExtent( map_body, (int64_t) map_len, at, (int64_t) sizeof( CrewsMembersEntry ), (int64_t) alignof( CrewsMembersEntry ), NULL, ids, reason ) ) { return false; } + continue; + } + if ( !r.skip( field_kind ) ) { return true; } + } +} + +// CrewsExtentAt: the node extent Crews's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as CrewsExtentPack advances it (§2.8, §2.9). +template +inline bool CrewsExtentAt( const Ctx & ctx, const Crews & value, int64_t & at ) +{ + { + TableMapCursor cursor = TableMapOrder( ctx, value.members ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( CrewsMembersEntry ) + at += (int64_t) cursor.count * (int64_t) sizeof( CrewsMembersEntry ); // the whole array FIRST + for ( int32_t i = 0; i < cursor.count; i++ ) // then, entry by entry in key order + { + if ( !CrewsMembersEntryExtentAt( ctx, *cursor[i], at ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// the whole extent of one node, from a fresh offset: what a pack reserves +// for it beside the record's own storage. +template +inline int64_t CrewsExtent( const Ctx & ctx, const Crews & value ) +{ + int64_t at = 0; + if ( !CrewsExtentAt( ctx, value, at ) ) { return -1; } + return at; +} + +// CrewsExtentPack: carve Crews's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset CrewsExtentAt advances (§2.8, §2.9). +template +inline bool CrewsExtentPack( const Ctx & ctx, const Crews & src, Crews & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + { + TableMapCursor cursor = TableMapOrder( ctx, src.members ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; + const int64_t bytes = (int64_t) cursor.count * (int64_t) sizeof( CrewsMembersEntry ); + if ( at + bytes > capacity ) { TableMapRelease( cursor ); return false; } + CrewsMembersEntry * placed = (CrewsMembersEntry *) ( extent + at ); + at += bytes; + dst.members.count = cursor.count; + dst.members.padding = 0; + dst.members.entries.value = cursor.count > 0 ? (int64_t) ( (uint8_t *) placed - (const uint8_t *) &dst.members.entries ) : 0; + for ( int32_t i = 0; i < cursor.count; i++ ) + { + memcpy( (void *) ( placed + i ), (const void *) cursor[i], sizeof( CrewsMembersEntry ) ); // trivially copyable, by construction + } + for ( int32_t i = 0; i < cursor.count; i++ ) + { + if ( !CrewsMembersEntryExtentPack( ctx, *cursor[i], placed[i], extent, at, capacity ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// ---- Crews.members: the builder's five and the side index (§2.8) ---- + +// INSERT: the key is copied, the value is handed back at its defaults to +// fill. A DUPLICATE key REPLACES — the value is reset and the same entry +// handed back, key and address unchanged — so a caller that wants to know +// writes Find first. NULL is NOT INSERTED: a key longer than the bound, +// because a truncated key would be a merged entry, and an arena that +// cannot carve another segment, alike. +// +// It is a WRAPPER: TableMapPlace owns the lookup, the reset, the +// allocation and the key copy, and this half is the bound and the +// const char * key's length. Nothing here mutates an entry (§2.8). +inline CrewsMembersEntry * CrewsMembersInsert( TableWorker & worker, TableMap & map, uint32_t key ) +{ + CrewsMembersEntry * entry = TableMapPlace( worker, map, key ); + return entry != NULL ? TableEntryValue( entry ) : NULL; +} + +// FIND on the builder: the same linear scan, O( n ) key compares over the +// segments in insertion order. NULL when absent. The builder builds NO +// INDEX, and that is a rule — the sort happens once, at Lock, Save or +// Cook, and every lookup that matters runs over the sorted region. +inline CrewsMembersEntry * CrewsMembersFind( TableArena & arena, TableMap & map, uint32_t key ) +{ + CrewsMembersEntry * found = TableMapScan( arena, map, key ); + return found != NULL ? TableEntryValue( found ) : NULL; +} + +// ERASE: marks the entry DEAD, one bit in the segment's slot and not in the +// entry table. False when absent. Its storage is held until the builder +// resets and never reused mid-build, because reusing a slot would make "an +// entry's address is stable" false for exactly one case. +inline bool CrewsMembersErase( TableArena & arena, TableMap & map, uint32_t key ) +{ + return TableMapErase( arena, map, key ); +} + +// EACH on the builder: INSERTION order, live entries only. +inline TableMapEach CrewsMembersEach( const TableArena & arena, const TableMap & map ) +{ + return TableMapEachOf( arena, map ); +} + +// ---- the OPTIONAL INDEX: caller-owned, built at load, never stored ---- +// +// Open addressing with linear probing over the sorted array, for a map large +// enough that log n compares over a cold array cost more than one hash and a +// probe. ITS HASH AND ITS LOAD FACTOR ARE NOT A CROSS-PORT CONTRACT: the +// index is never stored, so no golden, no cook-check rule and no +// build-version line ever names either. What a port is held to is the +// CONTRACT of the lookup — the same value the sorted array's Find returns +// for the same key, and no allocation past the storage the caller handed in. +inline int64_t CrewsMembersIndexMeasure( const TableMap & map ) +{ + return (int64_t) TableMapIndexSlots( map.count ) * (int64_t) sizeof( int32_t ); +} + +inline TableMapIndex CrewsMembersIndex( const TableMap & map, void * storage, int64_t bytes ) +{ + TableMapIndex index; + const int32_t slots = TableMapIndexSlots( map.count ); + if ( storage == NULL || bytes < (int64_t) slots * (int64_t) sizeof( int32_t ) ) { return index; } + index.slots = (int32_t *) storage; + index.capacity = slots; + for ( int32_t i = 0; i < slots; i++ ) { index.slots[i] = 0; } + const CrewsMembersEntry * entries = map.Entries(); + for ( int32_t i = 0; i < map.count; i++ ) // ONE PASS over the sorted array + { + int32_t at = (int32_t) ( TableMapHash( (uint64_t) entries[i].key ) & (uint64_t) ( slots - 1 ) ); + while ( index.slots[at] != 0 ) { at = ( at + 1 ) & ( slots - 1 ); } + index.slots[at] = i + 1; // slots are ENTRY INDICES; 0 is an empty slot + } + index.good = true; + return index; +} + +inline const CrewsMembersEntry * CrewsMembersIndexFind( const TableMapIndex & index, const TableMap & map, uint32_t key ) +{ + if ( !index.good ) { return map.Find( key ); } // an index that did not build is not a wrong answer + const CrewsMembersEntry * entries = map.Entries(); + int32_t at = (int32_t) ( TableMapHash( (uint64_t) key ) & (uint64_t) ( index.capacity - 1 ) ); + for ( int32_t probe = 0; probe < index.capacity; probe++ ) + { + const int32_t slot = index.slots[at]; + if ( slot == 0 ) { return NULL; } + if ( TableEntryOrder( entries[slot - 1], key ) == 0 ) { return TableEntryFound( entries + slot - 1 ); } + at = ( at + 1 ) & ( index.capacity - 1 ); + } + return NULL; +} + +// CrewsMembersEntryNumber: number everything CrewsMembersEntry POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool CrewsMembersEntryNumber( const Ctx & ctx, TableNumbering & numbering, const CrewsMembersEntry & value ) +{ + for ( int32_t k = 0; k < value.value_count && k < 2; k++ ) // value: [..2]*Item + { + { + const Item * pointee = ItemAt( ctx, value.value[k] ); // value + if ( pointee != NULL ) + { + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( numbering.seen, (const void *) pointee, + (int64_t) ( numbering.count + 2 ), taken, slot ); // its index, if this is its first visit + if ( entry == NULL ) { return false; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return false; } // a data cycle + } + else + { + TableNodeEntry node; + node.node = (const void *) pointee; + node.type_id = 0x52cfa1d198476806ull; // fnv1a64( "Item" ) + node.type_slot = 65; // its slot in the unit's vocabulary (§3.3) + node.measure = &TableNodeMeasureThunk; + node.save = &TableNodeSaveThunk; + node.message_measure = &TableNodeMessageMeasureThunk; + node.message_save = &TableNodeMessageSaveThunk; + if ( !TableNumberingAppend( numbering, node ) ) { return false; } + if ( !ItemNumber( ctx, numbering, *pointee ) ) { return false; } + TablePackMapClose( numbering.seen, (const void *) pointee, slot ); + } + } + } + } + return true; +} + +// CrewsMembersEntryPackMeasure: the packed region bytes of everything CrewsMembersEntry POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t CrewsMembersEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const CrewsMembersEntry & value ) +{ + int64_t bytes = 0; + for ( int32_t k = 0; k < value.value_count && k < 2; k++ ) // value: [..2]*Item + { + { + const Item * pointee = ItemAt( ctx, value.value[k] ); // value + if ( pointee != NULL ) + { + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( seen, (const void *) pointee, 0, taken, slot ); + if ( entry == NULL ) { return -1; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return -1; } // a data cycle + } + else + { + int64_t inner = ItemPackMeasure( ctx, seen, *pointee ); + if ( inner < 0 ) { return -1; } + TablePackMapClose( seen, (const void *) pointee, slot ); + bytes += TableAlignUp64( (int64_t) sizeof( Item ) ) + inner; + } + } + } + } + return bytes; +} + +// CrewsMembersEntryPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool CrewsMembersEntryPackEdges( const Ctx & ctx, TablePackMap & seen, const CrewsMembersEntry & src, CrewsMembersEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool CrewsMembersEntryPack( const Ctx & ctx, TablePackMap & seen, const CrewsMembersEntry & src, CrewsMembersEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( CrewsMembersEntry ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( CrewsMembersEntry ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !CrewsMembersEntryExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return CrewsMembersEntryPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool CrewsMembersEntryPackEdges( const Ctx & ctx, TablePackMap & seen, const CrewsMembersEntry & src, CrewsMembersEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + for ( int32_t k = 0; k < src.value_count && k < 2; k++ ) // value: [..2]*Item + { + { + dst.value[k].value = 0; // value + const Item * pointee = ItemAt( ctx, src.value[k] ); + if ( pointee != NULL ) + { + int64_t at = TableAlignUp64( used ); // where it WOULD land, if this is its first visit + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( seen, (const void *) pointee, at, taken, slot ); + if ( entry == NULL ) { return false; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return false; } // a data cycle + dst.value[k].value = (int64_t) ( ( base + entry->offset ) - (const uint8_t *) &dst.value[k] ); // the one body it already has + } + else + { + if ( at + (int64_t) sizeof( Item ) > capacity ) { return false; } + used = at + TableAlignUp64( (int64_t) sizeof( Item ) ); + Item * child = new ( base + at ) Item; // lifetime only: the Pack below memcpy's the whole node over it + dst.value[k].value = (int64_t) ( ( base + at ) - (const uint8_t *) &dst.value[k] ); + if ( !ItemPack( ctx, seen, *pointee, *child, base, capacity, used ) ) { return false; } + TablePackMapClose( seen, (const void *) pointee, slot ); + } + } + } + } + return true; +} + +// CrewsNumber: number everything Crews POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool CrewsNumber( const Ctx & ctx, TableNumbering & numbering, const Crews & value ) +{ + { // members: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_members = TableMapOrder( ctx, value.members ); + if ( !cursor_members.ok ) { return false; } + for ( int32_t i = 0; i < cursor_members.count; i++ ) + { + if ( !CrewsMembersEntryNumber( ctx, numbering, *cursor_members[i] ) ) { TableMapRelease( cursor_members ); return false; } + } + TableMapRelease( cursor_members ); + } + return true; +} + +// CrewsPackMeasure: the packed region bytes of everything Crews POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t CrewsPackMeasure( const Ctx & ctx, TablePackMap & seen, const Crews & value ) +{ + int64_t bytes = 0; + { // members: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_members = TableMapOrder( ctx, value.members ); + if ( !cursor_members.ok ) { return -1; } + for ( int32_t i = 0; i < cursor_members.count; i++ ) + { + int64_t inner = CrewsMembersEntryPackMeasure( ctx, seen, *cursor_members[i] ); + if ( inner < 0 ) { TableMapRelease( cursor_members ); return -1; } + bytes += inner; + } + TableMapRelease( cursor_members ); + } + return bytes; +} + +// CrewsPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool CrewsPackEdges( const Ctx & ctx, TablePackMap & seen, const Crews & src, Crews & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool CrewsPack( const Ctx & ctx, TablePackMap & seen, const Crews & src, Crews & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( Crews ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( Crews ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !CrewsExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return CrewsPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool CrewsPackEdges( const Ctx & ctx, TablePackMap & seen, const Crews & src, Crews & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + { // members: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_members = TableMapOrder( ctx, src.members ); + if ( !cursor_members.ok ) { return false; } + CrewsMembersEntry * placed_members = (CrewsMembersEntry *) ( dst.members.entries.value != 0 ? ( (uint8_t *) &dst.members.entries + dst.members.entries.value ) : NULL ); + for ( int32_t i = 0; i < cursor_members.count; i++ ) + { + if ( !CrewsMembersEntryPackEdges( ctx, seen, *cursor_members[i], placed_members[i], base, capacity, used ) ) { TableMapRelease( cursor_members ); return false; } + } + TableMapRelease( cursor_members ); + } + return true; +} + +// ---- Crews: the variable-length life (docs/SPEC-TABLES.md §2, §6, §9) ---- +// +// MUTABLE: CrewsBuilder — allocate nodes, wire them together, then Lock. +// CONST: one packed region, root at its base. Lock produces it and Load +// produces it, so a locked structure and a loaded one are the +// SAME representation with one view API. There is no unlock: +// re-editing means loading the const form into a fresh builder. +// Crews is never held by value — a file-format-scale structure is a region +// and a root pointer, not a struct you copy. + +struct CrewsBuilder +{ + TableArena arena; + TableWorker main; // the calling thread's allocation front + TableRef root_ref; + uint8_t * region = NULL; // the packed const form, produced by Lock() + int64_t region_bytes = 0; + + // THE ALLOCATOR IS THE BUILDER'S, and everything this structure ever + // allocates goes through it: the arena's segments, Lock's identity map, + // the packed region, the wire walks' numbering, and the tool path's node + // directory. Name your own and a profiler sees every byte under it. + CrewsBuilder( TableAllocator allocator = TableDefaultAllocator() ) + { + TableArenaInit( arena, allocator ); + main.arena = &arena; + TableSlot slot = main.Alloc(); + root_ref = slot.ref; + } + ~CrewsBuilder() { TableArenaShutdown( arena ); arena.allocator.free( arena.allocator.context, region ); } + CrewsBuilder( const CrewsBuilder & ) = delete; + CrewsBuilder & operator=( const CrewsBuilder & ) = delete; + + // Alloc a node in THIS thread's slab: no lock, no atomic per node. + // The result is usable both as the node pointer and as the reference + // to store in a pointer field. + template TableSlot Alloc() { return main.Alloc(); } + // a BYTE BUFFER's node of exactly `length` bytes (docs/SPEC-TABLES.md §2.5): + // the bytes to write through, and the reference to store in a *bytes + // or *string slot; a blob past a slab takes a span of its own + TableBytesSlot AllocBytes( int64_t length ) { return main.AllocBytes( length ); } + TableStringSlot AllocString( int64_t length ) { return main.AllocString( length ); } + // one worker per thread; allocate on your own, and synchronize your own + // writes to nodes another worker allocated + TableWorker Worker() { TableWorker worker; worker.arena = &arena; return worker; } + + // GetRoot/AsConst, not Root/Const: a member function hides the type + // name it shares, and `table Root` is this spec's own canonical + // example. The checker refuses a table named after any member here, + // so the remaining spellings cannot collide either. + Crews * GetRoot() { return arena.locked ? NULL : (Crews *) TableArenaAt( arena, (uint32_t) root_ref.value ); } + bool Locked() const { return arena.locked; } + const Crews * AsConst() const { return (const Crews *) region; } + const uint8_t * Region() const { return region; } + int64_t RegionBytes() const { return region_bytes; } + + // Lock is ONE WAY and it is the compaction: the segmented arena becomes + // one exact-packed region with zero slack, references rewritten + // self-relative, and the mutable life released. Single-threaded: call + // it after the workers have joined. + bool Lock(); +}; + +inline bool CrewsBuilder::Lock() +{ + if ( arena.locked ) { return region != NULL; } + if ( root_ref.null() ) { return false; } + TableArenaCtx ctx = { &arena }; + const Crews & root = *(const Crews *) TableArenaAt( arena, (uint32_t) root_ref.value ); + // The ROOT takes the map's first entry: it is packed at offset 0, and its + // descent is open for the whole walk (docs/SPEC-TABLES.md §3.1). + TablePackMap seen; + TablePackMapInit( seen, arena.allocator ); + bool root_taken = false; + int64_t root_slot = 0; + int64_t below = -1; + if ( TablePackMapReach( seen, (const void *) &root, 0, root_taken, root_slot ) != NULL ) + { + below = CrewsPackMeasure( ctx, seen, root ); + } + if ( below < 0 ) { TablePackMapShutdown( seen ); return false; } // a data cycle, named at the reference that closes it + int64_t root_extent = CrewsExtent( ctx, root ); + if ( root_extent < 0 ) { TablePackMapShutdown( seen ); return false; } // the sort could not run + int64_t total = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ) + below; + // the AUTHORING path may allocate (§6.5), and it does so through the + // builder's own pair. The region comes back ZEROED, which is the + // allocator's contract: a packed region carries node padding. + uint8_t * packed = (uint8_t *) arena.allocator.alloc( arena.allocator.context, total ); + if ( packed == NULL ) { TablePackMapShutdown( seen ); return false; } + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + Crews * destination = new ( packed ) Crews; // lifetime only: the Pack below memcpy's the whole node over it + // The pack walk RE-DERIVES the same numbering rather than carrying the + // measure's — nothing passes between them, which is what makes + // `used == total` below a real check and not a tautology (§3.1). The + // map keeps the capacity the measure paid for, so the second walk + // rehashes nothing. + TablePackMapReset( seen ); + if ( TablePackMapReach( seen, (const void *) &root, 0, root_taken, root_slot ) == NULL || + !CrewsPack( ctx, seen, root, *destination, packed, total, used ) || used != total ) + { + TablePackMapShutdown( seen ); + arena.allocator.free( arena.allocator.context, packed ); + return false; + } + TablePackMapShutdown( seen ); + region = packed; + region_bytes = total; + arena.locked = true; // MONOTONIC: there is no unlock + TableArenaShutdown( arena ); + return true; +} + +// ---- Crews on the wire: the FLAT NODE TABLE (docs/SPEC-TABLES.md §3.1) ---- +// +// A pointered save writes every reachable node ONCE, into a node table under +// the reserved id 0xFFFF, and a pointer field rides as a u32 INDEX into it +// under kind 17. No pointer edge is a nesting level, so a chain's length is +// not a depth and two references to one node are one node. + +// CrewsNodeStorage: the region bytes one record commands, or -1 for a type id +// this build cannot name — which keeps its index and reads null. A BYTE +// BUFFER's record commands its header and its bytes (docs/SPEC-TABLES.md §2.5), +// which is the one answer the record's LENGTH decides, and a blob past the +// size cap answers kTableNodeRefused with its reason (§3.1, §6.5). +// A MAP'S ENTRIES RIDE IN THEIR HOLDER'S EXTENT (docs/SPEC-TABLES.md §2.8), +// so a record's storage is its type's PLUS N x sizeof( Entry ) at every +// depth, summed from the FRAMING: N is framing and not a value, and this +// reads no field. kTableNodeRefused is a wire whose N its L cannot carry. +inline int64_t CrewsNodeStorage( uint64_t type_id, int64_t length, TableRefuseReason & reason ) +{ + (void) length; // no byte buffer below this root: every node's storage is its type's + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( (int64_t) sizeof( Item ) ); // Item + default: break; + } + (void) reason; // no blob and no extent below this root: nothing here refuses + return -1; +} + +// CrewsNodePlace: start one record's node's lifetime in the storage pass one +// reserved for it, holding exactly the declared defaults — a byte buffer's +// header holds its length, and its bytes come in pass two. +inline void CrewsNodePlace( uint64_t type_id, uint8_t * at, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: { Item * node = new ( at ) Item; ItemReset( *node ); break; } // Item + default: break; + } +} + +// CrewsNodeRecordBytes: one record's OWN storage, before the extent its maps +// take (docs/SPEC-TABLES.md §2.8) — where a node's extent begins. +inline int64_t CrewsNodeRecordBytes( uint64_t type_id ) +{ + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( (int64_t) sizeof( Item ) ); // Item + default: break; + } + return 0; +} + +// CrewsNodeAlloc: the TOOL's path — one record's node in the builder's arena. +// Zero is the arena's null, and it is also what a type id this build cannot +// name answers. +inline uint32_t CrewsNodeAlloc( uint64_t type_id, TableWorker & worker, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return (uint32_t) worker.Alloc().ref.value; // Item + default: break; + } + return 0; +} + +// CrewsNodeBody: PASS TWO's half — decode one record's body into the storage it +// already owns. +inline void CrewsNodeBody( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = CrewsNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? CrewsNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ItemLoadBody( r, *(Item *) at ); break; // Item + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// CrewsNodeMessageStorage: the region bytes one record commands on the message +// wire, or -1 for a type id this build cannot name. A table's is its own +// storage plus the extent its maps take; a byte buffer's is its header and +// its bytes, which is the one answer the record's LENGTH decides. +inline int64_t CrewsNodeMessageStorage( uint64_t type_id, int64_t extent, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Item ) ) + extent ); // Item + default: break; + } + return -1; +} + +// CrewsNodeMessageExtent: step over one TABLE record's body, tallying the extent +// its maps take where its type has any (§2.8). A type this build cannot +// name is stepped over by its announced shapes and takes no extent. +inline bool CrewsNodeMessageExtent( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & extent ) +{ + extent = 0; + (void) type_id; // no map below any node this root can name + return TableMessageSkipBody( r, vocabulary, index_bits ); +} + +// CrewsNodeMessageBody: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +inline bool CrewsNodeMessageBody( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at ) +{ + TableExtentCarve carve; + carve.at = at + CrewsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !CrewsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemLoadMessageBody( r, vocabulary, report, index_bits, *(Item *) at ); break; // Item + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// The numbering both wire walks derive, and NEITHER CARRIES THE OTHER'S: the +// root takes index 1 and its entry stays open for the whole walk, so a +// reference back at it is the cycle it is (§3.1). +template +inline bool CrewsNumberFrom( const Ctx & ctx, TableNumbering & numbering, const Crews & root ) +{ + bool taken = false; + int64_t slot = 0; + if ( TablePackMapReach( numbering.seen, (const void *) &root, (int64_t) kTableNodeIndexRoot, taken, slot ) == NULL ) { return false; } + return CrewsNumber( ctx, numbering, root ); +} + +template +inline int64_t CrewsMeasureWire( const Ctx & ctx, const Crews & root, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bytes = -1; + if ( CrewsNumberFrom( ctx, numbering, root ) ) + { + TableIds ids; + bytes = CrewsMeasureBody( ctx, numbering, ids, root ); + if ( bytes >= 0 ) + { + const int64_t table = TableNodeTableMeasure( ctx, ids, numbering ); + // the FORM BYTE, the ROOT BODY — its own fields, the node table + // and the terminator — and the ID TABLE (docs/SPEC-TABLES.md §3) + bytes = table < 0 || ids.overflow ? -1 : 1 + bytes + table + TableIdsBytes( ids ); + } + } + TableNumberingShutdown( numbering ); + return bytes; +} + +template +inline int64_t CrewsSaveWire( const Ctx & ctx, const Crews & root, uint8_t * buffer, int64_t capacity, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + if ( !CrewsNumberFrom( ctx, numbering, root ) ) { TableNumberingShutdown( numbering ); return -1; } + TableWriter w( buffer, capacity ); + TableIds ids; + w.put8( kTableWireForm ); // the FORM BYTE is the whole header (§3) + // the root's own fields, then the node table's field, then the + // terminator: a reader that gives up inside the table has already + // decoded the ROOT'S OWN FIELDS (§3.1) + bool ok = CrewsSaveBodyFields( ctx, numbering, w, ids, root ) && TableNodeTableSave( ctx, w, ids, numbering ); + TableNumberingShutdown( numbering ); + if ( !ok || ids.overflow ) { return -1; } + w.put8( 0 ); // the ZERO REFERENCE that ends the root body + TableIdsWrite( w, ids ); + if ( w.overflow ) { return -1; } // the caller's buffer was too small + return w.offset; // == CrewsMeasure( root ) +} + +inline int64_t CrewsMeasure( const Crews * root, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return CrewsMeasureWire( ctx, *root, allocator ); +} + +inline int64_t CrewsSave( const Crews * root, uint8_t * buffer, int64_t capacity, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return CrewsSaveWire( ctx, *root, buffer, capacity, allocator ); +} + +inline int64_t CrewsMeasure( const CrewsBuilder & builder ) +{ + if ( builder.region != NULL ) { return CrewsMeasure( builder.AsConst(), builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return CrewsMeasureWire( ctx, *(const Crews *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), builder.arena.allocator ); +} + +inline int64_t CrewsSave( const CrewsBuilder & builder, uint8_t * buffer, int64_t capacity ) +{ + if ( builder.region != NULL ) { return CrewsSave( builder.AsConst(), buffer, capacity, builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return CrewsSaveWire( ctx, *(const Crews *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), buffer, capacity, builder.arena.allocator ); +} + +// ---- Crews on the MESSAGE wire: the batch over a region (docs/SPEC-TABLES.md §3.3) ---- + +// CrewsMessageBodyBits: one root body's bits at bit position `at` of the batch, +// with the numbering derived from the graph, the node table FIRST, then the +// fields, then the zero reference. Measure derives the numbering and save +// derives the same one, and nothing passes between them (§3.1). +template +inline int64_t CrewsMessageBodyBits( const Ctx & ctx, const Crews & root, int64_t at, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bits = -1; + if ( CrewsNumberFrom( ctx, numbering, root ) ) + { + const int64_t index_bits = TableBitsRequired( 0, numbering.count + 1 ); + const int64_t table = TableMessageNodeTableMeasure( ctx, numbering, index_bits, at ); + if ( table >= 0 ) + { + const int64_t body = CrewsMeasureMessageBody( ctx, numbering, index_bits, at + table, root ); + bits = body < 0 ? -1 : table + body; + } + } + TableNumberingShutdown( numbering ); + return bits; +} + +template +inline bool CrewsMessageBodySave( const Ctx & ctx, const Crews & root, TableBitWriter & w, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + bool ok = false; + if ( CrewsNumberFrom( ctx, numbering, root ) ) + { + const int64_t index_bits = TableBitsRequired( 0, numbering.count + 1 ); + ok = TableMessageNodeTableSave( ctx, numbering, index_bits, w ) && CrewsSaveMessageBody( ctx, numbering, index_bits, w, root ); + } + TableNumberingShutdown( numbering ); + return ok && !w.overflow; +} + +// THE PRIMITIVE IS A BATCH (§3.3): a number of ROOTS in one buffer, one count +// and one continuous bit stream, each body carrying its own numbering. A +// root is a locked region's, `builder.AsConst()`, or a loaded one's. M above +// 256 is a refusal by name, batch_too_large, with nothing written. +inline int64_t CrewsMeasureMessages( const Crews * const * roots, int64_t count, TableReport * report, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( roots == NULL || count < 1 ) { return -1; } + if ( count > kTableMessageBatchMax ) { TableMessageRefuseBatch( report ); return -1; } + TableRegionCtx ctx; + int64_t bits = 8; // the body count + for ( int64_t i = 0; i < count; i++ ) + { + if ( roots[i] == NULL ) { return -1; } + const int64_t body = CrewsMessageBodyBits( ctx, *roots[i], bits, allocator ); + if ( body < 0 ) { return -1; } + bits += body; + } + return 1 + ( bits + 7 ) / 8; +} + +inline int64_t CrewsSaveMessages( const Crews * const * roots, int64_t count, uint8_t * buffer, int64_t capacity, TableReport * report, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( roots == NULL || count < 1 ) { return -1; } + if ( count > kTableMessageBatchMax ) { TableMessageRefuseBatch( report ); return -1; } + TableMessageBatch batch; + if ( !TableMessageBatchBegin( batch, buffer, capacity, count ) ) { return -1; } + TableRegionCtx ctx; + for ( int64_t i = 0; i < count; i++ ) + { + if ( roots[i] == NULL || !CrewsMessageBodySave( ctx, *roots[i], batch.w, allocator ) ) { return -1; } + batch.written++; + } + return TableMessageBatchEnd( batch ); // == CrewsMeasureMessages( roots, count, report, allocator ) +} + +// CrewsMessageRecordScan: one node record's type id and the extent its maps +// take, or a blob's length, the reader left after the record. A type id +// reference of 0, one past E, or one naming anything but a kind-0 entry is +// damage, as §3.1 and §3.3 say. +inline bool CrewsMessageRecordScan( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, uint64_t & type_id, int64_t & extent, int64_t & length ) +{ + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { return false; } + TableMessageEntry type_entry; + if ( !TableMessageNameEntry( vocabulary, type_ref, type_entry ) ) { return false; } + type_id = type_entry.id; + extent = 0; + length = 0; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + // A BLOB RECORD CARRIES A LENGTH AT THIRTY-TWO RAW BITS, then ALIGNS, + // then the bytes verbatim (§3.3) + uint64_t n = 0; + if ( !r.get( n, 32 ) || !r.align() || !r.skip( (int64_t) n * 8 ) ) { return false; } + length = (int64_t) n; + return true; + } + return CrewsNodeMessageExtent( type_id, r, vocabulary, index_bits, extent ); +} + +// CrewsMessageBodyStorage: one body's node count and data bytes from the FRAMING +// alone, the reader left at the next body. The node table is walked record +// by record, a table record's body stepped over by its announced shapes and +// a blob's by its length, then the root's own fields. False is a numbering +// that could not be sized; `complete` false is a ROOT body whose own framing +// gave out, which the load meets as damage inside this body after the +// bodies before it were delivered, so the batch is sized through this body +// and no further (§3.3). +inline bool CrewsMessageBodyStorage( TableBitReader & r, const TableVocabulary & vocabulary, int64_t & records, int64_t & data, bool & complete ) +{ + complete = true; + records = 0; + data = 0; + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { return false; } + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !CrewsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { return false; } + const int64_t storage = CrewsNodeMessageStorage( type_id, extent, length ); + if ( storage > 0 ) { data += storage; } // a type id this build cannot name commands none + records++; + } + int64_t root_extent = 0; + if ( !CrewsMessageExtent( r, vocabulary, index_bits, root_extent ) ) { complete = false; } + data += TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + return true; +} + +// CrewsLoadMeasure's MESSAGE overload: the exact region bytes ONE BATCH needs, +// which is one measurement, one allocation and one bounds check for however +// many bodies ride (§3.3, §6.5). It is a scan by the announced shapes and +// reads no field value. The answer is the data bytes plus the attribution, +// one node directory a body, and -1 for a wire it cannot size: no vocabulary, +// another form, or framing that gives out. +inline int64_t CrewsLoadMeasure( const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, int64_t * attribution_bytes = NULL ) +{ + TableReport ignored; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, &ignored ); + if ( bodies < 0 ) { return -1; } + int64_t data = 0, attribution = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + int64_t records = 0, body_data = 0; + bool complete = true; + if ( !CrewsMessageBodyStorage( br.r, vocabulary, records, body_data, complete ) ) { return -1; } + data += body_data; + attribution += ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( !complete ) { break; } // damage inside this body: the load delivers the ones before it + } + if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; } + return data + attribution; +} + +// CrewsLoadMessageBodyInto: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +inline bool CrewsLoadMessageBodyInto( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Crews * & root_out ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !CrewsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = CrewsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + CrewsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !CrewsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xc85b940060088651ull; + Crews * root = new ( region + used ) Crews; // lifetime only: LoadMessageBody's first act is CrewsReset + CrewsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Crews ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + out->unknown += unknown_records; + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !CrewsNodeMessageBody( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return CrewsLoadMessageBody( r, vocabulary, out, nodes, index_bits, *root ); +} + +// CrewsLoadMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +inline bool CrewsLoadMessages( const Crews ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !CrewsLoadMessageBodyInto( br.r, vocabulary, out, region, region_bytes, used, roots[b] ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + +// CrewsLoadMeasure: the exact region bytes a wire buffer will need, and it is +// ONE SCAN — a record's type id gives its storage size, its length gives the +// next record — reading no field value at all, so the caller owns the +// allocation and can refuse a number it did not expect (§6.5). +// +// It reports the DATA bytes and the ATTRIBUTION bytes separately, because the +// attribution is the wire's numbering made resident (§6.3) and a caller may +// release it once Load returns. The answer is their sum. +inline int64_t CrewsLoadMeasure( const uint8_t * wire_file, int64_t wire_file_bytes, int64_t * attribution_bytes = NULL, TableRefuseReason * reason_out = NULL ) +{ + TableReport ignored; + TableIdTable ids_table; + int64_t body_bytes = 0; + // a FORM BYTE this build does not carry is refused by name (§3, §6.5); + // a trailer that cannot be read whole is damage and names no reason + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict == TableOpenRefused ) { if ( reason_out != NULL ) { *reason_out = unknown_form; } return -1; } + if ( verdict != TableOpenOk ) { return -1; } + // ANY BYTE BETWEEN THE ROOT'S TERMINATOR AND THE TABLE'S FIRST ENTRY + // IS MALFORMED (docs/SPEC-TABLES.md §3): the two ends of the file have + // met, nothing is decoded, and no region is sized from it. + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) { return -1; } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &ignored, &ids_table ); + TableRefuseReason reason = count_over_length; + int64_t root_extent = 0; + if ( !CrewsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { if ( reason_out != NULL ) { *reason_out = reason; } return -1; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + int64_t records = 0; + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = CrewsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { if ( reason_out != NULL ) { *reason_out = reason; } return -1; } // an N the record's framing cannot carry, or a blob past the cap (§2.8, §2.9, §3.1) + if ( storage > 0 ) { data += storage; } // a type id this build cannot name commands none + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; } + return data + attribution; +} + +// CrewsLoad: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +inline const Crews * CrewsLoad( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Crews ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !CrewsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = CrewsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0xc85b940060088651ull; + Crews * root = new ( region ) Crews; // lifetime only: LoadBody's first act is CrewsReset + CrewsReset( *root ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = CrewsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + CrewsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + if ( nodes.good ) { out->unknown += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + CrewsNodeBody( type_id, sub, nodes, region + directory[k + 1].offset ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Crews ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + CrewsLoadBody( r, nodes, *root ); + return root; +} + +// CrewsLoadBuilder: the TOOL's path — the same tolerant decode into a fresh +// builder, so loaded data can be edited and locked again. The numbering is +// the same one; what differs is where a node lives and therefore what a +// resolved slot holds — an arena offset here, a self-relative delta there. +inline bool CrewsLoadBuilder( CrewsBuilder & builder, const uint8_t * wire_file, int64_t wire_file_bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; } + return false; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return false; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + Crews * root = builder.GetRoot(); + if ( root == NULL ) { out->malformed = true; return false; } + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) { records++; } + } + // the AUTHORING side may allocate (§6.5), and this is the tool's path. + // It goes through the builder's own pair, like everything else the + // builder reaches, and the entries come back zeroed. + const TableAllocator allocator = builder.arena.allocator; + TableNodeDirEntry * directory = (TableNodeDirEntry *) allocator.alloc( allocator.context, ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ) ); + if ( directory == NULL ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) builder.root_ref.value; + directory[0].type_id = 0xc85b940060088651ull; + TableNodeMap nodes; + nodes.base = NULL; + nodes.entries = directory; + nodes.count = records + 1; + nodes.arena = true; // a resolved slot holds the node's ARENA OFFSET here + nodes.worker = &builder.main; // and a map's entries and a list's elements are the arena's, not a node extent's (§2.8, §2.9) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + uint32_t at = CrewsNodeAlloc( type_id, builder.main, length ); + if ( at == 0 ) + { + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + } + else + { + directory[k + 1].offset = (uint64_t) at; + } + directory[k + 1].type_id = type_id; + k++; + } + nodes.good = TableNodeScanWhole( scan ); + if ( nodes.good ) { out->unknown += unknown_records; } else { out->malformed = true; } + } + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + CrewsNodeBody( type_id, sub, nodes, TableArenaAt( builder.arena, (uint32_t) directory[k + 1].offset ) ); + } + k++; + } + } + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.worker = &builder.main; + nodes.carve = &root_carve; + bool ok = CrewsLoadBody( r, nodes, *root ); + // A COUNT ABOVE THE int32 CAP is this path's refusal (docs/SPEC-TABLES.md + // §2.9): the partial builder is the caller's to discard, and the report + // holds what it held when the count was met + ok = ok && !nodes.refused; + allocator.free( allocator.context, directory ); + return ok; +} + +template +inline int64_t CrewsMembersEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + if ( value.value_count < 0 || value.value_count > 2 ) { return -1; } // storage invariant + if ( value.value_count > 0 ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( value.value_count ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < value.value_count; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return -1; } + body_value += TableLebBytes( slot_index ); + } + } + bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); // value: [..2]*Item + } + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool CrewsMembersEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 8 ); // key + w.put32( uint32_t( value.key ) ); + } + if ( value.value_count < 0 || value.value_count > 2 ) { return false; } // storage invariant + if ( value.value_count > 0 ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( value.value_count ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < value.value_count; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + body_value += TableLebBytes( slot_index ); + } + } + w.putleb( ref_value ); w.put8( 14 ); w.putleb( (uint64_t) body_value ); // value + w.put8( 17 ); w.putleb( (uint64_t) ( value.value_count ) ); + for ( int32_t elem_i = 0; elem_i < value.value_count; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + w.putleb( slot_index ); + } + } + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool CrewsMembersEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !CrewsMembersEntrySaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool CrewsMembersEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, CrewsMembersEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + CrewsMembersEntryReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x3dc94a19365b10ecull: // key + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.key = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.key = decoded_v; + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER — the element kind byte and the + // count, so fewer than two bytes — is INERT (§4): the field keeps the + // value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + // A DAMAGED COUNT stops the elements and nothing else: the field + // RODE, so an optional is still PRESENT (§2.3) — only a foreign + // ELEMENT KIND says the payload is not this array's at all. + if ( !counted_ok ) { r.report->malformed = true; } + else if ( elem_kind != 17 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + uint64_t keep = count; + if ( keep > 2 ) { keep = 2; r.report->clamped++; } + // elements are BOUNDED by the field body: a count the length + // cannot cover keeps the decoded prefix, flags malformed, and + // the parent continues at the next field — following fields' + // bytes are never fabricated into elements + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint64_t decoded = 0; + for ( uint64_t i = 0; i < keep; i++ ) + { + { + uint64_t node_index = 0; + if ( !sub.getleb( node_index ) ) { r.report->malformed = true; break; } + TableNodeResolve( nodes, value.value[(int32_t) i], node_index, 0x52cfa1d198476806ull, r.report ); // *Item + } + decoded = i + 1; + } + value.value_count = (int32_t) decoded; + } + } + r.offset = body_end; // excess elements and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +template +inline int64_t CrewsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // members: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_members = TableMapOrder( ctx, value.members ); + if ( !order_members.ok ) { return -1; } // the sort could not run + if ( order_members.count > 0 ) + { + const uint64_t ref_members = ids.ref( 0x79d594675e391090ull ); + int64_t body_members = 1 + TableLebBytes( (uint64_t) order_members.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_members = CrewsMembersEntryMeasureBodyRetain( ctx, numbering, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_members < 0 ) { TableMapRelease( order_members ); return -1; } + body_members += TableLebBytes( (uint64_t) ( elem_members ) ) + ( elem_members ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_members ) + 1 + TableLebBytes( (uint64_t) ( body_members ) ) + ( body_members ); + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool CrewsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + { + TableMapCursor order_members = TableMapOrder( ctx, value.members ); // members + if ( !order_members.ok ) { return false; } + if ( order_members.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_members = ids.ref( 0x79d594675e391090ull ); + int64_t body_members = 1 + TableLebBytes( (uint64_t) order_members.count ); + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_members = CrewsMembersEntryMeasureBodyRetain( ctx, numbering, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_members < 0 ) { TableMapRelease( order_members ); return false; } + body_members += TableLebBytes( (uint64_t) ( elem_members ) ) + ( elem_members ); + } + w.putleb( ref_members ); w.put8( 14 ); w.putleb( (uint64_t) body_members ); + w.put8( 13 ); w.putleb( (uint64_t) order_members.count ); + for ( int32_t i = 0; i < order_members.count; i++ ) + { + const int64_t elem_len_members = CrewsMembersEntryMeasureBodyRetain( ctx, numbering, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_len_members < 0 ) { TableMapRelease( order_members ); return false; } + w.putleb( (uint64_t) elem_len_members ); + if ( !CrewsMembersEntrySaveBodyRetain( ctx, numbering, w, ids, *order_members[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_members ); return false; } + } + } + TableMapRelease( order_members ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool CrewsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !CrewsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool CrewsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Crews & value, TableRetain * retain, const TableRetainPath & path ) +{ + CrewsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x79d594675e391090ull: // members + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.members, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + CrewsMembersEntryKeyRead read = CrewsMembersEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + CrewsMembersEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + CrewsMembersEntryLoadBodyRetain( elem, nodes, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// CrewsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void CrewsNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = CrewsNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? CrewsNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ItemLoadBodyRetain( r, *(Item *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Item + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// CrewsLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +// UNDER RETENTION it also fills the caller's two stores with the fields +// this build cannot name, and the report carries what it could not keep +// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the +// reader's data is exactly what it would have been with retention off. +inline const Crews * CrewsLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Crews ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !CrewsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = CrewsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0xc85b940060088651ull; + Crews * root = new ( region ) Crews; // lifetime only: LoadBody's first act is CrewsReset + CrewsReset( *root ); + + // LoadRetain RESETS BOTH STORES and writes into neither id list: a + // retained record carries its field's identity in the record itself, + // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer + // belongs to this region from here on. + TableRetainReset( retain, nodes, region ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Crews ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = CrewsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + CrewsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + CrewsNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Crews ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + CrewsLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// CrewsMeasureRetain and CrewsSaveRetain: the pair, with the retained tail in +// every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same +// records under the same walk, so Measure's answer is the size the save +// writes even where a record could not be placed. +template +inline int64_t CrewsMeasureWireRetain( const Ctx & ctx, const Crews & root, TableRetain * retain, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bytes = -1; + auto retain_measure = []( const Ctx & c, const TableNumbering & nn, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> int64_t + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemMeasureBodyRetain( ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return -1; + }; + if ( CrewsNumberFrom( ctx, numbering, root ) ) + { + TableRetainIds ids( retain ); + if ( retain != NULL ) { retain->id_used = 0; } // one walk fills the list, and the save's own walk refills it + bytes = CrewsMeasureBodyRetain( ctx, numbering, ids, root, retain, TableRetainPathRoot( (const void *) &root, 1 ) ); + if ( bytes >= 0 ) + { + const int64_t table = TableNodeTableMeasureRetain( ctx, ids, numbering, retain, retain_measure ); + bytes = table < 0 || ids.overflow ? -1 : 1 + bytes + table + TableRetainIdsBytes( ids ); + } + } + TableNumberingShutdown( numbering ); + return bytes; +} + +template +inline int64_t CrewsSaveWireRetain( const Ctx & ctx, const Crews & root, TableRetain * retain, uint8_t * buffer, int64_t capacity, TableReport * report ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, TableDefaultAllocator() ); + auto retain_measure = []( const Ctx & c, const TableNumbering & nn, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> int64_t + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemMeasureBodyRetain( ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return -1; + }; + auto retain_save = []( const Ctx & c, const TableNumbering & nn, TableWriter & ww, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> bool + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ww; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemSaveBodyRetain( ww, ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return false; + }; + if ( !CrewsNumberFrom( ctx, numbering, root ) ) { TableNumberingShutdown( numbering ); return -1; } + TableWriter w( buffer, capacity ); + TableRetainIds ids( retain ); + if ( retain != NULL ) { retain->id_used = 0; } + TableRetainClearPlaced( retain ); + w.put8( kTableWireForm ); // the FORM BYTE is the whole header (§3) + // the root's own fields, then the RETAINED TAIL, then the node table's + // field: a retained field is one of the root's own values, and the tail + // is pinned before the large and damage-prone part (§6.6, §3.1) + bool ok = CrewsSaveBodyFieldsRetain( ctx, numbering, w, ids, root, retain, TableRetainPathRoot( (const void *) &root, 1 ) ) && + TableNodeTableSaveRetain( ctx, w, ids, numbering, retain, retain_measure, retain_save ); + TableNumberingShutdown( numbering ); + if ( !ok || ids.overflow ) { return -1; } + w.put8( 0 ); // the ZERO REFERENCE that ends the root body + TableRetainIdsWrite( w, ids ); + if ( w.overflow ) { return -1; } // the caller's buffer was too small + // THE SAVE'S OWN SHARE OF retain_lost, read after the save (§6.6): every + // record the walk did not place, counted once. + TableRetainCountLost( retain, report ); + return w.offset; +} + +inline int64_t CrewsMeasureRetain( const Crews * root, TableRetain * retain, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return CrewsMeasureWireRetain( ctx, *root, retain, allocator ); +} + +// SaveRetain REFUSES A NULL REPORT and returns -1 (docs/SPEC-TABLES.md +// §6.6): the save is the only place a caller learns that a record was +// dropped, so the report is required here where it is optional everywhere +// else. A surface that let a caller retain, save and never find out would +// be a promise it could not check. +inline int64_t CrewsSaveRetain( const Crews * root, TableRetain * retain, uint8_t * buffer, int64_t capacity, TableReport * report ) +{ + if ( root == NULL || report == NULL ) { return -1; } + TableRegionCtx ctx; + return CrewsSaveWireRetain( ctx, *root, retain, buffer, capacity, report ); +} + +// CrewsSaveRetainMessages: RETENTION WRITING FORM 2 IS REFUSED BY NAME +// (docs/SPEC-TABLES.md §3.3). It is a MISUSE refusal on §6.6's own +// precedent and never a silent drop, and the two answers are named: a +// caller that must carry unknowns across a rewrite writes the FILE form, +// which carries its own table and takes §6.6 unchanged, and a RELAY +// forwards the sending peer's announcement and its batch bytes verbatim. +template +inline int64_t CrewsSaveRetainMessages( Args &&... ) +{ + static_assert( sizeof...( Args ) == (size_t) -1, + "Crews: a form 2 writer names entries through slots of a vocabulary the compiler settled, and a retained id is one this build's closure does not contain, so it has neither a slot nor an announced shape. Retention writing the MESSAGE form is refused by name (docs/SPEC-TABLES.md §3.3). Write the FILE form, which carries its own table and takes §6.6 unchanged, or relay the sender's announcement and batch bytes verbatim." ); + return -1; +} + +// ---- the cooked form: point at a cook (docs/SPEC-TABLES.md §7) ---- + +// CrewsOpen: match the header and POINT. On a match the bytes ARE what this +// build wrote, in this build's layout and this build's byte order, so there +// is nothing to validate and nothing to fix up and the root comes back as it +// lies. On ANY refusal it returns NULL and NAMES the refusal in the caller's +// TableRefuseReason, the first failing clause in §7's order (a wrong build +// version is a re-cook, a foreign order a cross-endian cook, a truncated +// file a bad download, an unaligned base the caller's own buffer), and the +// caller falls back to a wire load, which is the path that carries every +// version. The reason is written on the refusal path only; a caller that +// passes nothing gets the null alone. +// +// It is O(1) IN THE FILE'S SIZE — the header and nothing per node — so a one +// megabyte cook and a one gigabyte cook open in the same time, and a mapped +// file's pages are touched only as they are used. That is a property of +// touching nothing at open rather than a separate mechanism. +// +// A REFERENCE INSIDE THE REGION IS DEREFERENCED THROUGH CrewsAt: the slot holds +// the signed self-relative byte delta of §6.3, so a deref is one add and +// needs no base pointer, a whole region relocates by plain memcpy, and a +// delta of zero is null. +// +// There is ONE entry point and no tolerant twin: a build either wrote this +// file or it did not, and the build version is what says which. Validating a +// file whose provenance a person doubts is schema cook-check, offline, +// over the ATTRIBUTION part beside the data — a person's decision, never a +// parameter on a load. +inline const Crews * CrewsOpen( const void * bytes, uint64_t length, TableRefuseReason * reason = NULL ) +{ + return (const Crews *) TableCookOpen( bytes, length, (uint64_t) sizeof( Crews ), (uint64_t) alignof( Crews ), reason ); +} + +// ---- the cooked form: WRITE a cook (docs/SPEC-TABLES.md §7.6) ---- +// +// The bytes are `schema cook`'s, and the tool stays the reference: the two +// writers are held to one file, byte for byte, in both byte orders. A cook is +// content-addressed by (asset hash, build version), so two writers of one +// instance produce ONE artifact or the pair means nothing. + +template inline bool CrewsMembersEntryCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const CrewsMembersEntry & value, TableByteOrder order ); +template inline bool CrewsCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Crews & value, TableByteOrder order ); + +template inline bool CrewsMembersEntryCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const CrewsMembersEntry & value, TableByteOrder order ) +{ + table_cook_put( at + 0, (uint64_t) value.key, 4, order ); + for ( int32_t i = 0; i < 2; i++ ) // value: an array of pointers, every slot + { + if ( !table_cook_ref( region, at + 8 + i * 8, (const void *) ItemAt( ctx, value.value[ i ] ), order ) ) { return false; } + } + table_cook_put( at + 24, (uint64_t) (uint32_t) value.value_count, 4, order ); + return true; +} + +template inline bool CrewsCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Crews & value, TableByteOrder order ) +{ + (void) ctx; (void) region; // no reference resolves in this body: a list's and a map's slots are the extent writer's, and the class was decided elsewhere in the closure + table_cook_put( at + 0, 0, 8, order ); // members: the array's delta, filled by the extent writer + table_cook_put( at + 8, 0, 4, order ); // and its count + table_cook_put( at + 16, (uint64_t) value.after, 4, order ); + return true; +} + +template inline bool CrewsMembersEntryCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const CrewsMembersEntry & value, TableByteOrder order ); +template inline bool CrewsCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const Crews & value, TableByteOrder order ); + +// CrewsMembersEntryCookExtent: CrewsMembersEntry's arrays into the node's extent, PRE-ORDER, a map's entries +// in ASCENDING key order and a list's elements in INDEX order, each through its +// own cook writer (§2.8, §2.9, §7.6). +template inline bool CrewsMembersEntryCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const CrewsMembersEntry & value, TableByteOrder order ) +{ + (void) ctx; (void) region; (void) extent; (void) at; (void) record; (void) value; (void) order; + return true; // no list or map below this record +} + +// CrewsCookExtent: Crews's arrays into the node's extent, PRE-ORDER, a map's entries +// in ASCENDING key order and a list's elements in INDEX order, each through its +// own cook writer (§2.8, §2.9, §7.6). +template inline bool CrewsCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const Crews & value, TableByteOrder order ) +{ + (void) region; // a table element's and an entry's references resolve through their own bodies + { // members + TableMapCursor cursor = TableMapOrder( ctx, value.members ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( CrewsMembersEntry ) + uint8_t * array = extent + at; + at += (int64_t) cursor.count * 32; // the whole array FIRST + // the SIXTEEN BYTES of the slot: the self-relative delta, then the count + table_cook_put( record + 0, cursor.count > 0 ? (uint64_t) (int64_t) ( array - ( record + 0 ) ) : 0, 8, order ); + table_cook_put( record + 8, (uint64_t) (uint32_t) cursor.count, 4, order ); + for ( int32_t i = 0; i < cursor.count; i++ ) + { + if ( !CrewsMembersEntryCookBody( ctx, region, array + i * 32, *cursor[i], order ) ) { return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// CrewsMembersEntryCookNode: one node, the record, then the extent its lists and maps take (§2.8, §2.9). +template inline bool CrewsMembersEntryCookNode( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const CrewsMembersEntry & value, TableByteOrder order ) +{ + if ( !CrewsMembersEntryCookBody( ctx, region, at, value, order ) ) { return false; } + int64_t extent_at = 0; + return CrewsMembersEntryCookExtent( ctx, region, at + 32, extent_at, at, value, order ); +} + +// CrewsCookNode: one node, the record, then the extent its lists and maps take (§2.8, §2.9). +template inline bool CrewsCookNode( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Crews & value, TableByteOrder order ) +{ + if ( !CrewsCookBody( ctx, region, at, value, order ) ) { return false; } + int64_t extent_at = 0; + if ( !CrewsCookExtent( ctx, region, at + 24, extent_at, at, value, order ) ) { return false; } + return extent_at == CrewsExtent( ctx, value ); // the extent written is the extent measured, or no header is written +} + +// CrewsCookLayout: the tool's own Layout (docs/SPEC-TABLES.md §7.2) over one +// numbering — the root at zero, then every node in index order at +// align_up( offset, alignof ) for its OWN type, no slack between them, the +// data length rounded to the greatest alignment among them and never below +// eight. The offsets go into the region's table when it has one, and are only +// summed when it does not (a measure). A type id the numbering carries that +// this root cannot name is the two walks disagreeing, and it is refused. +// A NODE'S SIZE DEPENDS ON ITS VALUE where a list or a map rides in its extent +// (docs/SPEC-TABLES.md §2.8), so the layout takes the resolution context +// the numbering walked and reads the same arrays that walk read. +template +inline bool CrewsCookLayout( const Ctx & ctx, const Crews & root, const TableNumbering & numbering, TableCookRegion & region ) +{ + region.numbering = &numbering; + region.count = numbering.count + 1; + const int64_t root_extent = CrewsExtent( ctx, root ); + if ( root_extent < 0 ) { return false; } + int64_t offset = 24 + root_extent; // the root at zero, its extent behind it + int64_t align = 8; + if ( region.offsets != NULL ) { region.offsets[0] = 0; } + for ( int64_t k = 0; k < numbering.count; k++ ) + { + int64_t size = 0; + int64_t node_align = 0; + switch ( numbering.entries[k].type_id ) + { + case 0x52cfa1d198476806ull: size = 4; node_align = 4; break; // Item + default: return false; + } + offset = ( offset + node_align - 1 ) & ~( node_align - 1 ); + if ( region.offsets != NULL ) { region.offsets[k + 1] = offset; } + offset += size; + if ( node_align > align ) { align = node_align; } + } + region.bytes = ( offset + align - 1 ) & ~( align - 1 ); + region.align = align; + return true; +} + +// CrewsCookMeasureFrom: the whole cooked file's bytes for one graph — the header, +// the data part and the attribution part (§7.1). IT DEPENDS ON THE VALUE, +// because the answer is the numbering: the depth-first walk of §3.1 is run +// here and run again by the write, and neither carries the other's (§7.6). A +// data cycle is refused by the walk and answers -1. +template +inline int64_t CrewsCookMeasureFrom( const Ctx & ctx, const Crews & root, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + TableCookRegion region; + int64_t bytes = -1; + if ( CrewsNumberFrom( ctx, numbering, root ) && CrewsCookLayout( ctx, root, numbering, region ) ) + { + const int64_t data_offset = ( kTableCookHeaderBytes + region.align - 1 ) & ~( region.align - 1 ); + bytes = data_offset + region.bytes + region.count * (int64_t) sizeof( TableNodeDirEntry ); + } + TableNumberingShutdown( numbering ); + return bytes; +} + +// CrewsCookFrom: write one cooked file of a pointered graph, in the byte order +// the caller names. The bytes are `schema cook`'s, byte for byte (§7.6). +// +// THE CALLER OWNS THE OUTPUT and nothing is allocated toward it. What is +// allocated is the numbering — the identity map, the entry array and one +// offset per node — through the pair handed in, and released before this +// returns (§6.5, §13.9). A capacity short of the measure writes nothing. +// +// THE HEADER IS WRITTEN LAST. A reference the numbering did not carry is +// found while a body is being written, and a write that refuses there has +// already put bytes in the buffer; with no magic ahead of them, no Open can +// mistake them for a cook. +template +inline bool CrewsCookFrom( const Ctx & ctx, const Crews & root, void * out, uint64_t capacity, TableByteOrder order, TableAllocator allocator ) +{ + if ( out == NULL ) { return false; } + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + TableCookRegion region; + bool ok = CrewsNumberFrom( ctx, numbering, root ); + if ( ok ) + { + region.offsets = (int64_t *) allocator.alloc( allocator.context, ( numbering.count + 1 ) * (int64_t) sizeof( int64_t ) ); + ok = region.offsets != NULL && CrewsCookLayout( ctx, root, numbering, region ); + } + if ( ok ) + { + const int64_t data_offset = ( kTableCookHeaderBytes + region.align - 1 ) & ~( region.align - 1 ); + const int64_t attribution = region.count * (int64_t) sizeof( TableNodeDirEntry ); + const int64_t need = data_offset + region.bytes + attribution; + ok = (uint64_t) need <= capacity; + if ( ok ) + { + uint8_t * raw = (uint8_t *) out; + memset( raw, 0, (size_t) need ); // EVERY BYTE NO FIELD COVERS IS ZERO (§7.2) + region.base = raw + data_offset; + // the DATA part: the root at the region's base, then every numbered + // node at the offset the layout gave it, each through its own writer + ok = CrewsCookNode( ctx, region, region.base, root, order ); + for ( int64_t k = 0; ok && k < numbering.count; k++ ) + { + uint8_t * at = region.base + region.offsets[k + 1]; + const void * node = numbering.entries[k].node; + switch ( numbering.entries[k].type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemCookNode( ctx, region, at, *(const Item *) node, order ); break; // Item + default: ok = false; break; + } + } + // the ATTRIBUTION part: the node directory (§6.3), one entry per node + // in index order, for `schema cook-check` + uint8_t * entry = raw + data_offset + region.bytes; + table_cook_put( entry, 0, 8, order ); + table_cook_put( entry + 8, 0xc85b940060088651ull, 8, order ); // the root: fnv1a64( "Crews" ) + for ( int64_t k = 0; k < numbering.count; k++ ) + { + entry += sizeof( TableNodeDirEntry ); + table_cook_put( entry, (uint64_t) region.offsets[k + 1], 8, order ); + table_cook_put( entry + 8, numbering.entries[k].type_id, 8, order ); + } + // and the HEADER (§7.1), every word a u64 in the order the file is + // produced in; the two RESERVED words are the memset's zeros + if ( ok ) + { + table_cook_put( raw + 0, TableCookMagic, 8, order ); + table_cook_put( raw + 8, BuildVersion, 8, order ); + table_cook_put( raw + 16, (uint64_t) ( order == TableByteOrder::Big ? 2 : 1 ), 8, order ); + table_cook_put( raw + 24, (uint64_t) region.bytes, 8, order ); + table_cook_put( raw + 32, (uint64_t) attribution, 8, order ); + table_cook_put( raw + 40, (uint64_t) region.align, 8, order ); + } + } + } + allocator.free( allocator.context, region.offsets ); + TableNumberingShutdown( numbering ); + return ok; +} + +// CrewsCookMeasure / CrewsCook over a REGION root — a locked builder's AsConst, a +// region CrewsLoad produced, or an opened cook — with the pair the numbering +// allocates through as an optional last argument, as the wire's own entries +// take it (§13.9). +inline int64_t CrewsCookMeasure( const Crews * root, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return CrewsCookMeasureFrom( ctx, *root, allocator ); +} + +inline bool CrewsCook( const Crews * root, void * out, uint64_t capacity, TableByteOrder order, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return false; } + TableRegionCtx ctx; + return CrewsCookFrom( ctx, *root, out, capacity, order, allocator ); +} + +// and over a BUILDER, locked or not: the builder's own pair, and the arena +// encoding while it is still mutable (§6.3). +inline int64_t CrewsCookMeasure( const CrewsBuilder & builder ) +{ + if ( builder.region != NULL ) { return CrewsCookMeasure( builder.AsConst(), builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return CrewsCookMeasureFrom( ctx, *(const Crews *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), builder.arena.allocator ); +} + +inline bool CrewsCook( const CrewsBuilder & builder, void * out, uint64_t capacity, TableByteOrder order ) +{ + if ( builder.region != NULL ) { return CrewsCook( builder.AsConst(), out, capacity, order, builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return false; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return CrewsCookFrom( ctx, *(const Crews *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), out, capacity, order, builder.arena.allocator ); +} + +// ---- relocatability, enforced: the wire is a pure length-prefixed +// stream AND the decoded storage is pointer-free — every closure type +// must stay trivially copyable and standard-layout, so instances can be +// memcpy'd, mmap'd, shared across processes, and walked through +// descriptor offsets. A failure here means a pointer, virtual or +// non-trivial member crept into generated storage. +// +// They ask the COMPILER ITSELF, which is what every C++ standard library +// answers the same two questions with — and it costs this header no +// include at all. +// A pointer FIELD is a TableRef — eight bytes and no address — so the +// property holds in BOTH forms: a fixed-size table is one relocatable +// struct, and a packed region is one relocatable block whose references +// are self-relative and therefore survive a plain memcpy. +static_assert( __is_trivially_copyable( CrewsMembersEntry ), "CrewsMembersEntry must stay relocatable" ); +static_assert( __is_standard_layout( CrewsMembersEntry ), "CrewsMembersEntry must stay standard-layout for offsetof" ); +static_assert( __is_trivially_copyable( Crews ), "Crews must stay relocatable" ); +static_assert( __is_standard_layout( Crews ), "Crews must stay standard-layout for offsetof" ); + +// ---- the cook's layout contract (docs/SPEC-TABLES.md §20.3) ---- +// +// The compiler derived every number below from the declaration and folded it +// into the BUILD VERSION; these asserts are this compiler saying whether it +// agrees. The model is not self-evidently right — on 32-bit System V +// alignof(uint64_t) is 4, not 8 — which is precisely why it is asserted +// rather than assumed. +static_assert( sizeof( CrewsMembersEntry ) == 32, "CrewsMembersEntry's sizeof moved: the build version was taken over 32, so a cook of it would not be this build's file (docs/SPEC-TABLES.md §20.3)" ); +static_assert( alignof( CrewsMembersEntry ) == 8, "CrewsMembersEntry's alignof moved: the build version was taken over 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( CrewsMembersEntry, key ) == 0, "CrewsMembersEntry's field key moved: the build version was taken over offset 0 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( CrewsMembersEntry, value ) == 8, "CrewsMembersEntry's field value moved: the build version was taken over offset 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( sizeof( Crews ) == 24, "Crews's sizeof moved: the build version was taken over 24, so a cook of it would not be this build's file (docs/SPEC-TABLES.md §20.3)" ); +static_assert( alignof( Crews ) == 8, "Crews's alignof moved: the build version was taken over 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( Crews, members ) == 0, "Crews's field members moved: the build version was taken over offset 0 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( Crews, after ) == 16, "Crews's field after moved: the build version was taken over offset 16 (docs/SPEC-TABLES.md §20.3)" ); + + +// ---- reflection descriptors (tables only, docs/SPEC-TABLES.md) ---- + +inline const TableTypeInfo * CrewsMembersEntryTableType(); +inline const TableTypeInfo * CrewsTableType(); +// The descriptors are CONSTANT-INITIALISED data, and a field's target is +// the ADDRESS of another descriptor. These declarations are what let a +// self- or mutually-referential graph — Node naming itself through *Node — +// be expressed as constant data instead of a lazy link, which could not +// have been written race-free OR recursion-safe. The whole reflection +// surface is therefore immutable: read it from any thread, any time. +extern const TableTypeInfo CrewsMembersEntryTableInfo; +extern const TableTypeInfo CrewsTableInfo; + +inline const TableFieldInfo CrewsMembersEntryTableFields[] = { + { "key", "key", "uint32", 0x3dc94a19365b10ecull, 8, false, false, NULL, NULL, false, false, 0, (uint32_t) offsetof( CrewsMembersEntry, key ), (uint32_t) sizeof( CrewsMembersEntry::key ), 0xffffffffu, 0xffffffffu, NULL, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, + { "value", "value", "Item", 0x7ce4fd9430e80ceaull, 17, true, true, []( const void * slot ) -> const void * { return (const void *) ItemAt( *(const TableRef *) slot ); }, []( TableWorker & worker, void * slot ) -> void * { return (void *) ItemEmplace( worker, *(TableRef *) slot ); }, true, false, 2, (uint32_t) offsetof( CrewsMembersEntry, value ), (uint32_t) sizeof( CrewsMembersEntry::value[0] ), (uint32_t) offsetof( CrewsMembersEntry, value_count ), 0xffffffffu, &ItemTableInfo, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, +}; +inline const TableTypeInfo CrewsMembersEntryTableInfo = { "CrewsMembersEntry", (uint32_t) sizeof( CrewsMembersEntry ), 2, CrewsMembersEntryTableFields, +[]( void * p ) { CrewsMembersEntryReset( *(CrewsMembersEntry *) p ); }, true, TableDocNone, 0, NULL }; +inline const TableTypeInfo * CrewsMembersEntryTableType() { return &CrewsMembersEntryTableInfo; } + +inline const TableFieldInfo CrewsTableFields[] = { + { "members", "members", "map[uint32]*Item", 0x79d594675e391090ull, 13, true, false, NULL, NULL, true, false, 0, (uint32_t) offsetof( Crews, members ), (uint32_t) sizeof( CrewsMembersEntry ), (uint32_t) offsetof( Crews, members.count ), 0xffffffffu, &CrewsMembersEntryTableInfo, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, []( TableWorker & worker, void * slot, const char *, int32_t, int64_t key_value ) -> void * { return (void *) TableMapPlace( worker, *(TableMap *) slot, (uint32_t) key_value ); }, "", TableDocNone, 0, NULL }, + { "after", "after", "int32", 0xbf82010f6f71eae9ull, 4, false, false, NULL, NULL, false, false, 0, (uint32_t) offsetof( Crews, after ), (uint32_t) sizeof( Crews::after ), 0xffffffffu, 0xffffffffu, NULL, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, +}; +inline const TableTypeInfo CrewsTableInfo = { "Crews", (uint32_t) sizeof( Crews ), 2, CrewsTableFields, +[]( void * p ) { CrewsReset( *(Crews *) p ); }, true, TableDocNone, 0, NULL }; +inline const TableTypeInfo * CrewsTableType() { return &CrewsTableInfo; } + +// ---- the text form (docs/SPEC-TABLES.md §16) ---- + +// Crews in and out of a JSON text (docs/SPEC-TABLES.md §16.7): read into a +// builder, written from a region's const root. A node named more than once +// carries `&node` in the text. Defined in CrewsTable.cpp; link it to use them. +bool CrewsFromJson( CrewsBuilder & builder, const char * text, int64_t bytes, TableReport * report ); +int64_t CrewsToJsonMeasure( const Crews * root, TableAllocator allocator = TableDefaultAllocator() ); +int64_t CrewsToJson( const Crews * root, char * buffer, int64_t capacity, TableAllocator allocator = TableDefaultAllocator() ); + +} // namespace mapdemo diff --git a/testdata/golden/tables/maps/DepthTable.h b/testdata/golden/tables/maps/DepthTable.h index bc2fa8711..7cf956f68 100644 --- a/testdata/golden/tables/maps/DepthTable.h +++ b/testdata/golden/tables/maps/DepthTable.h @@ -377,7 +377,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -879,13 +879,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1407,7 +1407,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1425,10 +1425,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1437,6 +1437,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1445,54 +1446,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3430,25 +3440,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -5692,7 +5704,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6105,8 +6117,8 @@ inline bool TableEnumSlot( Slot value, uint64_t & slot ) switch ( value ) { case Slot::None: slot = 0; return true; - case Slot::Alpha: slot = 44; return true; - case Slot::Beta: slot = 45; return true; + case Slot::Alpha: slot = 50; return true; + case Slot::Beta: slot = 51; return true; default: return false; // no variant names this value: no wire identity } } @@ -6822,7 +6834,7 @@ inline bool SquadSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_roster.ok ) { return false; } // the sort could not run if ( order_roster.count > 0 ) { - w.put( 42, kTableMessageRefBitsHere ); + w.put( 46, kTableMessageRefBitsHere ); w.put( (uint64_t) order_roster.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_roster.count; i++ ) { @@ -7543,14 +7555,14 @@ inline bool DepthSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( body_one < 0 ) { return false; } if ( body_one > kTableMessageRefBitsHere ) // an all-default nested table elides { - w.put( 19, kTableMessageRefBitsHere ); + w.put( 22, kTableMessageRefBitsHere ); if ( !SquadSaveMessageBody( ctx, numbering, index_bits, w, value.one ) ) { return false; } } } if ( value.many_count < 0 || value.many_count > 3 ) { return false; } // storage invariant if ( value.many_count > 0 ) { - w.put( 20, kTableMessageRefBitsHere ); + w.put( 23, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.many_count ) - 0, 2 ); for ( int32_t i = 0; i < value.many_count; i++ ) { @@ -7569,7 +7581,7 @@ inline bool DepthSaveMessageBody( const Ctx & ctx, const TableNumbering & number } if ( pairs_keyed > 0 ) { - w.put( 21, kTableMessageRefBitsHere ); + w.put( 24, kTableMessageRefBitsHere ); w.put( (uint64_t) pairs_keyed, 2 ); // ASCENDING BY VARIANT ORDINAL, which is slot order. It is // this writer's choice and a reader must not rely on it: every @@ -7589,18 +7601,18 @@ inline bool DepthSaveMessageBody( const Ctx & ctx, const TableNumbering & number } if ( value.arm.type != ForceType::None ) { - w.put( 22, kTableMessageRefBitsHere ); + w.put( 25, kTableMessageRefBitsHere ); switch ( value.arm.type ) { case ForceType::Squad: { - w.put( 46, kTableMessageRefBitsHere ); + w.put( 52, kTableMessageRefBitsHere ); if ( !SquadSaveMessageBody( ctx, numbering, index_bits, w, value.arm.squad ) ) { return false; } break; } case ForceType::Plain: { - w.put( 47, kTableMessageRefBitsHere ); + w.put( 53, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.arm.plain ), 32 ); break; } @@ -7609,7 +7621,7 @@ inline bool DepthSaveMessageBody( const Ctx & ctx, const TableNumbering & number } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/DocsTable.h b/testdata/golden/tables/maps/DocsTable.h index acdfed2b6..00d6087fb 100644 --- a/testdata/golden/tables/maps/DocsTable.h +++ b/testdata/golden/tables/maps/DocsTable.h @@ -376,7 +376,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -878,13 +878,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1406,7 +1406,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1424,10 +1424,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1436,6 +1436,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1444,54 +1445,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3429,25 +3439,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -3752,6 +3764,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4128,6 +4150,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -5676,7 +5703,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6458,15 +6485,13 @@ inline bool DocsPagesEntryLoadMessageBody( TableBitReader & r, const TableVocabu { uint64_t n = 0; if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() || !r.has( (int64_t) n * 8 ) ) { report->malformed = true; return false; } - int32_t kept = 0; - if ( n > (uint64_t) 8 ) { kept = 8; report->clamped++; } else { kept = (int32_t) n; } - for ( uint64_t i = 0; i < n; i++ ) - { - uint64_t by = 0; - if ( !r.get( by, 8 ) ) { report->malformed = true; return false; } - if ( (int32_t) i < kept ) { value.key[i] = (char) by; } - } + const uint8_t * text = r.buffer + ( r.offset >> 3 ); + if ( !TableUtf8Valid( text, n ) ) { report->malformed = true; return false; } + const int32_t kept = (int32_t) TableUtf8Clamp( text, n, 8 ); + if ( (uint64_t) kept < n ) { report->clamped++; } + memcpy( value.key, text, (size_t) kept ); value.key[kept] = 0; + r.offset += (int64_t) n * 8; value.key_length = kept; } break; @@ -6766,7 +6791,7 @@ inline bool DocsSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi if ( !order_pages.ok ) { return false; } // the sort could not run if ( order_pages.count > 0 ) { - w.put( 23, kTableMessageRefBitsHere ); + w.put( 26, kTableMessageRefBitsHere ); w.put( (uint64_t) order_pages.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_pages.count; i++ ) { @@ -6777,7 +6802,7 @@ inline bool DocsSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body @@ -7193,7 +7218,7 @@ inline bool DocsPagesEntryNumber( const Ctx & ctx, TableNumbering & numbering, c TableNodeEntry node; node.node = (const void *) blob; node.type_id = kTableStringTypeId; - node.type_slot = 50; // its slot in the unit's vocabulary (§3.3) + node.type_slot = 56; // its slot in the unit's vocabulary (§3.3) node.measure = &TableBlobMeasureThunk; node.save = &TableBlobSaveThunk; node.message_measure = &TableBlobMessageMeasureThunk; diff --git a/testdata/golden/tables/maps/FleetTable.h b/testdata/golden/tables/maps/FleetTable.h index 96d00d68b..67ee3f150 100644 --- a/testdata/golden/tables/maps/FleetTable.h +++ b/testdata/golden/tables/maps/FleetTable.h @@ -376,7 +376,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -878,13 +878,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1406,7 +1406,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1424,10 +1424,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1436,6 +1436,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1444,54 +1445,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3429,25 +3439,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -5691,7 +5703,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6467,6 +6479,37 @@ inline ShipConfig * ShipConfigEmplace( TableWorker & worker, TableRef & slot ) return allocated.ptr; } +// Item is a pointer target. +inline const Item * ItemAt( const TableRef & ref ) // the const form's hot path: one add, no base +{ + return ref.value != 0 ? (const Item *) ( (const uint8_t *) &ref + ref.value ) : NULL; +} +inline Item * ItemAt( TableRef & ref ) +{ + return ref.value != 0 ? (Item *) ( (uint8_t *) &ref + ref.value ) : NULL; +} +inline const Item * ItemAt( const TableRegionCtx &, const TableRef & ref ) { return ItemAt( ref ); } +inline const Item * ItemAt( const TableArenaCtx & ctx, const TableRef & ref ) +{ + return ref.value != 0 ? (const Item *) TableArenaAt( *ctx.arena, (uint32_t) ref.value ) : NULL; +} +// while the builder is mutable, resolve against the arena itself +inline Item * ItemAt( TableArena & arena, const TableRef & ref ) +{ + return ref.value != 0 ? (Item *) TableArenaAt( arena, (uint32_t) ref.value ) : NULL; +} +inline const Item * ItemAt( const TableArena & arena, const TableRef & ref ) +{ + return ref.value != 0 ? (const Item *) TableArenaAt( arena, (uint32_t) ref.value ) : NULL; +} +// allocate one Item in the arena; the slot holds the arena offset +inline Item * ItemEmplace( TableWorker & worker, TableRef & slot ) +{ + TableSlot allocated = worker.Alloc(); + slot = allocated.ref; + return allocated.ptr; +} + // ---- codecs: measure/save/load per closure member ---- inline int64_t ShipConfigMeasureBody( TableIds & ids, const ShipConfig & value ); @@ -6502,6 +6545,9 @@ inline bool FleetLoadBody( TableReader & r, const TableNodeMap & nodes, Fleet & template inline bool ShipConfigNumber( const Ctx & ctx, TableNumbering & numbering, const ShipConfig & value ); template inline int64_t ShipConfigPackMeasure( const Ctx & ctx, TablePackMap & seen, const ShipConfig & value ); template inline bool ShipConfigPack( const Ctx & ctx, TablePackMap & seen, const ShipConfig & src, ShipConfig & dst, uint8_t * base, int64_t capacity, int64_t & used ); +template inline bool ItemNumber( const Ctx & ctx, TableNumbering & numbering, const Item & value ); +template inline int64_t ItemPackMeasure( const Ctx & ctx, TablePackMap & seen, const Item & value ); +template inline bool ItemPack( const Ctx & ctx, TablePackMap & seen, const Item & src, Item & dst, uint8_t * base, int64_t capacity, int64_t & used ); template inline bool FleetByIdEntryNumber( const Ctx & ctx, TableNumbering & numbering, const FleetByIdEntry & value ); template inline int64_t FleetByIdEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const FleetByIdEntry & value ); template inline bool FleetByIdEntryPack( const Ctx & ctx, TablePackMap & seen, const FleetByIdEntry & src, FleetByIdEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); @@ -6518,6 +6564,10 @@ template inline int64_t TableNodeMeasure( const Ctx &, const Tabl template inline bool TableNodeSave( const Ctx &, const TableNumbering &, TableWriter & w, TableIds & ids, const ShipConfig & value ) { return ShipConfigSaveBody( w, ids, value ); } template inline int64_t TableNodeMessageMeasure( const Ctx &, const TableNumbering &, int64_t, int64_t at, const ShipConfig & value ) { return ShipConfigMeasureMessageBody( at, value ); } template inline bool TableNodeMessageSave( const Ctx &, const TableNumbering &, int64_t, TableBitWriter & w, const ShipConfig & value ) { return ShipConfigSaveMessageBody( w, value ); } +template inline int64_t TableNodeMeasure( const Ctx &, const TableNumbering &, TableIds & ids, const Item & value ) { return ItemMeasureBody( ids, value ); } +template inline bool TableNodeSave( const Ctx &, const TableNumbering &, TableWriter & w, TableIds & ids, const Item & value ) { return ItemSaveBody( w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx &, const TableNumbering &, int64_t, int64_t at, const Item & value ) { return ItemMeasureMessageBody( at, value ); } +template inline bool TableNodeMessageSave( const Ctx &, const TableNumbering &, int64_t, TableBitWriter & w, const Item & value ) { return ItemSaveMessageBody( w, value ); } template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const FleetByIdEntry & value ) { return FleetByIdEntryMeasureBody( ctx, numbering, ids, value ); } template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const FleetByIdEntry & value ) { return FleetByIdEntrySaveBody( ctx, numbering, w, ids, value ); } template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const FleetByIdEntry & value ) { return FleetByIdEntryMeasureMessageBody( ctx, numbering, index_bits, at, value ); } @@ -7180,14 +7230,14 @@ inline bool ShipConfigSaveMessageBody( TableBitWriter & w, const ShipConfig & va if ( value.name_length < 0 || value.name_length > 64 ) { return false; } // storage invariant if ( value.name_length > 0 ) { - w.put( 37, kTableMessageRefBitsHere ); + w.put( 41, kTableMessageRefBitsHere ); w.put( (uint64_t) value.name_length, 7 ); w.align(); // a string or a bytes ALIGNS before its bytes w.putbytes( (const uint8_t *) value.name, value.name_length ); } if ( value.health != 0 ) { - w.put( 38, kTableMessageRefBitsHere ); + w.put( 42, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.health ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body @@ -7543,7 +7593,7 @@ inline bool ItemSaveMessageBody( TableBitWriter & w, const Item & value ) { if ( value.count != 0 ) { - w.put( 33, kTableMessageRefBitsHere ); + w.put( 36, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.count ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body @@ -9880,7 +9930,7 @@ inline bool FleetSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_ships.ok ) { return false; } // the sort could not run if ( order_ships.count > 0 ) { - w.put( 28, kTableMessageRefBitsHere ); + w.put( 31, kTableMessageRefBitsHere ); w.put( (uint64_t) order_ships.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_ships.count; i++ ) { @@ -9894,7 +9944,7 @@ inline bool FleetSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_by_id.ok ) { return false; } // the sort could not run if ( order_by_id.count > 0 ) { - w.put( 29, kTableMessageRefBitsHere ); + w.put( 32, kTableMessageRefBitsHere ); w.put( (uint64_t) order_by_id.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_by_id.count; i++ ) { @@ -9909,7 +9959,7 @@ inline bool FleetSaveMessageBody( const Ctx & ctx, const TableNumbering & number { uint64_t index_flagship = 0; if ( !TableNumberingIndex( numbering, (const void *) pointee_flagship, index_flagship ) ) { return false; } - w.put( 30, kTableMessageRefBitsHere ); + w.put( 33, kTableMessageRefBitsHere ); w.put( index_flagship, index_bits ); } } @@ -9918,7 +9968,7 @@ inline bool FleetSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_loadouts.ok ) { return false; } // the sort could not run if ( order_loadouts.count > 0 ) { - w.put( 31, kTableMessageRefBitsHere ); + w.put( 34, kTableMessageRefBitsHere ); w.put( (uint64_t) order_loadouts.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_loadouts.count; i++ ) { @@ -9932,7 +9982,7 @@ inline bool FleetSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_tiers.ok ) { return false; } // the sort could not run if ( order_tiers.count > 0 ) { - w.put( 32, kTableMessageRefBitsHere ); + w.put( 35, kTableMessageRefBitsHere ); w.put( (uint64_t) order_tiers.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_tiers.count; i++ ) { @@ -10344,6 +10394,41 @@ inline bool ShipConfigExtentPack( const Ctx & ctx, const ShipConfig & src, ShipC return true; } +// ItemWireExtent: the extent Item's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool ItemWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + (void) body; (void) length; (void) at; (void) ids; (void) reason; // no list or map below this record + return true; +} + +// ItemExtentAt: the node extent Item's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as ItemExtentPack advances it (§2.8, §2.9). +template +inline bool ItemExtentAt( const Ctx & ctx, const Item & value, int64_t & at ) +{ + (void) ctx; (void) value; (void) at; // no list or map below this record + return true; +} + +template +inline int64_t ItemExtent( const Ctx & ctx, const Item & value ) +{ + (void) ctx; (void) value; // no list or map below this record + return 0; +} + +// ItemExtentPack: carve Item's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset ItemExtentAt advances (§2.8, §2.9). +template +inline bool ItemExtentPack( const Ctx & ctx, const Item & src, Item & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + (void) ctx; (void) src; (void) dst; (void) extent; (void) at; (void) capacity; // no list or map below this record + return true; +} + // FleetByIdEntryWireExtent: the extent FleetByIdEntry's lists and maps command, from the FRAMING alone. // It reads no field value, so a caller can refuse a number it did not // expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). @@ -11168,6 +11253,59 @@ inline bool ShipConfigPackEdges( const Ctx & ctx, TablePackMap & seen, const Shi return true; } +// ItemNumber: number everything Item POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool ItemNumber( const Ctx & ctx, TableNumbering & numbering, const Item & value ) +{ + (void) ctx; (void) numbering; (void) value; // no pointers below this node + return true; +} + +// ItemPackMeasure: the packed region bytes of everything Item POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t ItemPackMeasure( const Ctx & ctx, TablePackMap & seen, const Item & value ) +{ + int64_t bytes = 0; + (void) ctx; (void) seen; (void) value; // no pointers below this node + return bytes; +} + +// ItemPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool ItemPackEdges( const Ctx & ctx, TablePackMap & seen, const Item & src, Item & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool ItemPack( const Ctx & ctx, TablePackMap & seen, const Item & src, Item & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( Item ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( Item ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !ItemExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return ItemPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool ItemPackEdges( const Ctx & ctx, TablePackMap & seen, const Item & src, Item & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + (void) ctx; (void) seen; (void) src; (void) dst; (void) base; (void) capacity; (void) used; + return true; +} + // FleetByIdEntryNumber: number everything FleetByIdEntry POINTS AT, in first-visit order — // the fields in declaration order, a by-value edge descended in place. // A reference to an entry whose descent is still OPEN is a data cycle, @@ -11193,7 +11331,7 @@ inline bool FleetByIdEntryNumber( const Ctx & ctx, TableNumbering & numbering, c TableNodeEntry node; node.node = (const void *) pointee; node.type_id = 0x758252d2d1b14f0dull; // fnv1a64( "ShipConfig" ) - node.type_slot = 61; // its slot in the unit's vocabulary (§3.3) + node.type_slot = 69; // its slot in the unit's vocabulary (§3.3) node.measure = &TableNodeMeasureThunk; node.save = &TableNodeSaveThunk; node.message_measure = &TableNodeMessageMeasureThunk; @@ -11381,7 +11519,7 @@ inline bool FleetNumber( const Ctx & ctx, TableNumbering & numbering, const Flee TableNodeEntry node; node.node = (const void *) pointee; node.type_id = 0x758252d2d1b14f0dull; // fnv1a64( "ShipConfig" ) - node.type_slot = 61; // its slot in the unit's vocabulary (§3.3) + node.type_slot = 69; // its slot in the unit's vocabulary (§3.3) node.measure = &TableNodeMeasureThunk; node.save = &TableNodeSaveThunk; node.message_measure = &TableNodeMessageMeasureThunk; diff --git a/testdata/golden/tables/maps/PairsTable.cpp b/testdata/golden/tables/maps/PairsTable.cpp new file mode 100644 index 000000000..fc3e9f2db --- /dev/null +++ b/testdata/golden/tables/maps/PairsTable.cpp @@ -0,0 +1,3448 @@ +// Code generated by the schema compiler from Pairs.schema. DO NOT EDIT. +// SPDX-License-Identifier: NONE — this generated output is yours, under terms of +// your choice. See the LICENSE exception in the schema compiler; the compiler is +// AGPL-3.0, its output is not. +// package mapdemo — the TABLE wire's text form (docs/SPEC-TABLES.md §16). +// Compile this file to use FromJson / ToJson; a project that +// never reads or writes a text does not compile it and pays nothing. + +#include "PairsTable.h" + +#include // the text form: number formatting +#include // the text form: exact number conversion +#include // the text form: the runtime's decimal point + +// The guard is not vestigial. Several mapdemo Table.cpp files may be +// concatenated into ONE translation unit — a unity build — and without it +// each would redefine the walk. It is also why the walk's functions may be +// weak (vague linkage) across separate objects: ODR requires their +// definitions to be token-identical, and the generic-walk gate is what +// proves that, byte for byte, across every generated .cpp. +#ifndef MAPDEMO_SCHEMA_TABLE_JSON +#define MAPDEMO_SCHEMA_TABLE_JSON + +namespace mapdemo { + +// ---- the pointer adapters (docs/SPEC-TABLES.md §16.7) ---- +// +// The walk below is ONE walk, byte-identical in every generated .cpp, and a +// pointer is the one kind it cannot walk alone: reading one needs the +// builder's arena and writing one needs a region's deref, and neither exists +// in a unit that declares no pointer. So the walk calls these three and does +// not define them. A unit with no pointer defines them as stubs no field ever +// reaches; a pointered unit defines them in the graph half that follows the +// walk. + +struct TableJsonIn; +struct TableJsonOut; + +// a pointer field's object, or the `&node` reference standing in for it, into +// the slot; the cursor is on the opening brace +inline bool TableJsonReadPointer( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); +// the node a pointer slot names, in place — or as `&node` when it is shared +inline bool TableJsonWritePointer( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// the FIRST key of an object the walk is skipping begins with `&`: the cursor is +// on its value. A dropped definition still takes its label (§16.7); a fixed reader +// skips the value whole, as it skips everything else it does not place. +inline bool TableJsonSkippedAmpersand( TableJsonIn & in, const char * key, int32_t depth ); + +// ---- the map and list adapters (docs/SPEC-TABLES.md §2.8, §2.9, §16) ---- +// +// A MAP and an UNBOUNDED ARRAY are the other constructs the walk cannot walk +// alone: their arrays live behind a TableMap or a TableList this +// walk has no name for, reading one needs the builder's arena, and neither +// exists in a unit that declares neither construct. Same shape as the +// pointer's three: declared here, defined after the walk by whichever half +// the unit carries. Both are OUT-OF-LINE ARRAYS to the descriptors (§8.1): +// array_bound = 0 is the tell, and the type name says which of the two. + +// a map field: an out-of-line array whose type name spells the map +inline bool TableJsonIsMap( const TableFieldInfo * f ); +// the map as a plain JSON object keyed by the KEY, in ASCENDING key order +inline bool TableJsonWriteMap( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// that object back into the slot, in whatever order the text gives it +inline bool TableJsonReadMap( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); +// an unbounded array: the other out-of-line array +inline bool TableJsonIsList( const TableFieldInfo * f ); +// the list as a JSON array, in INDEX order +inline bool TableJsonWriteList( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// that array back into the slot, every element the text carries +inline bool TableJsonReadList( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); + +// ---- json walk: begin ---- +// +// The TEXT form (docs/SPEC-TABLES.md §16): one table, one text, one walk over the +// reflection descriptors (§8). Reading fills ONE caller-owned instance and +// allocates nothing beyond it; writing targets a caller buffer with the +// wire's measure/write symmetry. Everything AROUND this — which file goes +// with which instance, what key an instance is filed under, how instances +// link into a root table's collections — is a packer's opinion and stays +// with the tool that holds it. +// +// The dialect: trailing commas are accepted on read (the authoring files +// this exists for carry them) and never written; comments are not JSON and +// are refused; unknown keys are skipped and counted; a duplicate key is +// last-wins and counted; a key present with the wrong JSON type is skipped +// and counted, never coerced. + +static const int32_t kTableJsonMaxDepth = 128; + +// A key longer than this cannot name a field, so it is skipped as unknown. +static const int32_t kTableJsonMaxKey = 256; + +// The longest numeric token the walk will convert. Anything longer is a +// value no field can hold and counts as a kind mismatch. +static const int32_t kTableJsonMaxNumber = 512; + +// The decimal point the C runtime is CURRENTLY using. Number conversion is +// the one locale-sensitive corner of the grammar — JSON's point is always +// '.', the runtime's is whatever the program set — so every number crosses +// this one character on the way out and on the way back in. Nothing else in +// the walk consults the locale. +inline char TableJsonDecimalPoint() +{ + const struct lconv * conv = localeconv(); + if ( conv != NULL && conv->decimal_point != NULL && conv->decimal_point[0] != 0 ) + { + return conv->decimal_point[0]; + } + return '.'; +} + +// ---- storage access: the descriptors give an offset and a width, and the +// ---- storage is the HOST's, so every load and store goes through a width +// ---- switch rather than a memcpy into the low bytes of a wider word + +// finite: not a NaN, not an infinity. Written without — the walk's +// runtime surface stays the handful of functions it already names. +// A vocabulary entry the descriptor could not spell. The generated name +// functions answer "???" for a value outside the declared set, and that is +// not a name — writing it would put a spelling in the text that the reader +// then counts as unknown, turning a refusal into a silent loss. +inline bool TableJsonNamed( const char * name ) +{ + return name != NULL && strcmp( name, "???" ) != 0; +} + +inline bool TableJsonFinite( double v ) +{ + return v == v && v <= 1.7976931348623157e308 && v >= -1.7976931348623157e308; +} + +inline uint64_t TableJsonGetRaw( const void * storage, uint32_t width ) +{ + switch ( width ) + { + case 1: { uint8_t v = 0; memcpy( &v, storage, 1 ); return v; } + case 2: { uint16_t v = 0; memcpy( &v, storage, 2 ); return v; } + case 4: { uint32_t v = 0; memcpy( &v, storage, 4 ); return v; } + case 8: { uint64_t v = 0; memcpy( &v, storage, 8 ); return v; } + } + return 0; +} + +inline void TableJsonSetRaw( void * storage, uint32_t width, uint64_t value ) +{ + switch ( width ) + { + case 1: { uint8_t v = (uint8_t) value; memcpy( storage, &v, 1 ); break; } + case 2: { uint16_t v = (uint16_t) value; memcpy( storage, &v, 2 ); break; } + case 4: { uint32_t v = (uint32_t) value; memcpy( storage, &v, 4 ); break; } + case 8: { uint64_t v = value; memcpy( storage, &v, 8 ); break; } + } +} + +inline int64_t TableJsonGetSigned( const void * storage, uint32_t width ) +{ + uint64_t raw = TableJsonGetRaw( storage, width ); + if ( width < 8 ) + { + uint64_t sign = uint64_t( 1 ) << ( width * 8 - 1 ); + if ( ( raw & sign ) != 0 ) + { + raw |= ~( ( sign << 1 ) - 1 ); + } + } + return (int64_t) raw; +} + +// ---- the WIDE kinds (docs/SPEC-TABLES.md §3, §16.2) ---- +// +// The 128-bit integers and the fixed-point family convert EXACTLY, over two +// 64-bit lanes: a 128-bit integer is a decimal integer, a fixed value a +// decimal in WHOLE UNITS (1.0, -0.25, 3.0000152587890625) and nothing +// on either path passes through a double. Nothing here needs a 128-bit type +// either, which is what keeps this walk one text for every unit. +struct TableJsonWide +{ + uint64_t lo; + uint64_t hi; +}; + +inline bool TableJsonKindWide( uint8_t kind ) { return kind >= 18 && kind <= 29; } +inline bool TableJsonKindWideSigned( uint8_t kind ) { return kind == 18 || ( kind >= 20 && kind <= 24 ); } +inline bool TableJsonKindFixed( uint8_t kind ) { return kind >= 20 && kind <= 29; } + +inline bool TableJsonWideZero( TableJsonWide v ) { return v.lo == 0 && v.hi == 0; } +inline bool TableJsonWideNegative( TableJsonWide v ) { return ( v.hi >> 63 ) != 0; } + +inline int TableJsonWideCompare( TableJsonWide a, TableJsonWide b, bool is_signed ) +{ + if ( is_signed && TableJsonWideNegative( a ) != TableJsonWideNegative( b ) ) { return TableJsonWideNegative( a ) ? -1 : 1; } + if ( a.hi != b.hi ) { return a.hi < b.hi ? -1 : 1; } + if ( a.lo != b.lo ) { return a.lo < b.lo ? -1 : 1; } + return 0; +} + +inline TableJsonWide TableJsonWideShl( TableJsonWide v, int n ) +{ + TableJsonWide r = { 0, 0 }; + if ( n <= 0 ) { return v; } + if ( n >= 128 ) { return r; } + if ( n >= 64 ) { r.hi = v.lo << ( n - 64 ); return r; } + r.hi = ( v.hi << n ) | ( v.lo >> ( 64 - n ) ); + r.lo = v.lo << n; + return r; +} + +inline TableJsonWide TableJsonWideShr( TableJsonWide v, int n ) +{ + TableJsonWide r = { 0, 0 }; + if ( n <= 0 ) { return v; } + if ( n >= 128 ) { return r; } + if ( n >= 64 ) { r.lo = v.hi >> ( n - 64 ); return r; } + r.lo = ( v.lo >> n ) | ( v.hi << ( 64 - n ) ); + r.hi = v.hi >> n; + return r; +} + +inline TableJsonWide TableJsonWideNeg( TableJsonWide v ) +{ + TableJsonWide r; + r.lo = ~v.lo + 1; + r.hi = ~v.hi + ( r.lo == 0 ? 1 : 0 ); + return r; +} + +// v = v * m + a; the return is the carry out of 128 bits +inline uint32_t TableJsonWideMulAdd( TableJsonWide * v, uint32_t m, uint32_t a ) +{ + uint64_t limb[4] = { v->lo & 0xffffffffull, v->lo >> 32, v->hi & 0xffffffffull, v->hi >> 32 }; + uint64_t carry = a; + for ( int i = 0; i < 4; i++ ) + { + uint64_t p = limb[i] * m + carry; + limb[i] = p & 0xffffffffull; + carry = p >> 32; + } + v->lo = limb[0] | ( limb[1] << 32 ); + v->hi = limb[2] | ( limb[3] << 32 ); + return (uint32_t) carry; +} + +// v = v / d; the return is the remainder +inline uint32_t TableJsonWideDiv( TableJsonWide * v, uint32_t d ) +{ + uint64_t limb[4] = { v->lo & 0xffffffffull, v->lo >> 32, v->hi & 0xffffffffull, v->hi >> 32 }; + uint64_t rem = 0; + for ( int i = 3; i >= 0; i-- ) + { + uint64_t cur = ( rem << 32 ) | limb[i]; + limb[i] = cur / d; + rem = cur % d; + } + v->lo = limb[0] | ( limb[1] << 32 ); + v->hi = limb[2] | ( limb[3] << 32 ); + return (uint32_t) rem; +} + +// The storage of a wide kind, as lanes. A sixteen-byte storage is serialize's +// pair — native __int128 in the host's byte order, or the emulated struct with +// its low lane first — so the lanes are read in the host's order; a narrower +// storage is one lane, sign-extended for a signed kind. +inline TableJsonWide TableJsonWideLoad( const void * storage, uint32_t width, bool is_signed ) +{ + TableJsonWide v = { 0, 0 }; + if ( width == 16 ) + { + uint64_t half[2]; + memcpy( half, storage, 16 ); + uint16_t probe = 1; + bool little = *(const uint8_t *) &probe == 1; + v.lo = little ? half[0] : half[1]; + v.hi = little ? half[1] : half[0]; + return v; + } + v.lo = is_signed ? (uint64_t) TableJsonGetSigned( storage, width ) : TableJsonGetRaw( storage, width ); + v.hi = ( is_signed && ( v.lo >> 63 ) != 0 ) ? ~uint64_t( 0 ) : 0; + return v; +} + +inline void TableJsonWideStore( void * storage, uint32_t width, TableJsonWide v ) +{ + if ( width == 16 ) + { + uint16_t probe = 1; + bool little = *(const uint8_t *) &probe == 1; + uint64_t half[2]; + half[0] = little ? v.lo : v.hi; + half[1] = little ? v.hi : v.lo; + memcpy( storage, half, 16 ); + return; + } + TableJsonSetRaw( storage, width, v.lo ); +} + +// a counted field's companion: a string's length, a bytes' length, a counted +// array's count. Bounded by the declared extent on the way out, so a storage +// invariant a caller broke cannot walk off the end of the array. +inline int32_t TableJsonCount( const void * base, const TableFieldInfo * f ) +{ + if ( !f->counted ) + { + return f->array_bound; + } + int32_t count = 0; + memcpy( &count, (const uint8_t *) base + f->count_offset, sizeof( count ) ); + if ( count < 0 ) { count = 0; } + if ( count > f->array_bound ) { count = f->array_bound; } + return count; +} + +inline void TableJsonSetCount( void * base, const TableFieldInfo * f, int32_t count ) +{ + if ( f->counted ) + { + memcpy( (uint8_t *) base + f->count_offset, &count, sizeof( count ) ); + } +} + +// ---- what a field's kind expects to see in the text ---- +// +// One classifier, consulted by both directions, so a reader and a writer can +// never disagree about a kind's JSON form. 'o' object, 'a' array, 's' +// string, 'n' number, 'b' boolean. +// +// A vocabulary field is spelled by NAME: an enum is one name, a flags mask +// is the array of the names of its set bits. The two are told apart by the +// id column — an enum variant rides under a wire id, a flags BIT never does +// (docs/SPEC-TABLES.md §4), so a name function with no id function is flags. +// +// bytes(N) is the one kind whose element kind does not decide its form: it +// shares u8 with a plain array of u8, and rides as base64. The schema type +// name settles it, and "bytes" is a keyword no declaration can claim. +inline bool TableJsonIsBytes( const TableFieldInfo * f ) +{ + return f->is_array && f->kind == 6 && strcmp( f->type_name, "bytes" ) == 0; +} + +// An ENUM-KEYED array (docs/SPEC-TABLES.md §2.4): its JSON form is an OBJECT +// keyed by variant name, not a positional array, because that is what the +// storage is — one slot per variant, addressed by the variant. +inline bool TableJsonIsKeyed( const TableFieldInfo * f ) +{ + return f->key_name != NULL; +} + +// THE KEY A STORAGE SLOT HOLDS (§2.4, §8): the storage shifts left, so slot i +// holds the key i + 1 and nothing is stored for None. This is the ONE place +// the walker spells the shift. +inline uint64_t TableJsonKeyedSlotKey( int64_t slot ) +{ + return (uint64_t) ( slot + 1 ); +} + +// A slot whose key names a variant of the keying enum. Every slot in +// [0, array_bound) does, unless the enum carries max-headroom variants outside +// a table closure, where a reserved value names nothing and its key id is 0 — +// the reserved id no declared name can fold to (§5). +inline bool TableJsonKeyedSlotValid( const TableFieldInfo * f, int64_t slot ) +{ + return f->key_id( TableJsonKeyedSlotKey( slot ) ) != 0; +} + +inline bool TableJsonIsFlags( const TableFieldInfo * f ) +{ + return f->enum_name != NULL && f->variant_id == NULL; +} + +inline bool TableJsonIsEnum( const TableFieldInfo * f ) +{ + return f->variant_id != NULL && f->arms == NULL; +} + +inline char TableJsonShape( const TableFieldInfo * f ) +{ + if ( TableJsonIsMap( f ) ) return 'o'; // a MAP: an object keyed by the KEY (§2.8) + if ( f->kind == 12 ) return 's'; // string + if ( f->kind == 33 ) return 's'; // wstring: the same text, transcoded (§16.2) + if ( TableJsonIsBytes( f ) ) return 's'; // bytes: base64 + if ( TableJsonIsKeyed( f ) ) return 'o'; // an object keyed by variant NAME + if ( f->is_array ) return 'a'; + if ( f->arms != NULL ) return 'o'; // union: an object with ONE key + if ( f->kind == 13 ) return 'o'; // nested table or type + if ( f->kind == 17 ) return f->table != NULL ? 'o' : 's'; // a pointer: the pointee's object in place, or null (§16.7); a byte buffer's string (§2.5) + if ( TableJsonIsEnum( f ) ) return 's'; + if ( TableJsonIsFlags( f ) ) return 'a'; + if ( f->kind == 1 ) return 'b'; + return 'n'; +} + +// the ELEMENT shape of an array field — the same classifier one level down +inline char TableJsonElementShape( const TableFieldInfo * f ) +{ + if ( f->arms != NULL ) return 'o'; // an element of an array of unions: one key, the arm (§2.6) + if ( f->kind == 13 ) return 'o'; + if ( TableJsonIsEnum( f ) ) return 's'; + if ( TableJsonIsFlags( f ) ) return 'a'; + if ( f->kind == 1 ) return 'b'; + return 'n'; +} + +// A guarded group rides only when its guard reads true — the wire's own +// elision (§4), carried into the text so a text and a wire written from one +// instance say the same thing. The guard is spelled as its branch condition +// over bool fields of the SAME type ("at_rest", "!at_rest", +// "active && has_target"), so evaluating it is a walk of the same +// descriptor. Nothing is inferred in the other direction: reading places +// every key it can name, and the guard is a plain bool key (§16.2). +inline bool TableJsonGuardHolds( const void * base, const TableTypeInfo * info, const char * guard ) +{ + const char * p = guard; + for ( ;; ) + { + while ( *p == ' ' || *p == '&' ) { p++; } + if ( *p == 0 ) { return true; } + bool want = true; + if ( *p == '!' ) { want = false; p++; } + const char * start = p; + while ( *p != 0 && *p != ' ' && *p != '&' ) { p++; } + size_t length = (size_t) ( p - start ); + bool value = false; + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + const TableFieldInfo * f = &info->fields[i]; + if ( strlen( f->name ) == length && strncmp( f->name, start, length ) == 0 ) + { + value = TableJsonGetRaw( (const uint8_t *) base + f->offset, f->elem_size ) != 0; + break; + } + } + if ( value != want ) { return false; } + } +} + +// ---- writing ---- + +// The writer sink MEASURES when the buffer is NULL and WRITES when it is +// not, over one code path — so measure and write agree byte for byte, the +// wire's invariant (§9) carried across. +struct TableJsonOut +{ + char * buffer; + int64_t capacity; + int64_t offset; + bool overflow; + void * graph; // the pointered write's identity map (§16.7); NULL for a fixed table + + void raw( const char * data, int64_t count ) + { + if ( buffer != NULL ) + { + if ( offset + count > capacity ) { overflow = true; return; } + memcpy( buffer + offset, data, (size_t) count ); + } + offset += count; + } + void put( char c ) { raw( &c, 1 ); } + void text( const char * s ) { raw( s, (int64_t) strlen( s ) ); } + void line( int32_t depth ) + { + put( '\n' ); + for ( int32_t i = 0; i < depth; i++ ) { raw( " ", 2 ); } + } +}; + +inline const char * TableJsonBase64Alphabet() +{ + return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +} + +inline void TableJsonWriteBase64( TableJsonOut & out, const uint8_t * data, int32_t length ) +{ + const char * alphabet = TableJsonBase64Alphabet(); + out.put( '"' ); + int32_t i = 0; + for ( ; i + 3 <= length; i += 3 ) + { + uint32_t triple = ( uint32_t( data[i] ) << 16 ) | ( uint32_t( data[i+1] ) << 8 ) | uint32_t( data[i+2] ); + char quad[4] = { alphabet[ ( triple >> 18 ) & 0x3f ], alphabet[ ( triple >> 12 ) & 0x3f ], + alphabet[ ( triple >> 6 ) & 0x3f ], alphabet[ triple & 0x3f ] }; + out.raw( quad, 4 ); + } + if ( i < length ) + { + int32_t left = length - i; + uint32_t triple = uint32_t( data[i] ) << 16; + if ( left == 2 ) { triple |= uint32_t( data[i+1] ) << 8; } + char quad[4] = { alphabet[ ( triple >> 18 ) & 0x3f ], alphabet[ ( triple >> 12 ) & 0x3f ], '=', '=' }; + if ( left == 2 ) { quad[2] = alphabet[ ( triple >> 6 ) & 0x3f ]; } + out.raw( quad, 4 ); + } + out.put( '"' ); +} + +// One UTF-8 sequence at s, or -1 when the bytes there are not one. Rejects +// the lot: a stray continuation, an overlong form, a surrogate half, and +// anything past U+10FFFF. +inline int32_t TableJsonUtf8( const char * s, int32_t remaining, int32_t * width ) +{ + unsigned char lead = (unsigned char) s[0]; + int32_t want = 0; + int32_t code = 0; + if ( lead < 0x80 ) { *width = 1; return lead; } + else if ( lead >= 0xc2 && lead <= 0xdf ) { want = 2; code = lead & 0x1f; } + else if ( lead >= 0xe0 && lead <= 0xef ) { want = 3; code = lead & 0x0f; } + else if ( lead >= 0xf0 && lead <= 0xf4 ) { want = 4; code = lead & 0x07; } + else { return -1; } + if ( remaining < want ) { return -1; } + for ( int32_t i = 1; i < want; i++ ) + { + unsigned char next = (unsigned char) s[i]; + if ( ( next & 0xc0 ) != 0x80 ) { return -1; } + code = ( code << 6 ) | ( next & 0x3f ); + } + if ( want == 3 && code < 0x800 ) { return -1; } // overlong + if ( want == 4 && code < 0x10000 ) { return -1; } // overlong + if ( code >= 0xd800 && code <= 0xdfff ) { return -1; } // a surrogate half + if ( code > 0x10ffff ) { return -1; } + *width = want; + return code; +} + +// The inverse: one code point encoded as UTF-8 into unit, its length +// answered. Both text kinds' writers reach it, and so does the escape +// grammar's U+FFFD replacement. +inline int32_t TableJsonEncodeUtf8( uint32_t code, char * unit ) +{ + if ( code < 0x80 ) { unit[0] = (char) code; return 1; } + if ( code < 0x800 ) + { + unit[0] = (char) ( 0xc0 | ( code >> 6 ) ); + unit[1] = (char) ( 0x80 | ( code & 0x3f ) ); + return 2; + } + if ( code < 0x10000 ) + { + unit[0] = (char) ( 0xe0 | ( code >> 12 ) ); + unit[1] = (char) ( 0x80 | ( ( code >> 6 ) & 0x3f ) ); + unit[2] = (char) ( 0x80 | ( code & 0x3f ) ); + return 3; + } + unit[0] = (char) ( 0xf0 | ( code >> 18 ) ); + unit[1] = (char) ( 0x80 | ( ( code >> 12 ) & 0x3f ) ); + unit[2] = (char) ( 0x80 | ( ( code >> 6 ) & 0x3f ) ); + unit[3] = (char) ( 0x80 | ( code & 0x3f ) ); + return 4; +} + +// A JSON text MUST be valid UTF-8 (RFC 8259 §8.1). The read path is +// byte-transparent — the wire imposes no encoding (§3) and a string may hold +// anything — so the WRITER is where that obligation is met: a byte that is +// not part of a well-formed sequence is written as U+FFFD, one per bad byte, +// and never raw. A text this walk writes is therefore readable by any +// conforming parser, which a raw byte would not be. The cost is stated +// plainly: for a string holding invalid UTF-8, the round trip is NOT +// byte-identical, because the alternative is emitting a text that is not +// JSON. +inline void TableJsonWriteString( TableJsonOut & out, const char * s, int32_t length ) +{ + static const char hex[] = "0123456789abcdef"; + out.put( '"' ); + for ( int32_t i = 0; i < length; i++ ) + { + unsigned char c = (unsigned char) s[i]; + switch ( c ) + { + case '"': out.raw( "\\\"", 2 ); break; + case '\\': out.raw( "\\\\", 2 ); break; + case '\b': out.raw( "\\b", 2 ); break; + case '\f': out.raw( "\\f", 2 ); break; + case '\n': out.raw( "\\n", 2 ); break; + case '\r': out.raw( "\\r", 2 ); break; + case '\t': out.raw( "\\t", 2 ); break; + default: + if ( c < 0x20 ) + { + char escape[6] = { '\\', 'u', '0', '0', hex[ c >> 4 ], hex[ c & 0xf ] }; + out.raw( escape, 6 ); + } + else if ( c < 0x80 ) + { + out.put( (char) c ); + } + else + { + int32_t width = 0; + if ( TableJsonUtf8( s + i, length - i, &width ) < 0 ) + { + out.raw( "\xef\xbf\xbd", 3 ); // U+FFFD, one per bad byte + } + else + { + out.raw( s + i, width ); + i += width - 1; + } + } + break; + } + } + out.put( '"' ); +} + +// A WIDE field's text: the code units transcoded back to UTF-8 (§16.2). A +// SURROGATE PAIR is one code point; an UNPAIRED SURROGATE is not a code point +// at all, encodes to nothing, and writes one U+FFFD per ill-formed unit; a +// ZERO UNIT is U+0000, which JSON has an escape for, and writes \u0000 +// (§16.3). No wire can put either into storage (§3), so both answer for +// storage a PROGRAM built. +inline void TableJsonWriteWString( TableJsonOut & out, const char16_t * s, int32_t length ) +{ + static const char hex[] = "0123456789abcdef"; + out.put( '"' ); + for ( int32_t i = 0; i < length; i++ ) + { + uint32_t code = (uint32_t) (uint16_t) s[i]; + if ( code >= 0xd800 && code <= 0xdbff && i + 1 < length ) + { + const uint32_t low = (uint32_t) (uint16_t) s[i + 1]; + if ( low >= 0xdc00 && low <= 0xdfff ) + { + code = 0x10000 + ( ( code - 0xd800 ) << 10 ) + ( low - 0xdc00 ); + i++; + } + } + if ( code >= 0xd800 && code <= 0xdfff ) { code = 0xfffd; } // an unpaired surrogate + switch ( code ) + { + case '"': out.raw( "\\\"", 2 ); continue; + case '\\': out.raw( "\\\\", 2 ); continue; + case '\b': out.raw( "\\b", 2 ); continue; + case '\f': out.raw( "\\f", 2 ); continue; + case '\n': out.raw( "\\n", 2 ); continue; + case '\r': out.raw( "\\r", 2 ); continue; + case '\t': out.raw( "\\t", 2 ); continue; + default: break; + } + if ( code < 0x20 ) + { + char escape[6] = { '\\', 'u', '0', '0', hex[ code >> 4 ], hex[ code & 0xf ] }; + out.raw( escape, 6 ); + continue; + } + char encoded[4]; + const int32_t encoded_length = TableJsonEncodeUtf8( code, encoded ); + out.raw( encoded, encoded_length ); + } + out.put( '"' ); +} + +inline void TableJsonWriteUnsigned( TableJsonOut & out, uint64_t value ) +{ + char digits[24]; + int32_t n = 0; + do + { + digits[n++] = (char) ( '0' + (int) ( value % 10 ) ); + value /= 10; + } while ( value != 0 ); + char text[24]; + for ( int32_t i = 0; i < n; i++ ) { text[i] = digits[n - 1 - i]; } + out.raw( text, n ); +} + +inline void TableJsonWriteSigned( TableJsonOut & out, int64_t value ) +{ + if ( value < 0 ) + { + out.put( '-' ); + TableJsonWriteUnsigned( out, uint64_t( 0 ) - (uint64_t) value ); + return; + } + TableJsonWriteUnsigned( out, (uint64_t) value ); +} + +// A wide kind writes its raw storage as §16.2's text: a 128-bit integer as a +// decimal integer; a fixed value in WHOLE UNITS as the shortest exact decimal +// with at least one fractional digit (1.0, -0.25), the spelling the schema text +// gives a fixed default. The fraction terminates because a dyadic fraction has +// a finite decimal expansion — at most F digits. +inline void TableJsonWriteWide( TableJsonOut & out, const void * storage, const TableFieldInfo * f ) +{ + bool is_signed = TableJsonKindWideSigned( f->kind ); + TableJsonWide v = TableJsonWideLoad( storage, f->elem_size, is_signed ); + if ( is_signed && TableJsonWideNegative( v ) ) + { + out.put( '-' ); + v = TableJsonWideNeg( v ); + } + int frac = f->frac_bits; + TableJsonWide whole = TableJsonWideShr( v, frac ); + char digits[40]; + int32_t n = 0; + do + { + digits[n++] = (char) ( '0' + (int) TableJsonWideDiv( &whole, 10 ) ); + } while ( !TableJsonWideZero( whole ) ); + char text[40]; + for ( int32_t i = 0; i < n; i++ ) { text[i] = digits[n - 1 - i]; } + out.raw( text, n ); + if ( !TableJsonKindFixed( f->kind ) ) { return; } + out.put( '.' ); + // the fraction bits alone: v with everything at and above bit F cleared + TableJsonWide fraction = v; + if ( frac < 64 ) { fraction.hi = 0; fraction.lo &= ( uint64_t( 1 ) << frac ) - 1; } + else { fraction.hi &= ( uint64_t( 1 ) << ( frac - 64 ) ) - 1; } + if ( frac == 0 ) { fraction.lo = 0; } + if ( TableJsonWideZero( fraction ) ) + { + out.put( '0' ); + return; + } + while ( !TableJsonWideZero( fraction ) ) + { + // ×10: the digit is what lands at and above bit F, including the + // carry out of 128 bits when F leaves no room for it below + uint32_t carry = TableJsonWideMulAdd( &fraction, 10, 0 ); + uint64_t digit = TableJsonWideShr( fraction, frac ).lo; + if ( frac > 64 ) { digit |= uint64_t( carry ) << ( 128 - frac ); } + out.put( (char) ( '0' + (int) digit ) ); + if ( frac < 64 ) { fraction.hi = 0; fraction.lo &= ( uint64_t( 1 ) << frac ) - 1; } + else { fraction.hi &= ( uint64_t( 1 ) << ( frac - 64 ) ) - 1; } + } +} + +// A float writes at the SHORTEST precision that reads back as the same value +// at the field's own width, so a round trip is exact and a text stays +// readable. Non-finite values have no JSON spelling at all, and the writer +// REFUSES rather than losing one silently — the same rule measure and save +// already apply to an enum value no variant names (§5). +inline bool TableJsonWriteFloat( TableJsonOut & out, double value, bool single ) +{ + if ( !TableJsonFinite( value ) ) { return false; } + char text[64]; + int low = single ? 6 : 15; + int high = single ? 9 : 17; + int length = 0; + for ( int digits = low; ; digits++ ) + { + length = snprintf( text, sizeof( text ), "%.*g", digits, value ); + if ( length <= 0 || length >= (int) sizeof( text ) ) { return false; } + if ( digits >= high ) { break; } + // the round-trip check runs BEFORE the decimal point is normalised: + // the token still carries whatever point snprintf just produced + if ( single ) + { + if ( (double) strtof( text, NULL ) == value ) { break; } + } + else + { + if ( strtod( text, NULL ) == value ) { break; } + } + } + char point = TableJsonDecimalPoint(); + if ( point != '.' ) + { + for ( int i = 0; i < length; i++ ) + { + if ( text[i] == point ) { text[i] = '.'; } + } + } + out.raw( text, length ); + return true; +} + +inline bool TableJsonWriteValue( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth ); +// a UNION ARM that names no declaration writes through the field walk one key +// down (docs/SPEC-TABLES.md §2.6, §16.2), which is defined below +inline bool TableJsonWriteField( TableJsonOut & out, const void * base, const TableFieldInfo * f, int32_t depth ); + +// one scalar, at one storage address: a nested object, a union, a +// vocabulary, or a number +inline bool TableJsonWriteScalar( TableJsonOut & out, const void * storage, const TableFieldInfo * f, int32_t depth ) +{ + if ( f->arms != NULL ) + { + // a union is an object with ONE key, the arm's name; None is {} + const TableUnionInfo * arms = f->arms(); + uint64_t tag = TableJsonGetRaw( (const uint8_t *) storage + arms->tag_offset, arms->tag_size ); + if ( tag == 0 ) + { + out.raw( "{}", 2 ); + return true; + } + if ( (int64_t) tag > f->enum_max ) + { + return false; // a tag no arm names, exactly as measure refuses it + } + const char * arm = f->enum_name( tag ); + // and refuse on the NAME, not merely on the bound: §16.2 says a value + // no variant NAMES is refused, so the check is the name. Writing + // whatever came back would emit "???", a spelling the reader counts + // as unknown — a silent round-trip loss in place of a refusal. + if ( !TableJsonNamed( arm ) ) { return false; } + out.put( '{' ); + out.line( depth + 1 ); + TableJsonWriteString( out, arm, (int32_t) strlen( arm ) ); + out.raw( ": ", 2 ); + // THE ARM'S VALUE TAKES THE ARM'S OWN ROW (§16.2): an arm that names + // no declaration carries the FIELD descriptor a field of its type + // would carry, offsets taken inside the union storage (§2.6), so the + // value walks through the field writer one key down. + if ( arms->arms[tag].field != NULL ) + { + if ( !TableJsonWriteField( out, storage, arms->arms[tag].field, depth + 1 ) ) + { + return false; + } + } + else if ( arms->arms[tag].table == NULL ) + { + out.raw( "null", 4 ); // a payload-free arm: the name selects it (§2.6) + } + else if ( !TableJsonWriteValue( out, (const uint8_t *) storage + arms->arms[tag].offset, arms->arms[tag].table, depth + 1 ) ) + { + return false; + } + out.line( depth ); + out.put( '}' ); + return true; + } + if ( f->kind == 13 ) + { + return TableJsonWriteValue( out, storage, f->table, depth ); + } + if ( TableJsonIsEnum( f ) ) + { + uint64_t value = TableJsonGetRaw( storage, f->elem_size ); + // a value no variant names has no text spelling, exactly as it has no + // wire identity: the writer REFUSES rather than writing None over it, + // the rule measure and save already apply (docs/SPEC-TABLES.md §5) + if ( (int64_t) value > f->enum_max ) { return false; } + if ( value != 0 && f->variant_id( value ) == 0 ) { return false; } + const char * name = f->enum_name( value ); + if ( !TableJsonNamed( name ) ) { return false; } + TableJsonWriteString( out, name, (int32_t) strlen( name ) ); + return true; + } + if ( TableJsonIsFlags( f ) ) + { + uint64_t bits = TableJsonGetRaw( storage, f->elem_size ); + if ( bits == 0 ) + { + out.raw( "[]", 2 ); + return true; + } + out.put( '[' ); + bool first = true; + for ( int64_t bit = 0; bit < 64; bit++ ) + { + if ( ( bits & ( uint64_t( 1 ) << bit ) ) == 0 ) { continue; } + if ( bit > f->enum_max ) + { + return false; // a bit no variant names has no text spelling + } + const char * name = f->enum_name( (uint64_t) bit ); + if ( !TableJsonNamed( name ) ) { return false; } + if ( !first ) { out.put( ',' ); } + first = false; + out.line( depth + 1 ); + TableJsonWriteString( out, name, (int32_t) strlen( name ) ); + } + out.line( depth ); + out.put( ']' ); + return true; + } + switch ( f->kind ) + { + case 1: + out.text( TableJsonGetRaw( storage, f->elem_size ) != 0 ? "true" : "false" ); + return true; + case 10: + { + float v = 0.0f; + memcpy( &v, storage, sizeof( v ) ); + return TableJsonWriteFloat( out, (double) v, true ); + } + case 11: + { + double v = 0.0; + memcpy( &v, storage, sizeof( v ) ); + return TableJsonWriteFloat( out, v, false ); + } + case 2: case 3: case 4: case 5: + TableJsonWriteSigned( out, TableJsonGetSigned( storage, f->elem_size ) ); + return true; + default: + if ( TableJsonKindWide( f->kind ) ) + { + TableJsonWriteWide( out, storage, f ); + return true; + } + TableJsonWriteUnsigned( out, TableJsonGetRaw( storage, f->elem_size ) ); + return true; + } +} + +inline bool TableJsonWriteField( TableJsonOut & out, const void * base, const TableFieldInfo * f, int32_t depth ) +{ + const uint8_t * storage = (const uint8_t *) base + f->offset; + if ( TableJsonIsMap( f ) ) + { + return TableJsonWriteMap( out, (const void *) storage, f, depth ); + } + if ( TableJsonIsList( f ) ) + { + return TableJsonWriteList( out, (const void *) storage, f, depth ); + } + if ( f->kind == 17 && !f->is_array ) + { + return TableJsonWritePointer( out, storage, f, depth ); + } + if ( f->kind == 17 ) + { + // an ARRAY OF POINTERS (§2.1): the pointer row per element — the + // pointee's object in place, null, or `&node` for a shared one (§16.7) + int32_t count = TableJsonCount( base, f ); + if ( count == 0 ) { out.raw( "[]", 2 ); return true; } + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + if ( !TableJsonWritePointer( out, storage + (int64_t) i * f->elem_size, f, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( ']' ); + return true; + } + if ( f->kind == 12 ) + { + TableJsonWriteString( out, (const char *) storage, TableJsonCount( base, f ) ); + return true; + } + if ( f->kind == 33 ) + { + TableJsonWriteWString( out, (const char16_t *) (const void *) storage, TableJsonCount( base, f ) ); + return true; + } + if ( TableJsonIsBytes( f ) ) + { + TableJsonWriteBase64( out, storage, TableJsonCount( base, f ) ); + return true; + } + if ( TableJsonIsKeyed( f ) ) + { + // one entry per SLOT, keyed by the variant that owns it, so inserting + // a variant next season moves nothing in the text either. Slot i holds + // the key i + 1: nothing is stored for None, so nothing is written for it. + out.put( '{' ); + bool first = true; + for ( int64_t slot = 0; slot < f->array_bound; slot++ ) + { + if ( !TableJsonKeyedSlotValid( f, slot ) ) { continue; } + if ( !first ) { out.put( ',' ); } + first = false; + out.line( depth + 1 ); + const char * key = f->key_name( TableJsonKeyedSlotKey( slot ) ); + TableJsonWriteString( out, key, (int32_t) strlen( key ) ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteScalar( out, storage + slot * f->elem_size, f, depth + 1 ) ) + { + return false; + } + } + if ( first ) { out.raw( "}", 1 ); return true; } + out.line( depth ); + out.put( '}' ); + return true; + } + if ( f->is_array ) + { + int32_t count = TableJsonCount( base, f ); + if ( count == 0 ) + { + out.raw( "[]", 2 ); + return true; + } + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + if ( !TableJsonWriteScalar( out, storage + (int64_t) i * f->elem_size, f, depth + 1 ) ) + { + return false; + } + } + out.line( depth ); + out.put( ']' ); + return true; + } + return TableJsonWriteScalar( out, storage, f, depth ); +} + +// One instance's fields, in DECLARATION ORDER, defaults included — a text is +// for people and tools, and a text that elides is a text a reader has to know +// the schema to complete. `any` says whether the object is already open on +// entry — a shared node's `&node` opens it before the fields (§16.7) — and +// whether it is open on return. +inline bool TableJsonWriteFields( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth, bool & any ) +{ + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + const TableFieldInfo * f = &info->fields[i]; + if ( f->guard[0] != 0 && !TableJsonGuardHolds( base, info, f->guard ) ) { continue; } + // an ABSENT optional writes no key: presence of the key IS the + // presence (§16.2), so an absent field is an absent key and nothing + // else would read back as absent + if ( f->optional && + TableJsonGetRaw( (const uint8_t *) base + f->present_offset, 1 ) == 0 ) + { + continue; + } + if ( !any ) { out.put( '{' ); } + else { out.put( ',' ); } + any = true; + out.line( depth + 1 ); + TableJsonWriteString( out, f->json, (int32_t) strlen( f->json ) ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteField( out, base, f, depth + 1 ) ) { return false; } + } + return true; +} + +// One instance as one object. The writer carries the reader's depth cap +// (§16.2): a pointer chain nests as deep as it is long (§16.7), and a text the +// writer produced past the cap would be a text the reader refuses. +inline bool TableJsonWriteValue( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { return false; } + bool any = false; + if ( !TableJsonWriteFields( out, base, info, depth, any ) ) { return false; } + if ( !any ) + { + out.raw( "{}", 2 ); + return true; + } + out.line( depth ); + out.put( '}' ); + return true; +} + +// ---- reading ---- + +struct TableJsonIn +{ + const char * text; + int64_t size; + int64_t pos; + TableReport * report; + bool bad; // the text is not JSON: the walk stops and keeps what it placed + void * graph; // the pointered read's builder and label map (§16.7); NULL for a fixed table +}; + +inline void TableJsonSpace( TableJsonIn & in ) +{ + while ( in.pos < in.size ) + { + char c = in.text[in.pos]; + if ( c == ' ' || c == '\t' || c == '\n' || c == '\r' ) { in.pos++; continue; } + // COMMENTS ARE ACCEPTED ON READ AND NEVER WRITTEN (docs/SPEC-TABLES.md + // §16.2): a line comment runs to the end of the line or of the input, + // a block comment to its closing delimiter, which does not nest, and + // an UNCLOSED block comment is malformed on the terms an unclosed + // string is. Both are legal wherever whitespace is; a lone slash is not JSON. + if ( c == '/' && in.pos + 1 < in.size && in.text[in.pos + 1] == '/' ) + { + in.pos += 2; + while ( in.pos < in.size && in.text[in.pos] != '\n' ) { in.pos++; } + continue; + } + if ( c == '/' && in.pos + 1 < in.size && in.text[in.pos + 1] == '*' ) + { + int64_t at = in.pos + 2; + while ( at + 1 < in.size && !( in.text[at] == '*' && in.text[at + 1] == '/' ) ) { at++; } + if ( at + 1 >= in.size ) { in.bad = true; in.pos = in.size; return; } + in.pos = at + 2; + continue; + } + if ( c == '/' ) { in.bad = true; } + return; + } +} + +inline char TableJsonPeek( TableJsonIn & in ) +{ + TableJsonSpace( in ); + return in.pos < in.size ? in.text[in.pos] : 0; +} + +// the shape of the value sitting at the cursor, without consuming it +inline char TableJsonValueShape( TableJsonIn & in ) +{ + char c = TableJsonPeek( in ); + switch ( c ) + { + case '{': return 'o'; + case '[': return 'a'; + case '"': return 's'; + case 't': case 'f': return 'b'; + case 'n': return 'z'; + case 0: return 0; + default: return 'n'; + } +} + +inline bool TableJsonLiteral( TableJsonIn & in, const char * word ) +{ + int64_t length = (int64_t) strlen( word ); + if ( in.pos + length > in.size || memcmp( in.text + in.pos, word, (size_t) length ) != 0 ) + { + in.bad = true; + return false; + } + in.pos += length; + return true; +} + +// one \uXXXX escape body; -1 when the four hex digits are not there +inline int TableJsonHex4( TableJsonIn & in ) +{ + if ( in.pos + 4 > in.size ) { return -1; } + int value = 0; + for ( int i = 0; i < 4; i++ ) + { + char c = in.text[in.pos + i]; + int digit; + if ( c >= '0' && c <= '9' ) { digit = c - '0'; } + else if ( c >= 'a' && c <= 'f' ) { digit = c - 'a' + 10; } + else if ( c >= 'A' && c <= 'F' ) { digit = c - 'A' + 10; } + else { return -1; } + value = ( value << 4 ) | digit; + } + in.pos += 4; + return value; +} + + +// One STRING BODY CHARACTER at the cursor, encoded into unit as UTF-8 and +// its length answered: an escape's code point, or a UTF-8 sequence read +// whole. It is ONE grammar serving both text kinds — the narrow scan places +// these bytes and the wide scan converts them back to code units — so the +// escape table, the lone-surrogate rule and the U+FFFD replacement are stated +// once. false means the text is not JSON and in.bad says so; a returned +// length of 0 means the closing quote was consumed and the string is done. +inline bool TableJsonScanUnit( TableJsonIn & in, char * unit, int32_t * unit_length_out ) +{ + int32_t unit_length = 0; + *unit_length_out = 0; + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos]; + if ( c == '"' ) { in.pos++; return true; } + if ( c == '\\' ) + { + in.pos++; + if ( in.pos >= in.size ) { in.bad = true; return false; } + char escape = in.text[in.pos++]; + switch ( escape ) + { + case '"': unit[0] = '"'; unit_length = 1; break; + case '\\': unit[0] = '\\'; unit_length = 1; break; + case '/': unit[0] = '/'; unit_length = 1; break; + case 'b': unit[0] = '\b'; unit_length = 1; break; + case 'f': unit[0] = '\f'; unit_length = 1; break; + case 'n': unit[0] = '\n'; unit_length = 1; break; + case 'r': unit[0] = '\r'; unit_length = 1; break; + case 't': unit[0] = '\t'; unit_length = 1; break; + case 'u': + { + int high = TableJsonHex4( in ); + if ( high < 0 ) { in.bad = true; return false; } + uint32_t code = (uint32_t) high; + if ( high >= 0xd800 && high <= 0xdbff && in.pos + 2 <= in.size && + in.text[in.pos] == '\\' && in.text[in.pos + 1] == 'u' ) + { + int64_t mark = in.pos; + in.pos += 2; + int low = TableJsonHex4( in ); + if ( low >= 0xdc00 && low <= 0xdfff ) + { + code = 0x10000 + ( ( (uint32_t) high - 0xd800 ) << 10 ) + ( (uint32_t) low - 0xdc00 ); + } + else + { + in.pos = mark; // a lone lead surrogate rides as itself + } + } + // a surrogate half that never found its partner has no + // UTF-8 encoding: encoding it anyway would manufacture + // CESU-8 — invalid UTF-8 — out of input that was valid + // JSON, so it reads as the replacement character + if ( code >= 0xd800 && code <= 0xdfff ) { code = 0xfffd; } + unit_length = TableJsonEncodeUtf8( code, unit ); + break; + } + default: in.bad = true; return false; + } + } + else if ( (unsigned char) c < 0x20 ) + { + in.bad = true; // a raw control character is not a JSON string body + return false; + } + else + { + // a UTF-8 sequence read WHOLE, so the clamp below can only land + // between code points. Only bytes that ACTUALLY look like + // continuations are taken: the wire imposes no encoding (§3), so + // a string may legitimately hold a stray lead byte, and one at + // the end of a text must not swallow the closing quote. + unsigned char lead = (unsigned char) c; + int32_t want = 1; + if ( ( lead & 0xe0 ) == 0xc0 ) { want = 2; } + else if ( ( lead & 0xf0 ) == 0xe0 ) { want = 3; } + else if ( ( lead & 0xf8 ) == 0xf0 ) { want = 4; } + unit[0] = c; + in.pos++; + unit_length = 1; + while ( unit_length < want && in.pos < in.size && + ( (unsigned char) in.text[in.pos] & 0xc0 ) == 0x80 ) + { + unit[unit_length++] = in.text[in.pos++]; + } + // A SEQUENCE THAT IS NOT A CODE POINT READS AS U+FFFD, which is + // §16.3's rule at the point the defect ENTERS rather than at the + // point it leaves: a lone surrogate escape already reads that way + // above, RFC 8259 requires a JSON text to be valid UTF-8, and a + // kind 12 payload is well-formed UTF-8 (§3), so storage the text + // form built has to be storage the wire can carry (§5). + if ( !TableUtf8Valid( (const uint8_t *) unit, unit_length ) ) + { + unit_length = TableJsonEncodeUtf8( 0xfffd, unit ); + } + } + } + *unit_length_out = unit_length; + return true; +} + +// Scan one JSON string into a caller buffer. Bytes are appended ONE CODE +// POINT AT A TIME — an escape's encoding, or a UTF-8 sequence read whole — +// so a string longer than the field is clamped AT A CODE POINT BOUNDARY and +// never cut through a multi-byte character. Clamping is counted, never +// fatal, exactly as it is on the wire (§4). A NULL destination scans past a +// string without keeping it. +// +// A CALLER THAT TAKES clamped_out OWNS THE COUNTER. The value paths leave it +// NULL, and the clamp is a value's clamp, counted here. A MAP KEY takes it, +// because a key never clamps: a key this buffer could not hold whole is not a +// shorter key, and its entry drops instead (§2.8). +inline bool TableJsonScanString( TableJsonIn & in, char * out, int32_t capacity, int32_t * length, + bool * clamped_out = NULL ) +{ + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + int32_t placed = 0; + bool clamped = false; + for ( ;; ) + { + char unit[4]; + int32_t unit_length = 0; + if ( !TableJsonScanUnit( in, unit, &unit_length ) ) { return false; } + if ( unit_length == 0 ) { break; } + if ( out == NULL ) + { + placed += unit_length; // measured and not kept: a byte buffer's read sizes its node this way (§2.5) + } + else if ( !clamped && placed + unit_length <= capacity ) + { + memcpy( out + placed, unit, (size_t) unit_length ); + placed += unit_length; + } + else + { + // A CLAMP IS A PREFIX. Once one code point does not fit, the scan + // stops placing: a later SHORTER code point sliding into the room + // the long one left would store a string the input never spelled, + // and one clamped count cannot tell the two apart. + clamped = true; + } + } + if ( clamped_out != NULL ) { *clamped_out = clamped; } + else if ( clamped ) { in.report->clamped++; } + if ( length != NULL ) { *length = placed; } + return true; +} + +// One UTF-8 sequence back to its CODE POINT, over bytes TableJsonScanUnit +// produced and therefore already well formed. It is the inverse of +// TableJsonEncodeUtf8 and nothing more. +inline uint32_t TableJsonDecodeUtf8( const char * unit, int32_t unit_length ) +{ + const unsigned char lead = (unsigned char) unit[0]; + if ( unit_length == 1 ) { return lead; } + uint32_t code = lead & ( unit_length == 2 ? 0x1fu : ( unit_length == 3 ? 0x0fu : 0x07u ) ); + for ( int32_t i = 1; i < unit_length; i++ ) + { + code = ( code << 6 ) | (uint32_t) ( (unsigned char) unit[i] & 0x3f ); + } + return code; +} + +// Scan one JSON string into a caller buffer of UTF-16 CODE UNITS: the wstring +// row of §16.2, the text TRANSCODED at the boundary. It shares +// TableJsonScanUnit with the narrow scan, so the escape grammar and the +// lone-surrogate rule are one grammar, and appends ONE CODE POINT AT A TIME — +// one unit below U+10000 and a surrogate PAIR above it. A string longer than +// the field is therefore clamped at N code units with a pair never split, and +// a high surrogate left without its low half is dropped with it, which is the +// same sentence the wire's clamp takes (§3). Clamping is counted, never fatal. +inline bool TableJsonScanWString( TableJsonIn & in, char16_t * out, int32_t capacity, int32_t * length ) +{ + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + int32_t placed = 0; + bool clamped = false; + for ( ;; ) + { + char unit[4]; + int32_t unit_length = 0; + if ( !TableJsonScanUnit( in, unit, &unit_length ) ) { return false; } + if ( unit_length == 0 ) { break; } + const uint32_t code = TableJsonDecodeUtf8( unit, unit_length ); + char16_t units[2]; + int32_t units_length = 1; + if ( code < 0x10000 ) + { + units[0] = (char16_t) code; + } + else + { + const uint32_t rest = code - 0x10000; + units[0] = (char16_t) ( 0xd800 + ( rest >> 10 ) ); + units[1] = (char16_t) ( 0xdc00 + ( rest & 0x3ff ) ); + units_length = 2; + } + if ( out == NULL ) + { + placed += units_length; + } + else if ( !clamped && placed + units_length <= capacity ) + { + for ( int32_t i = 0; i < units_length; i++ ) { out[placed + i] = units[i]; } + placed += units_length; + } + else + { + // A CLAMP IS A PREFIX, the narrow scan's own rule: once one code + // point does not fit, the scan stops placing. A pair is placed + // whole or not at all, so no clamp can leave an unpaired + // surrogate in storage. + clamped = true; + } + } + if ( clamped ) { in.report->clamped++; } + if ( length != NULL ) { *length = placed; } + return true; +} + + +// the numeric token at the cursor, copied out whole; false = not a number +// Scan one number, to JSON's OWN grammar (RFC 8259 §6) and not to a run of +// number-ish characters: +// +// number = [ "-" ] int [ frac ] [ exp ] +// int = "0" / ( digit1-9 *digit ) +// frac = "." 1*digit +// exp = ( "e" / "E" ) [ "-" / "+" ] 1*digit +// +// Scanning the production is what makes a typo in an authoring file a +// DIAGNOSTIC rather than a value: "1-2" scans as 1 and leaves "-2" where the +// object expects a comma, so the text is malformed — which is what §16.2 +// already promises. A permissive scan would hand "1-2" to a digit loop and +// report a clamp, and a config pipeline would never hear about it. Leading +// "+", leading zeros, ".5" and "3." are not JSON either. +inline bool TableJsonWalkNumber( TableJsonIn & in, bool * integral ) +{ + TableJsonSpace( in ); + bool whole = true; + if ( in.pos < in.size && in.text[in.pos] == '-' ) { in.pos++; } + // int: a lone zero, or a non-zero digit and any digits after it + if ( in.pos >= in.size ) { return false; } + if ( in.text[in.pos] == '0' ) + { + in.pos++; + } + else if ( in.text[in.pos] >= '1' && in.text[in.pos] <= '9' ) + { + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + } + else + { + return false; + } + // frac + if ( in.pos < in.size && in.text[in.pos] == '.' ) + { + in.pos++; + if ( in.pos >= in.size || in.text[in.pos] < '0' || in.text[in.pos] > '9' ) { return false; } + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + whole = false; + } + // exp + if ( in.pos < in.size && ( in.text[in.pos] == 'e' || in.text[in.pos] == 'E' ) ) + { + in.pos++; + if ( in.pos < in.size && ( in.text[in.pos] == '-' || in.text[in.pos] == '+' ) ) { in.pos++; } + if ( in.pos >= in.size || in.text[in.pos] < '0' || in.text[in.pos] > '9' ) { return false; } + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + whole = false; + } + *integral = whole; + return true; +} + +// the same production, with the token kept for conversion +inline bool TableJsonScanNumber( TableJsonIn & in, char * token, int32_t capacity, int32_t * length, bool * integral ) +{ + TableJsonSpace( in ); + int64_t start = in.pos; + if ( !TableJsonWalkNumber( in, integral ) ) { return false; } + int64_t count = in.pos - start; + if ( count <= 0 || count >= capacity ) { return false; } + memcpy( token, in.text + start, (size_t) count ); + token[count] = 0; + *length = (int32_t) count; + return true; +} + +// the token's exact double, through the runtime's own converter — which +// speaks the LOCALE's decimal point, so the token crosses back over that +// character on its way in +inline double TableJsonTokenDouble( const char * token, int32_t length, bool single ) +{ + char work[kTableJsonMaxNumber]; + memcpy( work, token, (size_t) length ); + work[length] = 0; + char point = TableJsonDecimalPoint(); + if ( point != '.' ) + { + for ( int32_t i = 0; i < length; i++ ) + { + if ( work[i] == '.' ) { work[i] = point; } + } + } + if ( single ) { return (double) strtof( work, NULL ); } + return strtod( work, NULL ); +} + +// ---- ONE CHECKED NUMERIC INTERPRETATION (docs/SPEC-TABLES.md §16.2) ---- +// +// JSON HAS ONE NUMBER TYPE, so every integer target reads a token the same way +// and the VALUE decides rather than the spelling: 2, 2.0 and 1e3 are the +// integers 2, 2 and 1000. What comes out of a token is a SIGN, a MAGNITUDE and +// a STATUS, and nothing on the way is cast through a type that cannot hold what +// it is handed. A uint64 magnitude past INT64_MAX is a magnitude and never a +// negative, and a double is consulted only for a spelling the digit path cannot +// read exactly. +// +// TWO POLICIES SIT ON TOP OF THE ONE VALUE and neither reinterprets the token: +// an ordinary FIELD clamps to its domain and counts, and a MAP KEY rejects the +// whole entry, because a key is an identity and a clamped one is two entries +// merged. That difference is the only difference between them. +// +// THE KEY READS ITS TOKEN EXACTLY AND A FIELD READS IT THROUGH THE DOUBLE, and +// that is the one place the two interpretations part. A key is an IDENTITY, so +// two spellings a 53-bit mantissa cannot tell apart are two keys and the key +// path carries TableJsonInterpretExact below. A field's value is a quantity +// under a clamp, and its interpretation is the one the C, Go and Rust ports +// read the same texts with, so it lives here and reads as they read. +struct TableJsonInteger +{ + uint64_t magnitude; // |value|, exact for every integral token 64 bits hold + bool negative; + bool fractional; // a genuinely fractional VALUE: the wrong shape for an integer + bool saturated; // a magnitude past what 64 bits hold, held at that edge + bool finite; // false: no integer target holds it at all +}; + +// THE FIELD'S INTERPRETATION: the token, parsed digit by digit so no width and +// no locale can move it, and through the runtime's converter only where the +// spelling carries a fraction or an exponent +inline TableJsonInteger TableJsonInterpret( const char * token, int32_t length, bool integral ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = false; + out.fractional = false; + out.saturated = false; + out.finite = true; + if ( integral ) + { + int32_t i = 0; + if ( i < length && token[i] == '-' ) // WalkNumber refuses a leading plus + { + out.negative = true; + i++; + } + for ( ; i < length; i++ ) + { + const uint64_t digit = (uint64_t) ( token[i] - '0' ); + if ( out.magnitude > ( UINT64_MAX - digit ) / 10 ) + { + out.magnitude = UINT64_MAX; + out.saturated = true; + break; + } + out.magnitude = out.magnitude * 10 + digit; + } + if ( out.magnitude == 0 ) { out.negative = false; } // -0 IS zero + return out; + } + const double d = TableJsonTokenDouble( token, length, false ); + if ( !TableJsonFinite( d ) ) { out.finite = false; return out; } + out.negative = d < 0; + const double whole = out.negative ? -d : d; + // THE DOMAIN IS ESTABLISHED BEFORE THE CAST: a magnitude past what sixty-four + // bits hold is answered here, so no value ever reaches a conversion that is + // undefined for it + if ( whole >= 18446744073709551616.0 ) + { + out.magnitude = UINT64_MAX; + out.saturated = true; + return out; + } + const uint64_t truncated = (uint64_t) whole; + if ( (double) truncated != whole ) { out.fractional = true; return out; } + out.magnitude = truncated; + if ( out.magnitude == 0 ) { out.negative = false; } + return out; +} + +// THE DECIMAL BAND a token is answered in without arithmetic: 10^20 is above +// UINT64_MAX whatever the digits are, so a point past it saturates and a token +// spelling 1e999999999 costs nothing to refuse. +const int64_t kTableJsonDecimalBand = 20; + +// THE MAP KEY'S INTERPRETATION, and no other path's: the token's own digits, +// read where they stand, so no 53-bit mantissa decides the identity of a 64-bit +// key. The int and frac runs are one digit string with the point after "point" +// of them, and the exponent moves the point rather than the digits, which is +// the normalization the wide kinds already use over one 64-bit lane. A zero +// fraction is the integer the token spells at every magnitude the kind holds, +// so 9007199254740993.0 is that key rather than the one a double rounds it to. +// No exact reader has an infinity, so finite is true here always. +inline TableJsonInteger TableJsonInterpretExact( const char * token, int32_t length ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = false; + out.fractional = false; + out.saturated = false; + out.finite = true; + int32_t i = 0; + if ( i < length && token[i] == '-' ) { out.negative = true; i++; } // WalkNumber refuses a leading plus + const char * int_digits = token + i; + int32_t int_len = 0; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { int_len++; i++; } + const char * frac_digits = token + i; + int32_t frac_len = 0; + if ( i < length && token[i] == '.' ) + { + i++; + frac_digits = token + i; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { frac_len++; i++; } + } + int64_t exp = 0; + if ( i < length && ( token[i] == 'e' || token[i] == 'E' ) ) + { + i++; + bool exp_negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { exp_negative = token[i] == '-'; i++; } + while ( i < length && token[i] >= '0' && token[i] <= '9' ) + { + if ( exp < 100000 ) { exp = exp * 10 + ( token[i] - '0' ); } + i++; + } + if ( exp_negative ) { exp = -exp; } + } + // leading and trailing zeros stripped, so the last digit kept is significant + int32_t start = 0, end = int_len + frac_len; + int64_t point = (int64_t) int_len + exp; + while ( start < end && ( start < int_len ? int_digits[start] : frac_digits[start - int_len] ) == '0' ) { start++; point--; } + while ( end > start && ( end - 1 < int_len ? int_digits[end - 1] : frac_digits[end - 1 - int_len] ) == '0' ) { end--; } + const int64_t digits = end - start; + if ( digits == 0 ) { out.negative = false; return out; } // the value is zero, and -0 IS zero + if ( point < digits ) { out.fractional = true; return out; } // a significant digit below the point + if ( point > kTableJsonDecimalBand ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + for ( int32_t k = start; k < end; k++ ) + { + const uint64_t digit = (uint64_t) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ); + if ( out.magnitude > ( UINT64_MAX - digit ) / 10 ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + out.magnitude = out.magnitude * 10 + digit; + } + for ( int64_t k = digits; k < point; k++ ) // the point's own zeros, which no digit spells + { + if ( out.magnitude > UINT64_MAX / 10 ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + out.magnitude *= 10; + } + return out; +} + +// a declared range bound as the same value. A bound is inside the field's own +// domain by construction, so nothing here saturates. +inline TableJsonInteger TableJsonIntegerOf( double bound ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = bound < 0; + out.fractional = false; + out.saturated = false; + out.finite = true; + const double whole = out.negative ? -bound : bound; + out.magnitude = whole >= 18446744073709551616.0 ? UINT64_MAX : (uint64_t) whole; + if ( out.magnitude == 0 ) { out.negative = false; } + return out; +} + +// THE TARGET DOMAIN, established before the value reaches storage: the bytes of +// storage, signed or not. Answers what the target holds and whether the domain +// MOVED it. An unsigned magnitude above INT64_MAX rides out as its bit pattern, +// which is the storage's own image of it and not a negative number. +inline int64_t TableJsonIntegerInDomain( const TableJsonInteger & number, bool is_signed, int32_t bytes, bool & moved ) +{ + moved = false; + uint64_t magnitude = number.magnitude; + if ( is_signed ) + { + const uint64_t high = bytes >= 8 ? (uint64_t) INT64_MAX : ( ( uint64_t( 1 ) << ( bytes * 8 - 1 ) ) - 1 ); + if ( number.negative ) + { + const uint64_t low = high + 1; // the floor's magnitude + if ( magnitude > low ) { magnitude = low; moved = true; } + return (int64_t) ( ~magnitude + 1 ); // two's complement, INT64_MIN included + } + if ( magnitude > high ) { magnitude = high; moved = true; } + return (int64_t) magnitude; + } + // A NEGATIVE TOKEN IN AN UNSIGNED FIELD CLAMPS TO ZERO, and -0 is zero, + // which is why the sign is dropped at a zero magnitude above + if ( number.negative ) { moved = true; return 0; } + const uint64_t high = bytes >= 8 ? UINT64_MAX : ( ( uint64_t( 1 ) << ( bytes * 8 ) ) - 1 ); + if ( magnitude > high ) { magnitude = high; moved = true; } + return (int64_t) magnitude; +} + +// A number token into a wide kind's raw storage (docs/SPEC-TABLES.md §16.2). A +// 128-bit integer takes any token whose VALUE is integral; a fixed field any +// token whose value is EXACTLY representable in its Q I.F — a finer fraction +// is the wrong shape for the field, counted as a kind mismatch and never +// rounded, the rule SPEC.md §4.6 gives a fixed default. A magnitude past 128 +// bits saturates and counts as a clamp, as an int64 field saturates at +// INT64_MAX; the declared range clamps after it, on the RAW scale, as it does +// for every bounded scalar. +// +// The token is normalized to its digits with the decimal point after "point" +// of them. An integer part past 40 digits is above 2^128 whatever the digits +// are, and a value below 10^-40 is finer than 2^-127, the finest fraction any +// F can spell — so outside that band the answer is known without the +// arithmetic, and a token spelling 1e999999999 costs nothing to refuse. +inline bool TableJsonReadWide( TableJsonIn & in, const char * token, int32_t length, void * storage, const TableFieldInfo * f ) +{ + bool is_signed = TableJsonKindWideSigned( f->kind ); + int frac = f->frac_bits; + int32_t i = 0; + bool negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { negative = token[i] == '-'; i++; } + const char * int_digits = token + i; + int32_t int_len = 0; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { int_len++; i++; } + const char * frac_digits = token + i; + int32_t frac_len = 0; + if ( i < length && token[i] == '.' ) + { + i++; + frac_digits = token + i; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { frac_len++; i++; } + } + int64_t exp = 0; + if ( i < length && ( token[i] == 'e' || token[i] == 'E' ) ) + { + i++; + bool exp_negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { exp_negative = token[i] == '-'; i++; } + while ( i < length && token[i] >= '0' && token[i] <= '9' ) + { + if ( exp < 100000 ) { exp = exp * 10 + ( token[i] - '0' ); } + i++; + } + if ( exp_negative ) { exp = -exp; } + } + // the digits, with the point after "point" of them; leading and trailing + // zeros stripped. digit( k ) reads the k-th of the int and frac runs. + int32_t start = 0, end = int_len + frac_len; + int64_t point = int_len + exp; + while ( start < end && ( start < int_len ? int_digits[start] : frac_digits[start - int_len] ) == '0' ) { start++; point--; } + while ( end > start && ( end - 1 < int_len ? int_digits[end - 1] : frac_digits[end - 1 - int_len] ) == '0' ) { end--; } + + TableJsonWide raw = { 0, 0 }; + bool saturated = false; + TableJsonWide signed_max = { ~uint64_t( 0 ), ~uint64_t( 0 ) >> 1 }; + TableJsonWide signed_min = { 0, uint64_t( 1 ) << 63 }; + TableJsonWide unsigned_max = { ~uint64_t( 0 ), ~uint64_t( 0 ) }; + if ( start == end ) + { + // zero, and -0 IS zero + } + else if ( point > 40 ) + { + saturated = true; + if ( !negative ) { raw = is_signed ? signed_max : unsigned_max; } + else if ( is_signed ) { raw = signed_min; } + } + else if ( point < -40 ) + { + in.report->kind_mismatch++; // finer than any F can spell + return true; + } + else + { + // the fraction FIRST, so an inexact value is the wrong shape whatever + // its magnitude: its digits, with the zeros a negative point puts in + // front, doubled F times; each doubling's carry is the next bit, and + // the value is exact iff nothing is left after the last one + char fd[kTableJsonMaxNumber + 48]; + int32_t fn = 0; + for ( int64_t z = point; z < 0; z++ ) { fd[fn++] = 0; } + for ( int32_t k = (int32_t) ( point > 0 ? point : 0 ) + start; k < end; k++ ) + { + fd[fn++] = (char) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ); + } + TableJsonWide fraction = { 0, 0 }; + for ( int b = 0; b < frac; b++ ) + { + int carry = 0; + for ( int32_t k = fn - 1; k >= 0; k-- ) + { + int d = fd[k] * 2 + carry; + fd[k] = (char) ( d % 10 ); + carry = d / 10; + } + fraction = TableJsonWideShl( fraction, 1 ); + fraction.lo |= (uint64_t) carry; + } + for ( int32_t k = 0; k < fn; k++ ) + { + if ( fd[k] != 0 ) + { + in.report->kind_mismatch++; + return true; + } + } + // then the whole part, saturating past 128 bits + TableJsonWide whole = { 0, 0 }; + for ( int64_t k = start; k < start + point && !saturated; k++ ) + { + uint32_t digit = k < end ? (uint32_t) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ) : 0; + if ( TableJsonWideMulAdd( &whole, 10, digit ) != 0 ) { saturated = true; } + } + if ( !saturated && frac > 0 && !TableJsonWideZero( TableJsonWideShr( whole, 128 - frac ) ) ) { saturated = true; } + if ( !saturated ) + { + raw = TableJsonWideShl( whole, frac ); + raw.lo |= fraction.lo; + raw.hi |= fraction.hi; + } + if ( is_signed ) + { + if ( !saturated && !negative && TableJsonWideNegative( raw ) ) { saturated = true; } + if ( !saturated && negative && TableJsonWideCompare( raw, signed_min, false ) > 0 ) { saturated = true; } + if ( saturated ) { raw = negative ? signed_min : signed_max; } + else if ( negative ) { raw = TableJsonWideNeg( raw ); } + } + else + { + if ( saturated ) { raw = unsigned_max; } + if ( negative && !TableJsonWideZero( raw ) ) { raw.lo = 0; raw.hi = 0; saturated = true; } + } + } + if ( saturated ) { in.report->clamped++; } + if ( f->wide != NULL ) + { + TableJsonWide lo = { f->wide->lo[0], f->wide->lo[1] }; + TableJsonWide hi = { f->wide->hi[0], f->wide->hi[1] }; + if ( TableJsonWideCompare( raw, lo, is_signed ) < 0 ) { raw = lo; in.report->clamped++; } + else if ( TableJsonWideCompare( raw, hi, is_signed ) > 0 ) { raw = hi; in.report->clamped++; } + } + TableJsonWideStore( storage, f->elem_size, raw ); + return true; +} + +inline bool TableJsonSkipValue( TableJsonIn & in, int32_t depth ); + +inline bool TableJsonSkipContainer( TableJsonIn & in, char close, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; // the opening bracket + bool first = true; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == close ) { in.pos++; return true; } + if ( c == 0 ) { in.bad = true; return false; } + if ( close == '}' ) + { + // the key is kept, because a skipped OBJECT may still be a + // pointer's: an `&node` opening it names a node the storage could + // not hold, and the numbering has to survive the drop (§16.7). + // Anywhere but first, the prefix is the reserved key out of place + // — in a pointered unit; a fixed unit skips the value whole. + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + if ( key[0] == '&' && in.graph != NULL ) + { + if ( !first ) { in.report->malformed = true; in.bad = true; return false; } + if ( !TableJsonSkippedAmpersand( in, key, depth ) ) { return false; } + first = false; + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == close ) { in.pos++; return true; } + in.bad = true; + return false; + } + } + first = false; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == close ) { in.pos++; return true; } + in.bad = true; + return false; + } +} + +inline bool TableJsonSkipValue( TableJsonIn & in, int32_t depth ) +{ + char c = TableJsonPeek( in ); + switch ( c ) + { + case '{': return TableJsonSkipContainer( in, '}', depth ); + case '[': return TableJsonSkipContainer( in, ']', depth ); + case '"': return TableJsonScanString( in, NULL, 0, NULL ); + case 't': return TableJsonLiteral( in, "true" ); + case 'f': return TableJsonLiteral( in, "false" ); + case 'n': return TableJsonLiteral( in, "null" ); + case 0: in.bad = true; return false; + default: + { + // consumed, never converted: skipping needs no buffer, and this + // is the one walk a hostile text drives to the depth cap. It is + // the SAME production the value path scans, so an unknown key + // cannot smuggle past a number a named key would refuse. + bool integral = false; + if ( !TableJsonWalkNumber( in, &integral ) ) { in.bad = true; return false; } + return true; + } + } +} + +inline bool TableJsonReadTable( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth ); +// a UNION ARM that names no declaration reads through the field walk one key +// down (docs/SPEC-TABLES.md §2.6, §16.2), which is defined below +inline bool TableJsonReadField( TableJsonIn & in, void * base, const TableFieldInfo * f, int32_t depth ); + +// place one scalar at one storage address +inline bool TableJsonReadScalar( TableJsonIn & in, void * storage, const TableFieldInfo * f, int32_t depth ) +{ + if ( f->arms != NULL ) + { + // a union is an object with ONE key, the arm's name; {} is None, and + // two keys is a text this walk will not guess at + const TableUnionInfo * arms = f->arms(); + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + TableJsonSetRaw( (uint8_t *) storage + arms->tag_offset, arms->tag_size, 0 ); + if ( TableJsonPeek( in ) == '}' ) { in.pos++; return true; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t tag = 0; + for ( int64_t t = 1; t <= f->enum_max; t++ ) + { + if ( strcmp( f->enum_name( (uint64_t) t ), key ) == 0 ) { tag = t; break; } + } + if ( tag == 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + void * payload = (uint8_t *) storage + arms->arms[tag].offset; + const TableFieldInfo * arm = arms->arms[tag].field; + bool placed = true; + if ( arm != NULL ) + { + // THE ARM'S VALUE TAKES THE ARM'S OWN ROW (§16.2). A value of + // the wrong shape for that row is a KIND MISMATCH: the union + // reads None, the event is counted, and the enclosing object + // continues — the rule a FIELD's value lives under, one key + // down. A pointer arm's null is a null pointer, not a shape + // error, exactly as a pointer field's is (§16.7). + char got = TableJsonValueShape( in ); + if ( arm->kind == 17 && !arm->is_array && got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + memset( payload, 0, (size_t) arms->arms[tag].size ); + } + else if ( got != TableJsonShape( arm ) ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else if ( arm->kind == 17 && !arm->is_array ) + { + // A POINTER ARM'S VALUE IS THE POINTEE IN PLACE, or a + // node reference to one (§16.7) — the read a pointer + // FIELD takes, which is not the scalar walk + memset( payload, 0, (size_t) arms->arms[tag].size ); + if ( !TableJsonReadPointer( in, payload, arm, depth + 1 ) ) { return false; } + } + else + { + // SELECTION ZERO-ESTABLISHES THE ARM (SPEC §5): an arm + // takes no specified default, so zero is the establish + memset( payload, 0, (size_t) arms->arms[tag].size ); + if ( !TableJsonReadField( in, storage, arm, depth + 1 ) ) { return false; } + } + } + else if ( arms->arms[tag].table != NULL ) + { + if ( TableJsonValueShape( in ) != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else + { + arms->arms[tag].table->reset( payload ); + if ( !TableJsonReadTable( in, payload, arms->arms[tag].table, depth + 1 ) ) { return false; } + } + } + else + { + // A PAYLOAD-FREE ARM'S VALUE IS null (§2.6): the arm name + // selects it and there is nothing to place + if ( TableJsonValueShape( in ) != 'z' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else if ( !TableJsonLiteral( in, "null" ) ) + { + return false; + } + } + if ( placed ) + { + TableJsonSetRaw( (uint8_t *) storage + arms->tag_offset, arms->tag_size, (uint64_t) tag ); + } + } + char c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; c = TableJsonPeek( in ); } + if ( c == '}' ) { in.pos++; return true; } + in.bad = true; // a second key: a one-of with two arms is not a value + return false; + } + if ( f->kind == 13 ) + { + f->table->reset( storage ); + return TableJsonReadTable( in, storage, f->table, depth + 1 ); + } + if ( TableJsonIsEnum( f ) ) + { + char name[kTableJsonMaxKey]; + int32_t name_length = 0; + if ( !TableJsonScanString( in, name, kTableJsonMaxKey - 1, &name_length ) ) { return false; } + name[name_length] = 0; + for ( int64_t v = 0; v <= f->enum_max; v++ ) + { + if ( strcmp( f->enum_name( (uint64_t) v ), name ) == 0 ) + { + TableJsonSetRaw( storage, f->elem_size, (uint64_t) v ); + return true; + } + } + // a name this build cannot name reads as None and counts as unknown, + // exactly as an unknown variant id does on the wire (§4) + TableJsonSetRaw( storage, f->elem_size, 0 ); + in.report->unknown++; + return true; + } + if ( TableJsonIsFlags( f ) ) + { + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + in.pos++; + uint64_t bits = 0; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + if ( c != '"' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + char name[kTableJsonMaxKey]; + int32_t name_length = 0; + if ( !TableJsonScanString( in, name, kTableJsonMaxKey - 1, &name_length ) ) { return false; } + name[name_length] = 0; + bool found = false; + for ( int64_t bit = 0; bit <= f->enum_max; bit++ ) + { + if ( strcmp( f->enum_name( (uint64_t) bit ), name ) == 0 ) + { + bits |= uint64_t( 1 ) << bit; + found = true; + break; + } + } + if ( !found ) { in.report->unknown++; } + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + TableJsonSetRaw( storage, f->elem_size, bits ); + return true; + } + if ( f->kind == 1 ) + { + char c = TableJsonPeek( in ); + if ( c == 't' ) { if ( !TableJsonLiteral( in, "true" ) ) { return false; } TableJsonSetRaw( storage, f->elem_size, 1 ); return true; } + if ( !TableJsonLiteral( in, "false" ) ) { return false; } + TableJsonSetRaw( storage, f->elem_size, 0 ); + return true; + } + char token[kTableJsonMaxNumber]; + int32_t length = 0; + bool integral = false; + if ( !TableJsonScanNumber( in, token, kTableJsonMaxNumber, &length, &integral ) ) + { + in.bad = true; + return false; + } + if ( TableJsonKindWide( f->kind ) ) + { + return TableJsonReadWide( in, token, length, storage, f ); + } + if ( f->kind == 10 || f->kind == 11 ) + { + bool single = f->kind == 10; + double value = TableJsonTokenDouble( token, length, single ); + // A magnitude the field's format cannot hold is the WRONG SHAPE for + // the kind, and it never reaches storage: 1e400 is not a float64 and + // 1e300 is not a float32. Storing the infinity the conversion + // produced would leave an instance this walk called CLEAN that + // ToJsonMeasure then refuses forever (a non-finite float has no JSON + // spelling), and §16.1's one invariant is that a text which reads + // clean writes back. + if ( !TableJsonFinite( value ) ) + { + in.report->kind_mismatch++; + return true; + } + if ( f->has_range ) + { + if ( value < f->range_min ) { value = f->range_min; in.report->clamped++; } + else if ( value > f->range_max ) { value = f->range_max; in.report->clamped++; } + } + if ( single ) + { + float narrow = (float) value; + if ( !TableJsonFinite( (double) narrow ) ) + { + in.report->kind_mismatch++; + return true; + } + memcpy( storage, &narrow, sizeof( narrow ) ); + } + else + { + memcpy( storage, &value, sizeof( value ) ); + } + return true; + } + // AN ORDINARY FIELD'S POLICY over the one interpreted value: it CLAMPS to + // its domain and counts. JSON has one number type, so 2.0 IS the integer 2 + // and 1e3 IS 1000. A library that round-trips numbers through a double + // emits them that way, and this walker's own float writer emits 1e+21. Only + // a genuinely fractional value is the wrong shape for the kind. + const bool is_signed = f->kind >= 2 && f->kind <= 5; + const TableJsonInteger number = TableJsonInterpret( token, length, integral ); + if ( !number.finite || number.fractional ) + { + in.report->kind_mismatch++; + return true; + } + if ( number.saturated ) { in.report->clamped++; } // past what sixty-four bits hold + // THE DECLARED RANGE FIRST, THEN THE STORAGE WIDTH, the wire's order (§4), + // so a text and a wire loaded from the same data land the same instance. + // The comparison is on the value's OWN scale, correctly signed past + // INT64_MAX, where the storage's bit pattern is not a number to compare. + TableJsonInteger bounded = number; + if ( f->has_range ) + { + const double scale = number.negative ? -(double) number.magnitude : (double) number.magnitude; + if ( scale < f->range_min ) { bounded = TableJsonIntegerOf( f->range_min ); in.report->clamped++; } + else if ( scale > f->range_max ) { bounded = TableJsonIntegerOf( f->range_max ); in.report->clamped++; } + } + bool moved = false; + const int64_t value = TableJsonIntegerInDomain( bounded, is_signed, (int32_t) f->elem_size, moved ); + if ( moved ) { in.report->clamped++; } + TableJsonSetRaw( storage, f->elem_size, (uint64_t) value ); + return true; +} + +inline bool TableJsonReadField( TableJsonIn & in, void * base, const TableFieldInfo * f, int32_t depth ) +{ + uint8_t * storage = (uint8_t *) base + f->offset; + if ( TableJsonIsMap( f ) ) + { + return TableJsonReadMap( in, (void *) storage, f, depth ); + } + if ( TableJsonIsList( f ) ) + { + return TableJsonReadList( in, (void *) storage, f, depth ); + } + + if ( f->kind == 12 ) + { + int32_t length = 0; + if ( !TableJsonScanString( in, (char *) storage, f->array_bound, &length ) ) { return false; } + storage[length] = 0; + TableJsonSetCount( base, f, length ); + return true; + } + if ( f->kind == 33 ) + { + char16_t * units = (char16_t *) (void *) storage; + int32_t length = 0; + if ( !TableJsonScanWString( in, units, (int32_t) f->array_bound, &length ) ) { return false; } + units[length] = 0; // the terminating zero UNIT at index length (§7.2, SPEC.md §4.12) + TableJsonSetCount( base, f, length ); + return true; + } + if ( TableJsonIsBytes( f ) ) + { + // base64 decodes STRAIGHT INTO the field's storage, six bits at a + // time — no window, no temporary, so a bytes(N) of any declared + // extent reads the same way. A base64 body carries no escapes, so a + // backslash in one is simply not an alphabet character. + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + memset( storage, 0, (size_t) f->array_bound ); + TableJsonSetCount( base, f, 0 ); + const char * alphabet = TableJsonBase64Alphabet(); + int32_t placed = 0; + uint32_t accumulator = 0; + int32_t held = 0; + bool clamped = false; + bool malformed = false; + for ( ;; ) + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos++]; + if ( c == '"' ) { break; } + if ( c == '=' || malformed ) { continue; } + const char * at = c != 0 ? strchr( alphabet, c ) : NULL; + if ( at == NULL ) { malformed = true; continue; } + accumulator = ( accumulator << 6 ) | (uint32_t) ( at - alphabet ); + held += 6; + if ( held >= 8 ) + { + held -= 8; + if ( placed < f->array_bound ) + { + storage[placed++] = (uint8_t) ( ( accumulator >> held ) & 0xff ); + } + else + { + clamped = true; + } + } + } + if ( malformed ) + { + // a body that is not base64 is the wrong shape for the kind: the + // field keeps its default and the event is counted + in.report->kind_mismatch++; + return true; + } + if ( clamped ) { in.report->clamped++; } + TableJsonSetCount( base, f, placed ); + return true; + } + if ( TableJsonIsKeyed( f ) ) + { + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + // every slot back to its declared defaults first, so a key the text + // omits keeps them and a repeated field key cannot leave an earlier + // occurrence's slots standing + for ( int32_t i = 0; i < f->array_bound; i++ ) + { + void * slot = storage + (int64_t) i * f->elem_size; + if ( f->kind == 13 ) { f->table->reset( slot ); } + else { memset( slot, 0, (size_t) f->elem_size ); } + } + char shape = TableJsonElementShape( f ); + // A KEYED OBJECT'S KEYS ARE KEYS: a variant named twice is a duplicate + // key like any other, last-wins and counted (§16.2). Tracked the way + // a table's own field keys are — a bounded, allocation-free bitmask; + // a vocabulary wider than this still reads, its repeats simply stop + // being counted. + uint64_t seen[8] = {}; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t slot = -1; + for ( int64_t v = 0; v < f->array_bound; v++ ) + { + // nothing is stored for None, so "None" finds no slot and is + // an unknown key like any other name this reader cannot place + if ( !TableJsonKeyedSlotValid( f, v ) ) { continue; } + if ( strcmp( f->key_name( TableJsonKeyedSlotKey( v ) ), key ) == 0 ) { slot = v; break; } + } + if ( slot >= 0 && slot < 512 ) + { + uint64_t bit = uint64_t( 1 ) << ( slot & 63 ); + if ( ( seen[slot >> 6] & bit ) != 0 ) { in.report->duplicate++; } + seen[slot >> 6] |= bit; + } + if ( slot < 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( TableJsonValueShape( in ) != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadScalar( in, storage + slot * f->elem_size, f, depth + 1 ) ) + { + return false; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; + } + if ( f->is_array ) + { + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + in.pos++; + // LAST WINS has to be true of a repeated ARRAY key too, and it is + // wire-visible: a fixed array writes every slot, so a second, shorter + // occurrence overlaying a prefix would leave the first occurrence's + // tail standing. The field goes back to its declared defaults before + // this occurrence's elements are placed — the re-establishment a nested + // table and a union arm already get. A table element's defaults are + // its own (the reset hook); every other element kind's storage + // default is zero, which is what the generated array declares. + if ( f->kind == 13 ) + { + for ( int32_t i = 0; i < f->array_bound; i++ ) + { + f->table->reset( storage + (int64_t) i * f->elem_size ); + } + } + else + { + memset( storage, 0, (size_t) f->array_bound * (size_t) f->elem_size ); + } + TableJsonSetCount( base, f, 0 ); + int32_t placed = 0; + char shape = TableJsonElementShape( f ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + if ( placed >= f->array_bound ) + { + // more elements than the reader's bound: the bounded prefix + // is kept and the excess counts, the wire's rule (§4) + in.report->clamped++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( f->kind == 17 ) + { + // an element of an ARRAY OF POINTERS (§2.1): null is a null slot, an + // object is the pointee in place or an `&node` reference (§16.7) + char got = TableJsonValueShape( in ); + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( storage + (int64_t) placed * f->elem_size, f->elem_size, 0 ); + } + else if ( got != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, storage + (int64_t) placed * f->elem_size, f, depth + 1 ) ) { return false; } + placed++; + } + else if ( TableJsonValueShape( in ) != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed++; + } + else + { + if ( !TableJsonReadScalar( in, storage + (int64_t) placed * f->elem_size, f, depth + 1 ) ) { return false; } + placed++; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + // a fixed array's tail keeps the defaults the prefill left there, + // exactly as a short wire count does + TableJsonSetCount( base, f, placed ); + return true; + } + return TableJsonReadScalar( in, storage, f, depth ); +} + +inline bool TableJsonReadTableKeys( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth, const char * first_key ); + +// ONE table object: keys are field keys, unknown ones are skipped and +// counted, a repeated key is last-wins and counted. The instance is already +// at its declared defaults when this is entered, so a key the text never +// mentions keeps the default an absent field takes on the wire (§4). +inline bool TableJsonReadTable( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + return TableJsonReadTableKeys( in, base, info, depth, NULL ); +} + +// The keys of an object whose brace is already consumed. A pointer's object +// opens the same way a table's does, but its FIRST key may be `&node` (§16.7) +// and the adapter that reads it has to scan the key to know — so it hands the +// key it scanned in as `first_key`, with the colon consumed, and this places +// it before scanning the rest. +inline bool TableJsonReadTableKeys( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth, const char * first_key ) +{ + // duplicate tracking, bounded and allocation-free: a table with more + // fields than this still reads, its repeats simply stop being counted + uint64_t seen[8] = {}; + for ( ;; ) + { + char key[kTableJsonMaxKey]; + char c = 0; + if ( first_key != NULL ) + { + memcpy( key, first_key, strlen( first_key ) + 1 ); // scanned into a buffer this size by the caller + first_key = NULL; + } + else + { + c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; return true; } + if ( c == 0 ) { in.bad = true; return false; } + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + } + int32_t index = -1; + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + if ( strcmp( info->fields[i].json, key ) == 0 ) { index = i; break; } + } + if ( key[0] == '&' ) + { + // THE AMPERSAND PREFIX IS RESERVED TO THE FORM (docs/SPEC-TABLES.md + // §16.7). No declaration may take a key beginning with it, so this + // is never a field this build lacks — it is the sharing construct + // somewhere it cannot stand: `&node` is the FIRST key of a pointer's + // object and nothing else, and the adapter that reads a pointer + // has consumed it before these keys are read. MALFORMED, refused + // and counted; never counted as unknown, never skipped. + in.report->malformed = true; + in.bad = true; + return false; + } + if ( index < 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + const TableFieldInfo * f = &info->fields[index]; + if ( index < 512 ) + { + uint64_t bit = uint64_t( 1 ) << ( index & 63 ); + if ( ( seen[index >> 6] & bit ) != 0 ) { in.report->duplicate++; } + seen[index >> 6] |= bit; + } + // PRESENCE OF THE KEY IS THE PRESENCE (§16.2): reaching this line + // is the key being present, so an optional is set present + // whatever its value — with one exception the page names: a JSON + // null, which reads as ABSENT rather than as a value. + char got = TableJsonValueShape( in ); + if ( f->kind == 17 && !f->is_array ) + { + // a pointer: null is a null pointer, an object is the pointee + // in place or an `&node` reference to one (§16.7), a string is + // a BYTE BUFFER's bytes (§2.5), and anything else is the wrong + // shape for the kind + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) base + f->offset, f->elem_size, 0 ); + } + else if ( got != TableJsonShape( f ) ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, (uint8_t *) base + f->offset, f, depth ) ) + { + return false; + } + } + else if ( f->optional && got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + // absent, and back at its defaults: a repeated key whose last + // occurrence is null must not leave an earlier value standing + if ( f->table != NULL ) { f->table->reset( (uint8_t *) base + f->offset ); } + else { memset( (uint8_t *) base + f->offset, 0, (size_t) f->elem_size ); } + TableJsonSetRaw( (uint8_t *) base + f->present_offset, 1, 0 ); + } + else + { + if ( got != TableJsonShape( f ) ) + { + // the wrong JSON type for the kind: skipped, never coerced + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadField( in, base, f, depth ) ) + { + return false; + } + if ( f->optional ) + { + TableJsonSetRaw( (uint8_t *) base + f->present_offset, 1, 1 ); + } + } + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; return true; } + in.bad = true; + return false; + } +} + +// ---- the two entry points the per-table wrappers name ---- + +inline bool TableJsonRead( void * value, const TableTypeInfo * info, const char * text, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableJsonIn in; + in.text = text; + in.size = bytes; + in.pos = 0; + in.report = report != NULL ? report : &ignored; + in.bad = false; + in.graph = NULL; + info->reset( value ); + if ( text == NULL || bytes < 0 ) + { + in.report->malformed = true; + return false; + } + bool ok = TableJsonReadTable( in, value, info, 0 ); + if ( ok ) + { + TableJsonSpace( in ); + if ( in.pos != in.size ) { in.bad = true; } // trailing rubbish is not one text + } + if ( in.bad || !ok ) + { + in.report->malformed = true; + return false; + } + return true; +} + +inline int64_t TableJsonWrite( const void * value, const TableTypeInfo * info, char * buffer, int64_t capacity ) +{ + TableJsonOut out; + out.buffer = buffer; + out.capacity = capacity; + out.offset = 0; + out.overflow = false; + out.graph = NULL; + if ( !TableJsonWriteValue( out, value, info, 0 ) ) { return -1; } + // THE CANONICAL TEXT ENDS WITH EXACTLY ONE NEWLINE (docs/SPEC-TABLES.md + // §16.1). Every writer emits it — this walk, the C# walk and + // "schema unpack" — and every reader accepts a text with or without one, + // because the trailing whitespace a read already skips is what makes the + // two the same text. It is a byte of the FORM rather than a file + // convention: a text that is written to a file, pasted into a diff and + // handed back through a pipe has to be one text in all three places, and a + // buffer whose last byte is a closing brace is the one shape that is not. + out.put( '\n' ); + if ( out.overflow ) { return -1; } + return out.offset; +} + +// ---- json walk: end ---- + +// ---- json graph walk: begin ---- +// +// THE VARIABLE CLASS's half of the text form (docs/SPEC-TABLES.md §16.7). The +// walk above places every kind but one; this defines the three adapters it +// calls for that one, and the two entry points a pointered table's wrappers +// name. The text is the fixed class's — a pointee is an object in place — and a +// node named more than once carries `&node`: defined once, with its fields, +// and referenced after by `{ "&node": N }` alone. + +// ---- the identity map ---- +// +// ONE map shape serves both directions. Writing keys it by a node's ADDRESS and +// counts the slots that name the node, so the second pass knows at a node's +// first occurrence whether it will be named again; reading keys it by the +// text's own label and answers the node it defined. Open addressing, a +// multiply-shift hash and quadrupling growth — TablePackMap's shape (§6.2), on +// the same terms: proportional to nodes, never to bytes, on the authoring +// side, and released before the call returns. + +struct TableJsonGraphEntry +{ + uint64_t key; // a node's address (write) or a label (read); 0 is an empty slot + int64_t count; // write: how many slots name this node + int64_t label; // write: the `&node` label assigned at its first write, 0 until then + uint8_t open; // the descent is still open: a reference here is a cycle (write), a self-reference (read) + uint32_t node; // read: the node's arena offset; 0 for a definition the reader dropped + const TableTypeInfo * type; // read: the node's table; NULL for a dropped one +}; + +struct TableJsonGraphMap +{ + TableJsonGraphEntry * entries; + int64_t capacity; // a power of two, or zero while empty + int64_t count; + TableAllocator allocator; // the caller's pair (§6.5): the builder's on read, the one handed to ToJson on write +}; + +inline void TableJsonGraphMapInit( TableJsonGraphMap & map, TableAllocator allocator ) +{ + map.entries = NULL; + map.capacity = 0; + map.count = 0; + map.allocator = allocator; +} + +inline void TableJsonGraphMapShutdown( TableJsonGraphMap & map ) +{ + map.allocator.free( map.allocator.context, map.entries ); + TableJsonGraphMapInit( map, map.allocator ); +} + +inline int64_t TableJsonGraphMapSlot( const TableJsonGraphMap & map, uint64_t key ) +{ + uint64_t hash = key * 0x9E3779B97F4A7C15ull; + hash ^= hash >> 29; + int64_t mask = map.capacity - 1; + int64_t at = (int64_t) ( hash & (uint64_t) mask ); + while ( map.entries[at].key != 0 && map.entries[at].key != key ) + { + at = ( at + 1 ) & mask; + } + return at; +} + +inline TableJsonGraphEntry * TableJsonGraphMapFind( TableJsonGraphMap & map, uint64_t key ) +{ + if ( map.capacity == 0 ) { return NULL; } + TableJsonGraphEntry * entry = &map.entries[ TableJsonGraphMapSlot( map, key ) ]; + return entry->key == key ? entry : NULL; +} + +inline bool TableJsonGraphMapGrow( TableJsonGraphMap & map ) +{ + TableJsonGraphMap grown; + grown.allocator = map.allocator; + grown.capacity = map.capacity != 0 ? map.capacity * 4 : 64; + grown.count = 0; + grown.entries = (TableJsonGraphEntry *) map.allocator.alloc( map.allocator.context, grown.capacity * (int64_t) sizeof( TableJsonGraphEntry ) ); // zeroed, by the pair's contract + if ( grown.entries == NULL ) { return false; } + for ( int64_t i = 0; i < map.capacity; i++ ) + { + if ( map.entries[i].key == 0 ) { continue; } + grown.entries[ TableJsonGraphMapSlot( grown, map.entries[i].key ) ] = map.entries[i]; + grown.count++; + } + map.allocator.free( map.allocator.context, map.entries ); + map = grown; + return true; +} + +// the entry for a key, made if it was not there; `taken` says which. NULL is the +// allocator refusing, and the walk refuses with it. +inline TableJsonGraphEntry * TableJsonGraphMapReach( TableJsonGraphMap & map, uint64_t key, bool & taken ) +{ + if ( ( map.count + 1 ) * 4 >= map.capacity * 3 ) // keep the load factor under three quarters + { + if ( !TableJsonGraphMapGrow( map ) ) { return NULL; } + } + TableJsonGraphEntry * entry = &map.entries[ TableJsonGraphMapSlot( map, key ) ]; + taken = entry->key != key; + if ( taken ) + { + entry->key = key; + map.count++; + } + return entry; +} + +// ---- reading: into a builder ---- + +struct TableJsonGraphIn +{ + TableWorker * worker; // where every node comes from + TableJsonGraphMap labels; // a label -> the node it defined +}; + +// `&node`'s value, the LABEL: a positive integer spelled as one — digits, no sign, no +// fraction, no exponent, no leading zero (§16.7). Anything else is malformed. +inline bool TableJsonScanLabel( TableJsonIn & in, uint64_t & label ) +{ + TableJsonSpace( in ); + if ( in.pos >= in.size || in.text[in.pos] < '1' || in.text[in.pos] > '9' ) + { + in.report->malformed = true; + in.bad = true; + return false; + } + uint64_t value = 0; + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) + { + uint64_t digit = (uint64_t) ( in.text[in.pos] - '0' ); + if ( value > ( UINT64_MAX - digit ) / 10 ) + { + in.report->malformed = true; + in.bad = true; + return false; + } + value = value * 10 + digit; + in.pos++; + } + label = value; + return true; +} + +// A BYTE BUFFER's text (docs/SPEC-TABLES.md §2.5, §16.2): a string. For a +// *string the string's bytes become the blob; for a *bytes the string is base64 +// and its decoded bytes do. The blob is allocated at EXACTLY the decoded +// length — the string is scanned once without keeping it to learn the length, +// and once into the node — so a blob of any size reads with no window and no +// bound to clamp against. A *bytes body that is not base64 is the wrong shape +// for the kind: the reference stays null and the event is counted. +inline bool TableJsonReadBlob( TableJsonIn & in, void * slot, const TableFieldInfo * f ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + TableRef * ref = (TableRef *) slot; + ref->value = 0; + if ( strcmp( f->type_name, "string" ) == 0 ) + { + const int64_t mark = in.pos; + int32_t length = 0; + if ( !TableJsonScanString( in, NULL, 0, &length ) ) { return false; } + in.pos = mark; + char * data = TableStringEmplace( *graph->worker, *ref, NULL, (int64_t) length ); + if ( data == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + int32_t placed = 0; + return TableJsonScanString( in, data, length, &placed ); + } + // base64: the alphabet characters decide the length, six bits apiece + const char * alphabet = TableJsonBase64Alphabet(); + const int64_t mark = in.pos + 1; + int64_t symbols = 0; + bool malformed = false; + in.pos++; + for ( ;; ) + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos++]; + if ( c == '"' ) { break; } + if ( c == '=' || malformed ) { continue; } + if ( c == 0 || strchr( alphabet, c ) == NULL ) { malformed = true; continue; } + symbols++; + } + if ( malformed ) + { + in.report->kind_mismatch++; + return true; + } + const int64_t length = ( symbols * 6 ) / 8; + uint8_t * data = TableBytesEmplace( *graph->worker, *ref, length ); + if ( data == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + int64_t placed = 0; + uint32_t accumulator = 0; + int32_t held = 0; + for ( int64_t at = mark; ; at++ ) + { + char c = in.text[at]; + if ( c == '"' ) { break; } + const char * symbol = c != '=' ? strchr( alphabet, c ) : NULL; + if ( symbol == NULL ) { continue; } + accumulator = ( accumulator << 6 ) | (uint32_t) ( symbol - alphabet ); + held += 6; + if ( held >= 8 ) + { + held -= 8; + if ( placed < length ) { data[placed++] = (uint8_t) ( ( accumulator >> held ) & 0xff ); } + } + } + return true; +} + +// A pointer's object. Its FIRST key decides what it is: `&node` naming a label not +// yet defined, with fields after it, is a DEFINITION; `&node` naming one already +// defined, alone, is a REFERENCE; any other key is a node named once, its +// object in place. The node comes from the +// builder's arena, and the slot holds its arena offset (§6.3). A pointer whose +// target is a BYTE BUFFER — no table — takes a string instead (§2.5). +inline bool TableJsonReadPointer( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( f->table == NULL ) { return TableJsonReadBlob( in, slot, f ); } + // the pointee nests one level down, exactly as a by-value table does, and + // takes the same cap: a chain nests as deep as it is long (§16.7) + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + char c = TableJsonPeek( in ); + if ( c == '}' ) + { + // an empty object: a node at its defaults, named once + in.pos++; + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + return true; + } + if ( c == 0 ) { in.bad = true; return false; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + if ( strcmp( key, "&node" ) != 0 ) + { + // a node named once: the pointee's object in place, and this key is + // its first field — unless it is the reserved prefix under a spelling + // this form does not have, which ReadTableKeys refuses + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } + return TableJsonReadTableKeys( in, node, f->table, depth + 1, key ); + } + uint64_t label = 0; + if ( !TableJsonScanLabel( in, label ) ) { return false; } + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph->labels, label, taken ); + if ( entry == NULL ) { in.report->malformed = true; in.bad = true; return false; } + // ONE SPELLING, and what follows the label says which half it is: fields + // after a label the text has not defined DEFINE it, and a label alone that + // the text has defined REFERS to it. The other two are malformed — a label + // alone that the text never defined, which would otherwise read as a default + // node under a silent report, and a field after a label already defined, + // which would be a second definition. That is what keeps a typo loud. + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; c = TableJsonPeek( in ); } + bool bare = c == '}'; + if ( bare == taken ) { in.report->malformed = true; in.bad = true; return false; } + if ( bare ) + { + // A REFERENCE. A label is defined when its object CLOSES, so a + // reference met inside its own definition — at any depth of by-value + // nesting — names a node whose descent is still open: the cycle the + // wire refuses (§3.1), refused here where it is written. A definition + // the reader dropped names no node, so the slot stays null with + // nothing more counted — the drop was counted where it happened. A + // node of another table than the slot declares is a kind mismatch, as + // on the wire. + in.pos++; + if ( entry->open != 0 ) { in.report->malformed = true; in.bad = true; return false; } + TableRef ref; + if ( entry->type == NULL ) + { + memcpy( slot, &ref, sizeof( ref ) ); + return true; + } + if ( entry->type != f->table ) + { + memcpy( slot, &ref, sizeof( ref ) ); + in.report->kind_mismatch++; + return true; + } + ref.value = (int64_t) entry->node; + memcpy( slot, &ref, sizeof( ref ) ); + return true; + } + // A DEFINITION: the node is allocated, the label is its, and the keys after + // `&node` are its fields. The entry is OPEN until the object closes, so a + // reference to the label from inside the node's own fields is refused as + // the cycle it is; the node and its table are filled in at the close. + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } + entry->open = 1; + if ( !TableJsonReadTableKeys( in, node, f->table, depth + 1, NULL ) ) { return false; } + entry = TableJsonGraphMapFind( graph->labels, label ); // the map may have grown under the descent + if ( entry == NULL ) { in.report->malformed = true; in.bad = true; return false; } + TableRef ref; + memcpy( &ref, slot, sizeof( ref ) ); + entry->node = (uint32_t) ref.value; + entry->type = f->table; + entry->open = 0; + return true; +} + +// An `&`-prefixed key opening an object the walk is SKIPPING — a value past an +// array's bound, an unknown key's value, a value of the wrong shape. A +// definition in there still takes its label, so the numbering survives whatever +// the storage could not hold (§16.7): the label is registered with no node, and a +// reference to it reads null. Any other prefixed key is the reserved prefix +// out of place. +inline bool TableJsonSkippedAmpersand( TableJsonIn & in, const char * key, int32_t ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL || strcmp( key, "&node" ) != 0 ) { in.report->malformed = true; in.bad = true; return false; } + uint64_t label = 0; + if ( !TableJsonScanLabel( in, label ) ) { return false; } + bool taken = false; + if ( TableJsonGraphMapReach( graph->labels, label, taken ) == NULL ) { in.report->malformed = true; in.bad = true; return false; } + return true; // a fresh entry is node 0, type NULL: a definition with no node +} + +// ---- writing: from a region's const root ---- + +struct TableJsonGraphOut +{ + TableJsonGraphMap nodes; // a node's address -> how many slots name it, and its `&node` once assigned + bool counting; // PASS ONE: count the references, refuse a cycle, emit nothing + int64_t next_label; +}; + +// The node a slot names: null as `null`, a node named once as its object in +// place, and a node named more than once under the construct. Which of the +// last two it is was learned in pass one; pass two spells it. +inline bool TableJsonWritePointer( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphOut * graph = (TableJsonGraphOut *) out.graph; + if ( graph == NULL ) { return false; } + const void * node = f->resolve( slot ); + if ( node == NULL ) + { + out.raw( "null", 4 ); + return true; + } + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph->nodes, (uint64_t) (uintptr_t) node, taken ); + if ( entry == NULL ) { return false; } + if ( f->table == NULL ) + { + // A BYTE BUFFER (§2.5, §16.7): its text is a string, which has no + // first key to carry `&node`, so a blob named from more than one + // slot has no spelling this form can carry and the graph is refused — + // as a shared node with nothing to write is. A blob named once is its + // bytes in place: base64 for a *bytes, the string itself for a *string. + if ( graph->counting ) { entry->count++; return true; } + if ( entry->count > 1 ) { return false; } + const TableBlob * blob = (const TableBlob *) node; + if ( blob->length > (uint32_t) 0x7fffffff ) { return false; } + if ( strcmp( f->type_name, "string" ) == 0 ) { TableJsonWriteString( out, (const char *) ( blob + 1 ), (int32_t) blob->length ); } + else { TableJsonWriteBase64( out, (const uint8_t *) ( blob + 1 ), (int32_t) blob->length ); } + return true; + } + if ( graph->counting ) + { + // PASS ONE: one visit per node, every slot that names it counted, and + // a reference to a node whose descent is still open is a cycle — + // refused here as the wire refuses it (§3.1) + entry->count++; + if ( !taken ) { return entry->open == 0; } + entry->open = 1; + if ( !TableJsonWriteValue( out, node, f->table, depth ) ) { return false; } + entry = TableJsonGraphMapFind( graph->nodes, (uint64_t) (uintptr_t) node ); // the map may have grown under the descent + if ( entry == NULL ) { return false; } + entry->open = 0; + return true; + } + // PASS TWO: a node named once is its object in place; a node named more + // than once is DEFINED at its first occurrence — `&node` first, then its + // fields — and REFERENCED by `&node` alone after that, spelled the same way at + // every site. Labels run from 1 in first-write order and are the text's own, + // so a stray number in a hand-edited text is most often one never defined. + if ( entry->count <= 1 ) + { + return TableJsonWriteValue( out, node, f->table, depth ); + } + if ( depth > kTableJsonMaxDepth ) { return false; } + if ( entry->label != 0 ) + { + out.put( '{' ); + out.line( depth + 1 ); + out.raw( "\"&node\": ", 9 ); + TableJsonWriteUnsigned( out, (uint64_t) entry->label ); + out.line( depth ); + out.put( '}' ); + return true; + } + entry->label = ++graph->next_label; + out.put( '{' ); + out.line( depth + 1 ); + out.raw( "\"&node\": ", 9 ); + TableJsonWriteUnsigned( out, (uint64_t) entry->label ); + bool any = true; + int64_t before = out.offset; + if ( !TableJsonWriteFields( out, node, f->table, depth, any ) ) { return false; } + // a definition carries at least one field, because a label alone is a + // reference: a shared node with nothing to write has no definition this + // form can spell, and the writer refuses it as it refuses any value it + // cannot spell (§16.3) + if ( out.offset == before ) { return false; } + out.line( depth ); + out.put( '}' ); + return true; +} + +// ---- the two entry points a pointered table's wrappers name ---- + +// The text into the builder's root. Every node the text names is allocated in +// the builder's arena through the field's own Emplace; the label map is the +// walk's, released before this returns. The root itself takes no label — nothing +// may name it (§16.7) — so an `&node` at the root is refused like any other key +// of the prefix. +inline bool TableJsonReadGraph( TableWorker & worker, void * root, const TableTypeInfo * info, const char * text, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + if ( worker.arena == NULL ) { if ( report != NULL ) { report->malformed = true; } return false; } + TableJsonGraphIn graph; + graph.worker = &worker; + TableJsonGraphMapInit( graph.labels, worker.arena->allocator ); + TableJsonIn in; + in.text = text; + in.size = bytes; + in.pos = 0; + in.report = report != NULL ? report : &ignored; + in.bad = false; + in.graph = &graph; + info->reset( root ); + if ( text == NULL || bytes < 0 ) + { + in.report->malformed = true; + return false; + } + bool ok = TableJsonReadTable( in, root, info, 0 ); + if ( ok ) + { + TableJsonSpace( in ); + if ( in.pos != in.size ) { in.bad = true; } // trailing rubbish is not one text + } + TableJsonGraphMapShutdown( graph.labels ); + if ( in.bad || !ok ) + { + in.report->malformed = true; + return false; + } + return true; +} + +// The text of a region's const root: measured when the buffer is NULL, written +// when it is not, over one code path. Two passes over one walk — the first +// counts how many slots name each node and refuses a cycle, the second writes +// — so a node's first occurrence knows whether it will be named again. The +// ROOT's entry is open for the whole first pass, so a reference back at it is +// the cycle it is (§3.1), and it takes no label. +inline int64_t TableJsonWriteGraph( const void * root, const TableTypeInfo * info, char * buffer, int64_t capacity, TableAllocator allocator ) +{ + if ( root == NULL ) { return -1; } + TableJsonGraphOut graph; + TableJsonGraphMapInit( graph.nodes, allocator ); + graph.counting = true; + graph.next_label = 0; + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph.nodes, (uint64_t) (uintptr_t) root, taken ); + if ( entry == NULL ) { TableJsonGraphMapShutdown( graph.nodes ); return -1; } + entry->open = 1; + TableJsonOut count; + count.buffer = NULL; + count.capacity = 0; + count.offset = 0; + count.overflow = false; + count.graph = &graph; + bool ok = TableJsonWriteValue( count, root, info, 0 ); + graph.counting = false; + TableJsonOut out; + out.buffer = buffer; + out.capacity = capacity; + out.offset = 0; + out.overflow = false; + out.graph = &graph; + if ( ok ) { ok = TableJsonWriteValue( out, root, info, 0 ); } + TableJsonGraphMapShutdown( graph.nodes ); + if ( !ok ) { return -1; } + out.put( '\n' ); // the canonical text ends with exactly one newline (§16.1) + if ( out.overflow ) { return -1; } + return out.offset; +} + +// ---- json graph walk: end ---- + +// ---- the out-of-line array's slot (docs/SPEC-TABLES.md §8.1) ---- + +inline int32_t TableJsonExtentCount( const void * slot ) +{ + int32_t count = 0; + memcpy( &count, (const uint8_t *) slot + 8, sizeof( count ) ); + return count < 0 ? 0 : count; +} + +inline const uint8_t * TableJsonExtentElements( const void * slot ) +{ + int64_t delta = 0; + memcpy( &delta, slot, sizeof( delta ) ); + return delta != 0 ? (const uint8_t *) slot + delta : NULL; +} + +// ---- json map walk: begin ---- + +inline bool TableJsonIsMap( const TableFieldInfo * f ) +{ + return f->is_array && f->array_bound == 0 && strncmp( f->type_name, "map[", 4 ) == 0; +} + +// the entry's two rows: fields[0] IS the key and fields[1] IS the value, which +// is what makes a user's own table of pairs the same bytes (§2.8) +inline const TableFieldInfo * TableJsonMapKeyField( const TableFieldInfo * f ) { return &f->table->fields[0]; } +inline const TableFieldInfo * TableJsonMapValueField( const TableFieldInfo * f ) { return &f->table->fields[1]; } + +inline bool TableJsonMapKeyIsString( const TableFieldInfo * key ) { return key->kind == 12; } +inline bool TableJsonMapKeySigned( const TableFieldInfo * key ) { return key->kind >= 2 && key->kind <= 5; } + +// AN INTEGER KEY IS THE INTEGER'S DECIMAL SPELLING, QUOTED, because a JSON +// object's keys are strings. Written digit by digit so no locale can move it. +inline void TableJsonWriteMapIntegerKey( TableJsonOut & out, const void * storage, const TableFieldInfo * key ) +{ + uint64_t magnitude = 0; + bool negative = false; + if ( TableJsonMapKeySigned( key ) ) + { + int64_t value = 0; + switch ( key->kind ) + { + case 2: value = (int64_t) *(const int8_t *) storage; break; + case 3: value = (int64_t) *(const int16_t *) storage; break; + case 4: value = (int64_t) *(const int32_t *) storage; break; + default: value = *(const int64_t *) storage; break; + } + negative = value < 0; + magnitude = negative ? ( ~(uint64_t) value ) + 1 : (uint64_t) value; + } + else + { + switch ( key->kind ) + { + case 6: magnitude = (uint64_t) *(const uint8_t *) storage; break; + case 7: magnitude = (uint64_t) *(const uint16_t *) storage; break; + case 8: magnitude = (uint64_t) *(const uint32_t *) storage; break; + default: magnitude = *(const uint64_t *) storage; break; + } + } + char digits[24]; + int32_t at = (int32_t) sizeof( digits ); + do { digits[--at] = (char) ( '0' + ( magnitude % 10 ) ); magnitude /= 10; } while ( magnitude != 0 ); + if ( negative ) { digits[--at] = '-'; } + TableJsonWriteString( out, digits + at, (int32_t) sizeof( digits ) - at ); +} + +inline void TableJsonWriteMapKey( TableJsonOut & out, const void * entry, const TableFieldInfo * key ) +{ + const uint8_t * storage = (const uint8_t *) entry + key->offset; + if ( TableJsonMapKeyIsString( key ) ) + { + // A STRING KEY IS THE STRING (§2.8): every JSON key of a map object is + // a KEY OF THE MAP and none is a field key, so the `&` prefix §16.7 + // reserves for field keys is ordinary data here. + TableJsonWriteString( out, (const char *) storage, *(const int32_t *) ( (const uint8_t *) entry + key->count_offset ) ); + return; + } + TableJsonWriteMapIntegerKey( out, (const void *) storage, key ); +} + +// ToJson WRITES ENTRIES IN ASCENDING KEY ORDER, so unpack then pack is +// byte-stable and a diff of two texts is a diff of two maps (§2.8, §17.2). +// A region holds them in that order already, so this is the array in place. +inline bool TableJsonWriteMap( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + const int32_t count = TableJsonExtentCount( slot ); + if ( count == 0 ) { out.raw( "{}", 2 ); return true; } + const TableFieldInfo * key = TableJsonMapKeyField( f ); + const TableFieldInfo * value = TableJsonMapValueField( f ); + const uint8_t * entries = TableJsonExtentElements( slot ); + out.put( '{' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + const void * entry = (const void *) ( entries + (int64_t) i * f->elem_size ); + TableJsonWriteMapKey( out, entry, key ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteField( out, entry, value, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( '}' ); + return true; +} + +// AN INTEGER KEY IS READ BY §16.2's INTEGER RULE AND BY NOTHING ELSE, so +// "2.0" and "1e3" are the integers 2 and 1000 and "-0" is zero. The token is +// walked as a JSON number over its own bytes; a token that rule calls +// malformed makes the KEY malformed, and a genuinely fractional value, or one +// outside the key kind's range, is kind_mismatch for that entry. +// +// THE KEY IS THE SPELLING AND NOTHING AROUND IT. The number walk steps over +// leading whitespace and comments, which is right BETWEEN tokens and wrong +// INSIDE one: a key is an identity, and a padded spelling that resolved to the +// same integer would be a second name for one entry. So the walk must begin at +// the token's first byte, and a token with anything before the number is not a +// JSON number at all, which is malformed on the terms "1-2" is. +inline bool TableJsonMapKeyValue( const char * token, int32_t length, const TableFieldInfo * key, + int64_t & value, bool & fits ) +{ + fits = false; + TableReport scratch; + TableJsonIn probe = { token, (int64_t) length, 0, &scratch, false, NULL }; + bool integral = false; + TableJsonSpace( probe ); + if ( probe.pos != 0 ) { return false; } // whitespace is never part of a key + if ( !TableJsonWalkNumber( probe, &integral ) ) { return false; } + if ( probe.pos != (int64_t) length ) { return false; } // trailing bytes: not a number + // A MAP KEY'S POLICY over the one interpreted value: it REJECTS THE WHOLE + // ENTRY. A key is an identity, so a clamped one is two entries merged, and + // a value the key kind does not hold is kind_mismatch for that entry, + // dropped and counted, never clamped. + const TableJsonInteger number = TableJsonInterpretExact( token, length ); + if ( number.fractional || number.saturated ) { return true; } + bool moved = false; + value = TableJsonIntegerInDomain( number, TableJsonMapKeySigned( key ), (int32_t) key->elem_size, moved ); + fits = !moved; + return true; +} + +// FromJson READS KEYS IN WHATEVER ORDER THE TEXT GIVES THEM. A repeated key is +// last-wins and counted duplicate, the object rule (§16.2) applied inside the +// map. An empty object is an empty map, and null is kind_mismatch. +inline bool TableJsonReadMap( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; + const TableFieldInfo * key = TableJsonMapKeyField( f ); + const TableFieldInfo * value = TableJsonMapValueField( f ); + const char shape = TableJsonShape( value ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + char token[kTableJsonMaxKey]; + int32_t token_length = 0; + bool key_over = false; // longer than THIS buffer: never truncated into a key + if ( !TableJsonScanString( in, token, kTableJsonMaxKey - 1, &token_length, &key_over ) ) { return false; } + token[token_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t key_value = 0; + bool place = true; + if ( !TableJsonMapKeyIsString( key ) ) + { + // AN INTEGER KEY PAST THIS SCAN'S BUFFER DROPS AS kind_mismatch, + // here and in the tool's walker. The bytes kept are a PREFIX, and a + // prefix is a different token, so the entry drops rather than a + // truncation being read as a value. The length alone does not + // settle it: a token that long can still spell a number an integer + // kind holds, "1" padded by an exponent of zeroes for one, and this + // read declines to find out. + bool fits = false; + if ( key_over ) { in.report->kind_mismatch++; place = false; } + else if ( !TableJsonMapKeyValue( token, token_length, key, key_value, fits ) ) + { + // A MALFORMED KEY STOPS THE READ where §16.1's rule stops it, + // with the instance holding what was placed before the stop. + in.report->malformed = true; + in.bad = true; + return false; + } + else if ( !fits ) { in.report->kind_mismatch++; place = false; } + } + else if ( key_over || token_length > key->array_bound ) + { + // A KEY LONGER THAN N DROPS ITS ENTRY AND COUNTS clamped, the + // wire's rule, because a clamped key is a merged entry (§2.8). The + // BOUND IS THE WALKER'S, tested here against the key field's own + // descriptor, so placement is left with one failure to report. A + // key past this scan's own buffer is the SAME event, because a + // truncated key is the merged entry the rule exists to prevent. + in.report->clamped++; + place = false; + } + const int32_t before = TableJsonExtentCount( (const void *) slot ); + void * entry = place ? f->place( *graph->worker, slot, token, token_length, key_value ) : NULL; + if ( place && entry == NULL ) + { + // AN ALLOCATION FAILURE IS NOT AN OVERSIZED KEY (§2.8, §16.1). The + // key was checked above, so the arena is what refused, and the read + // stops where the list, blob and pointer paths stop on one rather + // than handing back an instance short of entries the text spelled + // and calling itself clean. + in.report->malformed = true; + in.bad = true; + return false; + } + else if ( entry != NULL && TableJsonExtentCount( (const void *) slot ) == before ) + { + in.report->duplicate++; // last-wins, the object rule inside the map + } + const char got = TableJsonValueShape( in ); + if ( entry == NULL ) + { + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( value->kind == 17 && !value->is_array ) + { + // A POINTER VALUE IS SHARED EXACTLY AS A POINTER FIELD IS (§2.8): + // null is a null slot, an object is the pointee in place or an + // &node reference to one (§16.7), anything else is the wrong shape — + // the same three the field-key loop gives a pointer field, because + // an entry's value IS a field line. + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) entry + value->offset, value->elem_size, 0 ); + } + else if ( got != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, (uint8_t *) entry + value->offset, value, depth + 1 ) ) + { + return false; + } + } + else if ( got != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadField( in, entry, value, depth + 1 ) ) + { + return false; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; +} + +// ---- json map walk: end ---- + +// ---- json list walk: begin ---- + +// an unbounded array is the out-of-line array that is not a map (§8.1) +inline bool TableJsonIsList( const TableFieldInfo * f ) +{ + return f->is_array && f->array_bound == 0 && !TableJsonIsMap( f ); +} + +// ToJson WRITES THE ELEMENTS IN INDEX ORDER, which is the only order there is, +// so unpack then pack is byte-stable without a rule of its own (§2.9, §17.2). +// A region holds the array in place, so this steps it at the descriptor's pitch. +inline bool TableJsonWriteList( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + const int32_t count = TableJsonExtentCount( slot ); + if ( count == 0 ) { out.raw( "[]", 2 ); return true; } + const uint8_t * elements = TableJsonExtentElements( slot ); + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + const uint8_t * element = elements + (int64_t) i * f->elem_size; + if ( f->kind == 17 ) + { + // a []*T's elements take the pointer row (§16.7): the pointee's + // object in place, null, or `&node` for a shared one + if ( !TableJsonWritePointer( out, element, f, depth + 1 ) ) { return false; } + } + else if ( !TableJsonWriteScalar( out, element, f, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( ']' ); + return true; +} + +// FromJson READS EVERY ELEMENT THE TEXT CARRIES, appending each through the +// descriptor's place resolver: `[]` is an empty list, and null is +// kind_mismatch, the array row's own rule (§16.2). LAST WINS holds for a +// repeated key: the list goes back to EMPTY before this occurrence's elements +// land, the builder's storage being reclaimed at reset (§2.9). +inline bool TableJsonReadList( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; + TableJsonSetRaw( (uint8_t *) slot, 8, 0 ); + TableJsonSetRaw( (uint8_t *) slot + 8, 4, 0 ); + const char shape = TableJsonElementShape( f ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + void * element = f->place( *graph->worker, slot, NULL, 0, 0 ); + if ( element == NULL ) + { + // NOT ADDED: the arena could not carve another segment, or the + // count met the int32 cap. The text cannot be placed whole, and + // the read stops where §16.1's rule stops it. + in.report->malformed = true; + in.bad = true; + return false; + } + if ( f->kind == 17 ) + { + // an element of a []*T (§2.9): null is a null slot, an object is the + // pointee in place or an `&node` reference (§16.7) + char got = TableJsonValueShape( in ); + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) element, f->elem_size, 0 ); + } + else if ( got != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, element, f, depth + 1 ) ) { return false; } + } + else if ( TableJsonValueShape( in ) != shape ) + { + // the wrong shape for the element kind: the slot keeps its + // defaults and the event counts, the array row's rule (§16.2) + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadScalar( in, element, f, depth + 1 ) ) { return false; } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; +} + +// ---- json list walk: end ---- + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_JSON + +namespace mapdemo { + +bool PairsFromJson( PairsBuilder & builder, const char * text, int64_t bytes, TableReport * report ) +{ + Pairs * root = builder.GetRoot(); + if ( root == NULL ) { if ( report != NULL ) { report->malformed = true; } return false; } // locked, or the root allocation failed + return TableJsonReadGraph( builder.main, root, PairsTableType(), text, bytes, report ); +} + +int64_t PairsToJsonMeasure( const Pairs * root, TableAllocator allocator ) +{ + return TableJsonWriteGraph( root, PairsTableType(), NULL, 0, allocator ); +} + +int64_t PairsToJson( const Pairs * root, char * buffer, int64_t capacity, TableAllocator allocator ) +{ + return TableJsonWriteGraph( root, PairsTableType(), buffer, capacity, allocator ); +} + +} // namespace mapdemo diff --git a/testdata/golden/tables/maps/PairsTable.h b/testdata/golden/tables/maps/PairsTable.h new file mode 100644 index 000000000..b9b102c4d --- /dev/null +++ b/testdata/golden/tables/maps/PairsTable.h @@ -0,0 +1,9471 @@ +// Code generated by the schema compiler from Pairs.schema. DO NOT EDIT. +// SPDX-License-Identifier: NONE — this generated output is yours, under terms of +// your choice. See the LICENSE exception in the schema compiler; the compiler is +// AGPL-3.0, its output is not. +// package mapdemo — protocol id 0x1ac124decde5b2aa (packets only: tables version by field id, not by protocol id) +// The TABLE wire (evolution-tolerant, docs/SPEC-TABLES.md): no serialize +// dependency — includable from any TU. + +#pragma once + +#include +#include // the prefill's scalar-array fills +#include // offsetof, for the reflection descriptors + +// ---- the hooks (docs/USAGE.md, "the C++ table runtime's hooks") ---- +// +// schema_assert — the runtime's own assert, and the refusal a debugger reads. +// NDEBUG removes it, exactly as it removes assert. A caller who already routes +// serialize's asserts writes `#define schema_assert serialize_assert` before +// including this header and both halves land in one handler. +#ifndef schema_assert +#include +#define schema_assert assert +#endif // #ifndef schema_assert + +// schema_fatal — what stands after the assert on a path that cannot continue. +// NDEBUG does not remove it. Supply it and is never included. +#ifndef schema_fatal +#include // abort +#define schema_fatal abort +#endif // #ifndef schema_fatal + +// schema_allocate / schema_release — what "no allocator handed in" means for +// this program. schema_allocate hands back ZEROED bytes and NULL on failure: +// an arena segment is copied whole, padding included, so anything left +// uninitialized here would reach a packed region. Supply both and +// is never included; hand a TableAllocator to a builder to route one +// structure's allocations somewhere else again. +#ifndef schema_allocate +#include // calloc, free +#define schema_allocate( bytes ) calloc( (size_t) 1, (size_t) ( bytes ) ) +#define schema_release( pointer ) free( pointer ) +#endif // #ifndef schema_allocate +#include // a node's lifetime starts in arena storage (placement new) +#include // one atomic per slab: the arena is lock-free by ownership + +#include "Pairs.h" +#include "FleetTable.h" + +#ifndef MAPDEMO_SCHEMA_TABLE_PRIMITIVES +#define MAPDEMO_SCHEMA_TABLE_PRIMITIVES + +// THE CODEC DOES NOT DEPEND ON THE COMPILER'S INLINING BUDGET. A table of a +// realistic field count emits one large body per type, and the cursor a body +// writes through lives in the caller's `TableWriter`: across a call boundary +// that cursor round-trips through memory, and a `uint8_t *` store may alias the +// writer itself, so every put reloads it. When a budget runs out mid-body the +// codec silently degrades to that shape. Forcing the primitives and the +// fixed-class bodies inline is what keeps the cursor in registers and lets +// adjacent constant framing bytes merge into one store. +#if defined( _MSC_VER ) +#define MAPDEMO_TABLE_INLINE __forceinline +#elif defined( __GNUC__ ) || defined( __clang__ ) +#define MAPDEMO_TABLE_INLINE inline __attribute__(( always_inline )) +#else +#define MAPDEMO_TABLE_INLINE inline +#endif + +namespace mapdemo { + +// WHY A READ WAS REFUSED, by name (docs/SPEC-TABLES.md §3.3, §11). A REFUSAL +// is not one of §4's events: nothing is decoded, no counter moves and no +// damage is reported, so five zero counters and a false flag are what a clean +// read prints too and only the verdict tells them apart. The reason says which +// refusal it was. +// +// This is the MESSAGE PATH's vocabulary and not the cooked form's (§7.4): a +// caller meeting one of these has been refused a MESSAGE on a connection, +// which is a different recovery with a different owner than a file a header +// match turned down. +enum TableMessageReason +{ + newer_form, // a FORM BYTE this reader does not carry (§3) + no_vocabulary, // no table for this connection: the message arrived before the announcement, or after a refused one + second_announcement, // a second announcement on a connection: it sets nothing, amends nothing, and the connection closes + vocabulary_too_large, // an announcement above the receiver's declared bound, refused before an entry is touched + message_form_as_file, // a form 2 wire where a FILE was expected: its table is somewhere else + batch_too_large // a batch of more than 256 bodies on the write side, or of more than the caller has room for on the read side: nothing is written or decoded, and the count says what the wire carries +}; + +// The table-wire read report — the permissive contract's ledger. Silence +// (all zero) means the data matched this reader's schema exactly. +struct TableReport +{ + int32_t unknown = 0; // unknown field ids skipped (newer data) + int32_t kind_mismatch = 0; // known id, changed type — skipped, never misdecoded + // a kind that GREW since the writer (docs/SPEC-TABLES.md §4): an integer + // kind read into a wider one of the same signedness, or f32 into f64, + // decoded EXACTLY. One count per field or per map. It is the one counter + // that names no loss: the bytes were not the shape this reader declares, + // and the number survived. + int32_t widened = 0; + int32_t clamped = 0; // out-of-range values clamped to declared bounds + // a key the TEXT form saw twice: last wins, and the repeat is counted + // (docs/SPEC-TABLES.md §16.2). The wire never raises it — a body carrying an + // id twice is legal input whose last occurrence wins, silently (§3). + int32_t duplicate = 0; + bool malformed = false; // framing damage; decode stopped, partial result kept + // THE REFUSAL VERDICT, which is not one of §4's events and moves no counter + // (docs/SPEC-TABLES.md §3): a FORM BYTE this reader does not carry. Five + // zero counters and a false flag are what a clean read prints too, so the + // verdict is what tells the two apart. + bool refused = false; + // WHICH refusal, and it is read only when refused is set: a read that + // was not refused has no reason, and this member is the one the caller + // must not look at then (docs/SPEC-TABLES.md §3.3). + TableMessageReason reason = newer_form; + // RETAIN-UNKNOWN's pair (docs/SPEC-TABLES.md §6.6), on the same struct for + // the reason duplicate is: a caller has one report type and not two. Both + // are ZERO in every read that did not opt in, and retention moves no + // counter above. A retained field still counts unknown, because unknown + // says what a READER could not name and that stays true. + int32_t retained = 0; // unknown fields whose bytes were kept + int32_t retain_lost = 0; // every unknown this load or save could not keep +}; + + +// WHY A FILE WAS REFUSED, by name (docs/SPEC-TABLES.md §6.5, §7, §19.2): the +// one vocabulary a cook's Open, a block's BlockOpen and a load measure's -1 +// share, because a caller asking "why can I not have this file" is +// asking one question whichever call refused it. The FIRST failing clause names the +// reason, in the order §7 enumerates, so one file answers one value in every +// language. A refusal moves no counter, and a match writes nothing: the +// out-parameter is touched on the refusal path only. +// +// It is not the MESSAGE FORM's vocabulary (TableMessageReason, §3.3): a caller +// meeting one of these has been refused a FILE, by a header match or by a +// measure. +enum TableRefuseReason +{ + ok, // no clause failed: the only value beside a non-null root (§7) + not_a_cook, // the magic is neither this build's constant nor its byte reversal, or the byte-order word contradicts the magic + foreign_order, // the magic byte-reversed: a cook of the other byte order (§7.1) + wrong_build_version, // the build_version word is not this build's (§20) + reserved_not_zero, // a reserved header word is not zero (§7.1) + bad_alignment, // the alignment word is not a power of two, is below eight, is above sixty-four, or is not a multiple of the root's own alignof + truncated, // the part lengths against the caller's length, or a data part too short to hold the root + unaligned_base, // the pointer the caller passed is not aligned for the region: the caller's defect, not the file's + bad_layout, // BlockOpen (§19.2): a pitch, a count, an offset or an extent that disagrees with this build's or leaves the block + unknown_form, // at a MEASURE (§3, §6.5): a form byte this build does not carry, refused before any read + count_over_length, // an array or map count whose elements cannot fit the field's own L (§2.8, §2.9) + count_over_extent_cap, // a count above the int32 extent cap (§2.2), which no region can hold whatever its size + blob_over_size_cap, // a blob whose length is past the derived-size cap (§3.1, §11) + data_cycle // a data cycle reached from a builder: the AUTHORING side's -1 (§3.1, §7.6) +}; +// ---- reflection (tables only, docs/SPEC-TABLES.md) ---- +// +// Static field descriptors for every type in the table closure: name, wire +// id/kind, storage offset, bounds, ranges, enum names and branch guards — +// enough to walk, print, diff, edit or bind any table value at runtime with +// no RTTI and no schema files. TableType() returns X's descriptor. + +struct TableTypeInfo; + +// One arm of a union field: where its payload sits inside the union's storage +// and what its payload looks like. The arm's NAME and its table-wire id come +// from the field's enum_name/variant_id functions at the same tag, so nothing +// is spelled twice (docs/SPEC-TABLES.md §8). +struct TableFieldInfo; + +struct TableUnionArmInfo +{ + uint32_t offset; // offsetof the arm's payload within the union storage + const TableTypeInfo * table; // the arm payload's descriptor, or NULL + // AN ARM IS A FIELD LINE (docs/SPEC-TABLES.md §2.6): an arm that names no + // declared type or table carries the FIELD descriptor a field of that + // type would carry instead — offsets taken within the union storage — so + // a generic walk meets an arm's kind, width, bounds and companions where + // it meets a field's. Exactly one of the two is non-NULL on a set arm. + const TableFieldInfo * field; + uint32_t size; // the arm's whole storage, which selection zero-establishes +}; + +// A union field's shape: the tag, and the arms indexed by it. Arms run +// [0, enum_max]; index 0 is the EMPTY arm and carries no payload. +struct TableUnionInfo +{ + uint32_t tag_offset; // offsetof the tag within the union storage + uint32_t tag_size; // sizeof the tag + const TableUnionArmInfo * arms; +}; + +// The exact raw range of a wide-kind field (docs/SPEC-TABLES.md §8.2): two 128-bit +// values as 64-bit lanes, low lane first, two's complement for the signed kinds. +struct TableWideRange +{ + uint64_t lo[2]; + uint64_t hi[2]; +}; + +// THE SHARED EMPTY DOC (docs/SPEC-TABLES.md §8.1): a declaration with no /// +// block carries a doc column pointing at this one object, so absence costs a +// unit no string data and a printer concatenates doc columns with no null +// test. One definition for the whole unit: every absent doc compares equal by +// address. +inline const char TableDocNone[1] = ""; + +// the arena's allocation front, defined with the variable-length runtime +// below; a descriptor names it only through a pointer parameter. +struct TableWorker; + +struct TableFieldInfo +{ + const char * name; // schema field name, e.g. "health" + const char * json; // the TEXT form's key: the json = "key" attribute, else name (§16.3) + const char * type_name; // schema type name, e.g. "float32", "Grade" + uint64_t id; // table-wire field id: fnv1a64 of the name, of the was alias after a rename (§5) + uint8_t kind; // table-wire kind; for arrays/strings/bytes, the ELEMENT kind + bool is_array; // fixed or counted array (bytes included) + bool is_pointer; // a *T pointer field: storage is an 8-byte TableRef; the target is a table + // THE TWO THE TEXT FORM NEEDS (docs/SPEC-TABLES.md §16.7), and they + // are here for the same reason is_pointer is: the walk is ONE walk + // over descriptors and cannot spell a target's own At or + // Emplace. `resolve` reads a slot in a REGION and answers the + // node it names, or NULL; `emplace` allocates one in a BUILDER's + // arena and points the slot at it. NULL on every field that is not + // a pointer, and emitted only in a unit that declares one. + const void * (*resolve)( const void * slot ); + void * (*emplace)( TableWorker & worker, void * slot ); + bool counted; // a _count/_length int32 companion exists (counted arrays, strings, bytes) + bool optional; // a ?T field: a _present bool companion decides whether it rides + int32_t array_bound; // array capacity / string max length; 0 for plain scalars + uint32_t offset; // offsetof the storage member + uint32_t elem_size; // sizeof the member (element size for arrays) + uint32_t count_offset; // offsetof the _count/_length companion, or 0xffffffff + uint32_t present_offset; // offsetof the _present companion, or 0xffffffff + const TableTypeInfo * table; // nested table's descriptor, or NULL + bool has_range; // a declared [min, max] (int or float) + double range_min; // NOTE: int64 ranges beyond 2^53 lose precision here + double range_max; + // the WIDE kinds (18-29, docs/SPEC-TABLES.md §3, §8.2): frac_bits is a fixed + // field's F — its storage holds units × 2^F — and wide is the declared + // range on that RAW scale, exact, as two 128-bit two's-complement values + // in 64-bit lanes (low lane first). NULL where the declaration bounds + // nothing (a bare uint128) and for every other kind; frac_bits is 0 for + // every kind that is not fixed-point. range_min/range_max still carry + // the declared bounds as doubles — whole units for a fixed field — for + // a walker that only shows them. + uint8_t frac_bits; + const TableWideRange * wide; + int64_t enum_max; // enums: highest valid value (None = 0 always valid); + // unions: the arm count (tag range [0, enum_max]); + // flags: the highest declared BIT INDEX; else -1 + // the vocabulary's names, indexed the same way enum_max bounds: an enum's + // value -> name, a union's tag -> arm name, a FLAGS field's bit index -> + // variant name. NULL for every other kind. + const char * (*enum_name)( uint64_t value ); + // the TABLE-WIRE id of one variant (docs/SPEC-TABLES.md §5): for an enum, the + // hash of the variant's name; for a union, the hash of the arm's name. + // 0 is the reserved id — an enum's None, a union's empty. NULL for every + // other kind — a FLAGS field's variants have no per-variant wire id (§4), + // so a NULL here beside a non-NULL enum_name is what says "flags". + // Walk [0, enum_max] to enumerate a vocabulary and its ids. + uint64_t (*variant_id)( uint64_t value ); + // an ENUM-KEYED array (docs/SPEC-TABLES.md §2.4): the array has one slot per + // variant of key_type_name, indexed by the variant's value, and its slots + // ride under variant ids rather than positions. key_name and key_id are + // the key's vocabulary — walk [0, array_bound) to print slots by name. + // NULL on every other field. + const char * key_type_name; + const char * (*key_name)( uint64_t value ); + uint64_t (*key_id)( uint64_t value ); + // union fields: the tag and its arms, behind a function so the whole + // descriptor stays CONSTANT-INITIALISED (a captureless lambda converts to + // a function pointer at compile time; the arms themselves are a static + // inside it). NULL for every other kind. + const TableUnionInfo * (*arms)(); + // an OUT-OF-LINE array (docs/SPEC-TABLES.md §8.1): place one element and + // hand it back at its defaults. A MAP places BY KEY, a string key comes + // in as the bytes and the length, an integer key as the value, and NULL + // is NOT INSERTED: a key past the bound, or an arena that could not carve + // another segment. A LIST ignores the key and APPENDS, NULL at the arena + // or the int32 cap. NULL on every field that is neither. + void * ( * place )( TableWorker & worker, void * slot, const char * key, int32_t key_length, int64_t key_value ); + const char * guard; // branch guard, e.g. "at_rest" or "!at_rest"; "" if unguarded + // what a PERSON wrote about the field (docs/SPEC-TABLES.md §8.1): the /// + // block above it, verbatim (SPEC §4.1). It is TableDocNone when there is + // none, never NULL. Its tags (SPEC §4.2) follow in declared order, and an + // untagged field is 0 beside NULL. Static, constant-initialized, + // allocating nothing. + const char * doc; + int32_t num_tags; + const char * const * tags; +}; + +struct TableTypeInfo +{ + const char * name; // schema type name + uint32_t size; // sizeof the storage struct + int32_t num_fields; + const TableFieldInfo * fields; + // put one instance back at its declared defaults, in place. A generic + // walker that fills a value has to be able to establish the defaults an + // absent field takes, and it holds no type to spell — this is the one + // thing the descriptors could not express without it. Placement-new + // value-init, exactly what the wire's read path does, and no temporary. + void (*reset)( void * storage ); + // the DERIVED mode (docs/SPEC-TABLES.md): false = fixed-size, a plain + // relocatable struct; true = variable-length, built through a Builder + // and read through a region root. Nobody declares it; the compiler + // works it out. + bool variable; + // the declaration's own doc and tags, on the same terms as a field's + // (docs/SPEC-TABLES.md §8.1) + const char * doc; + int32_t num_tags; + const char * const * tags; +}; + +struct TableWriter +{ + uint8_t * buffer; + int64_t capacity; + int64_t offset = 0; + bool overflow = false; + + // the parameters do not repeat the member names: a parameter that hides a + // member is a warning the estate's compilers disagree about (gcc's + // -Wshadow and cl's C4458 refuse it, clang's -Wshadow does not), and this + // is a header a consumer compiles under its OWN flags + TableWriter( uint8_t * to_buffer, int64_t to_capacity ) : buffer( to_buffer ), capacity( to_capacity ) {} + + MAPDEMO_TABLE_INLINE void raw( const void * data, int64_t bytes ) + { + if ( offset + bytes > capacity ) { overflow = true; return; } + memcpy( buffer + offset, data, (size_t) bytes ); + offset += bytes; + } + MAPDEMO_TABLE_INLINE void put8( uint8_t v ) { raw( &v, 1 ); } + MAPDEMO_TABLE_INLINE void put16( uint16_t v ) { uint8_t b[2] = { uint8_t( v ), uint8_t( v >> 8 ) }; raw( b, 2 ); } + MAPDEMO_TABLE_INLINE void put32( uint32_t v ) { uint8_t b[4] = { uint8_t( v ), uint8_t( v >> 8 ), uint8_t( v >> 16 ), uint8_t( v >> 24 ) }; raw( b, 4 ); } + MAPDEMO_TABLE_INLINE void put64( uint64_t v ) { put32( uint32_t( v ) ); put32( uint32_t( v >> 32 ) ); } + // a 128-bit value as two lanes, the low half first (docs/SPEC-TABLES.md §3) + MAPDEMO_TABLE_INLINE void put128( uint64_t lo, uint64_t hi ) { put64( lo ); put64( hi ); } + // EVERY LENGTH, COUNT, INDEX AND ID REFERENCE IS ONE CANONICAL UNSIGNED + // LEB128 (docs/SPEC-TABLES.md §3): seven value bits a byte, the lowest + // group first, the high bit set on every byte but the last. One value has + // one spelling, so two conforming writers agree byte for byte. + MAPDEMO_TABLE_INLINE void putleb( uint64_t v ) + { + while ( v >= 0x80 ) { put8( uint8_t( v ) | 0x80 ); v >>= 7; } + put8( uint8_t( v ) ); + } +}; + +// TableLebBytes is one value's spelling length, which a MEASURE needs before +// the bytes exist — the length of a body has to be known before it is written, +// because a length whose own width moves cannot be patched in place. +inline int64_t TableLebBytes( uint64_t v ) +{ + int64_t n = 1; + while ( v >= 0x80 ) { v >>= 7; n++; } + return n; +} + +// THE ID TABLE, WRITER SIDE (docs/SPEC-TABLES.md §3). It holds every id the +// body used, once each, in FIRST-USE order over the whole wire, and the body +// names them by position: reference k is the kth entry, counted from 1, and +// reference 0 names NO ID. +// +// Its capacity is a COMPILE-TIME fact of the unit — the distinct names its +// table closure can spell — so a save allocates nothing: the table is a local +// of Measure and of Save. The bucket chain makes ref constant time and makes +// truncate constant time too, which is what an ELIDED field needs: a field +// that turns out not to ride costs nothing in the id table either, so the walk +// interns its id, builds the payload that decides, and undoes the entry when +// nothing rides. +struct TableIds +{ + static const int32_t kCapacity = 76; + static const int32_t kBuckets = 256; + + uint64_t ids[ kCapacity ]; + int32_t chain[ kCapacity ]; + int32_t head[ kBuckets ]; + int32_t count; + bool overflow; + + TableIds() : count( 0 ), overflow( false ) + { + for ( int32_t i = 0; i < kBuckets; i++ ) { head[i] = -1; } + } + + static MAPDEMO_TABLE_INLINE uint32_t bucket_of( uint64_t id ) + { + return uint32_t( ( id * 0x9E3779B97F4A7C15ull ) >> 56 ) & uint32_t( kBuckets - 1 ); + } + + // the reference an id takes: the file's own first-use entry, appended on + // first use. The MESSAGE form names no id at all: its references are + // compile-time slots of the announced vocabulary (docs/SPEC-TABLES.md §3.3). + MAPDEMO_TABLE_INLINE uint64_t ref( uint64_t id ) + { + const uint32_t b = bucket_of( id ); + for ( int32_t i = head[b]; i >= 0; i = chain[i] ) + { + if ( ids[i] == id ) { return uint64_t( i ) + 1; } + } + if ( count >= kCapacity ) { overflow = true; return 1; } + ids[count] = id; chain[count] = head[b]; head[b] = count; count++; + return uint64_t( count ); + } + + // undo every entry appended since mark. An entry removed is the most + // recent one in its bucket, so it sits at that bucket's head. + void truncate( int32_t mark ) + { + while ( count > mark ) + { + count--; + head[ bucket_of( ids[count] ) ] = chain[count]; + } + } +}; + +// TableIdsBytes is the trailer's own size: the entries, each a fixed +// little-endian u64, and the ENTRY COUNT, the one fixed-width number on the +// wire (docs/SPEC-TABLES.md §3). +inline int64_t TableIdsBytes( const TableIds & ids ) { return int64_t( ids.count ) * 8 + 8; } + +// TableIdsWrite puts the trailer where the walk ended: a writer never patches, +// because first-use order is known only when the walk ends. +inline void TableIdsWrite( TableWriter & w, const TableIds & ids ) +{ + for ( int32_t i = 0; i < ids.count; i++ ) { w.put64( ids.ids[i] ); } + w.put64( uint64_t( ids.count ) ); +} + +// THE ID TABLE, READER SIDE (docs/SPEC-TABLES.md §3). A reader locates it from +// the END of the wire and resolves it ONCE, at open: the entries are eight +// bytes each and a body names them by position, so every field dispatches +// through an index rather than through a search over hashes. +struct TableIdTable +{ + const uint8_t * entries = NULL; + int64_t count = 0; + + // the id a reference names. ref is 1-based and bounds-checked by the + // caller: a reference ABOVE the entry count is framing damage on the body + // that carries it, and 0 names no id at all. + uint64_t at( uint64_t ref ) const + { + const uint8_t * e = entries + ( ref - 1 ) * 8; + uint64_t lo = uint64_t( e[0] ) | uint64_t( e[1] ) << 8 | uint64_t( e[2] ) << 16 | uint64_t( e[3] ) << 24; + uint64_t hi = uint64_t( e[4] ) | uint64_t( e[5] ) << 8 | uint64_t( e[6] ) << 16 | uint64_t( e[7] ) << 24; + return lo | ( hi << 32 ); + } +}; + +struct TableReader +{ + const uint8_t * buffer; + int64_t size; + int64_t offset = 0; + TableReport * report; + const TableIdTable * ids = NULL; + // ONLY THE ROOT BODY CARRIES THE NODE TABLE (docs/SPEC-TABLES.md §3.1), so + // a body has to know which it is: the reserved id inside a NESTED body is + // malformed, because a second numbering cannot exist. Every reader made + // for a payload is nested; the two the wire surfaces make for a root say so. + bool nested = true; + + TableReader( const uint8_t * from_buffer, int64_t from_size, TableReport * to_report ) + : buffer( from_buffer ), size( from_size ), report( to_report ) {} + + TableReader( const uint8_t * from_buffer, int64_t from_size, TableReport * to_report, const TableIdTable * to_ids ) + : buffer( from_buffer ), size( from_size ), report( to_report ), ids( to_ids ) {} + + MAPDEMO_TABLE_INLINE bool has( int64_t bytes ) const { return offset + bytes <= size; } + // A LENGTH IS A 64-BIT NUMBER AND A BUFFER IS NOT (docs/SPEC-TABLES.md + // §3): every length, count and index on this wire has sixty-four bits of + // capability, so one past what remains must be compared UNSIGNED. Casting + // it to int64 first turns 0xFFFFFFFFFFFFFFFF into -1, and a negative + // length looks like room. + MAPDEMO_TABLE_INLINE bool room( uint64_t bytes ) const { return bytes <= (uint64_t) ( size - offset ); } + MAPDEMO_TABLE_INLINE uint8_t get8() { return buffer[offset++]; } + MAPDEMO_TABLE_INLINE uint16_t get16() { uint16_t v = uint16_t( buffer[offset] ) | uint16_t( buffer[offset+1] ) << 8; offset += 2; return v; } + MAPDEMO_TABLE_INLINE uint32_t get32() { uint32_t v = uint32_t( buffer[offset] ) | uint32_t( buffer[offset+1] ) << 8 | uint32_t( buffer[offset+2] ) << 16 | uint32_t( buffer[offset+3] ) << 24; offset += 4; return v; } + MAPDEMO_TABLE_INLINE uint64_t get64() { uint64_t lo = get32(); uint64_t hi = get32(); return lo | ( hi << 32 ); } + MAPDEMO_TABLE_INLINE void get128( uint64_t & lo, uint64_t & hi ) { lo = get64(); hi = get64(); } + + // ONE CANONICAL UNSIGNED LEB128 (docs/SPEC-TABLES.md §3), and a + // non-minimal spelling is MALFORMED: 0x80 0x00 and 0x00 both spell zero, + // and only the second is legal input. An encoding past ten bytes, or a + // tenth byte with a bit above the 64th value bit, is malformed on the same + // rule. false = framing damage on the body carrying it. + bool getleb( uint64_t & value ) + { + // A NUMBER THIS READER REFUSES LEAVES THE CURSOR WHERE IT WAS. The + // caller's next question is often "did this body end exactly at its + // L", and a rejected number that had moved the cursor would answer + // that question with the damage already stepped over. + const int64_t at = offset; + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( !has( 1 ) ) { offset = at; return false; } + const uint8_t b = get8(); + if ( i == 9 && b > 1 ) { offset = at; return false; } + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) + { + if ( i > 0 && b == 0 ) { offset = at; return false; } // a redundant continuation + return true; + } + shift += 7; + } + offset = at; + return false; + } + + // resolve one id reference against the file's table. false = a reference + // ABOVE the entry count, or a 0 where an id is required, both of which + // are framing damage on the body that carries it. + bool getid( uint64_t & id ) + { + uint64_t ref = 0; + if ( !getleb( ref ) ) { return false; } + if ( ref == 0 || ids == NULL || ref > (uint64_t) ids->count ) { return false; } + id = ids->at( ref ); + return true; + } + + // skip one payload by kind; false = framing damage. FOUR RULES COVER THE + // SET (docs/SPEC-TABLES.md §3), and a kind outside it is not skippable — + // which is why the set is closed and why kind 31 exists. + bool skip( uint8_t kind ) + { + switch ( kind ) + { + // the fixed-width kinds, each by its width: 18-29 are the 128-bit integers and + // the fixed-point family at every storage width (docs/SPEC-TABLES.md §3) + case 1: case 2: case 6: case 20: case 25: return has( 1 ) ? ( offset += 1, true ) : false; + case 3: case 7: case 21: case 26: return has( 2 ) ? ( offset += 2, true ) : false; + case 4: case 8: case 10: case 22: case 27: return has( 4 ) ? ( offset += 4, true ) : false; + case 5: case 9: case 11: case 23: case 28: return has( 8 ) ? ( offset += 8, true ) : false; + case 18: case 19: case 24: case 29: return has( 16 ) ? ( offset += 16, true ) : false; + case 17: case 30: // a NODE INDEX (§3.1) and an ENUM's variant reference: one LEB128 and stop + { + uint64_t ignored = 0; + return getleb( ignored ); + } + case 12: case 13: case 14: case 16: case 31: case 32: case 33: // 31 is the ESCAPE, 32 the payload-free kind, 33 wide text + { + uint64_t n = 0; + if ( !getleb( n ) ) return false; + return room( n ) ? ( offset += (int64_t) n, true ) : false; + } + case 15: // union: the arm id reference, then its kind, its L and its payload (reference 0 = empty) + { + uint64_t arm = 0; + if ( !getleb( arm ) ) return false; + if ( arm == 0 ) return true; + if ( !has( 1 ) ) return false; + offset += 1; // the arm's kind byte + uint64_t n = 0; + if ( !getleb( n ) ) return false; + return room( n ) ? ( offset += (int64_t) n, true ) : false; + } + // KIND 34 IS RESERVED FOR float16 AND IS NOT PART OF THIS MAJOR (§3): + // no writer emits it and no reader has a rule for it, so a reader + // meets it only as DAMAGE, exactly as it meets 35 or 200. A bare 34 + // is a writer that ignored the escape kind 31. + case 34: return false; + } + return false; + } +}; + + +// WIDENING (docs/SPEC-TABLES.md §4): a payload under a kind BELOW the reader's +// on the same ladder decodes exactly. The signed ladder is kinds 2, 3, 4, 5, +// 18, the unsigned one 6, 7, 8, 9, 19, and 10 into 11 is the float rung. Every +// other pair is a kind mismatch. The declared kind is a constant at every call +// site, so this folds to one or two comparisons on the mismatch path and to +// nothing on the matching one. +inline bool TableKindWidens( uint8_t kind, uint8_t declared ) +{ + switch ( declared ) + { + case 3: case 4: case 5: return kind >= 2 && kind < declared; + case 18: return kind >= 2 && kind <= 5; + case 7: case 8: case 9: return kind >= 6 && kind < declared; + case 19: return kind >= 6 && kind <= 9; + case 11: return kind == 10; + } + return false; +} + +// a fixed-width kind's payload width, for the one place the width is a +// runtime fact: an arm whose kind byte the reader widens, whose L must be the +// wire kind's own width (§3) +inline int64_t TableKindWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + } + return 0; +} + +// the payload of a kind on the SIGNED ladder (2 to 5), sign-extended to +// sixty-four bits; false = the body cannot cover it, which is framing damage +inline bool TableReadSignedAt( TableReader & r, uint8_t kind, int64_t & out ) +{ + switch ( kind ) + { + case 2: if ( !r.has( 1 ) ) { return false; } out = (int8_t) r.get8(); return true; + case 3: if ( !r.has( 2 ) ) { return false; } out = (int16_t) r.get16(); return true; + case 4: if ( !r.has( 4 ) ) { return false; } out = (int32_t) r.get32(); return true; + default: if ( !r.has( 8 ) ) { return false; } out = (int64_t) r.get64(); return true; + } +} + +// the payload of a kind on the UNSIGNED ladder (6 to 9), zero-extended +inline bool TableReadUnsignedAt( TableReader & r, uint8_t kind, uint64_t & out ) +{ + switch ( kind ) + { + case 6: if ( !r.has( 1 ) ) { return false; } out = r.get8(); return true; + case 7: if ( !r.has( 2 ) ) { return false; } out = r.get16(); return true; + case 8: if ( !r.has( 4 ) ) { return false; } out = r.get32(); return true; + default: if ( !r.has( 8 ) ) { return false; } out = r.get64(); return true; + } +} + +// f32 into f64, exact: a NaN's payload is data and rides on the bits, since +// the hardware conversion would set the quiet bit (§4) +inline double TableWidenF32( uint32_t bits ) +{ + if ( ( bits & 0x7F800000u ) == 0x7F800000u && ( bits & 0x007FFFFFu ) != 0 ) + { + const uint64_t sign = (uint64_t) ( bits >> 31 ) << 63; + const uint64_t payload = (uint64_t) ( bits & 0x007FFFFFu ) << 29; + const uint64_t nan_bits = sign | 0x7FF0000000000000ull | payload; + double d; memcpy( &d, &nan_bits, 8 ); return d; + } + float f; memcpy( &f, &bits, 4 ); return (double) f; +} + +// ILL-FORMED TEXT IS DAMAGE (docs/SPEC-TABLES.md §3, §4): a kind 12 payload is +// well-formed UTF-8 with no zero byte among its bytes, checked AS IT ARRIVES +// and before the reader's own bound, because a payload that is not text is not +// text at whatever length the reader would have kept. Rejects a zero byte, a +// truncated sequence, a bare continuation, an overlong encoding, a surrogate +// and a code point past U+10FFFF, which is SPEC.md §4.7's rule in this wire's +// idiom: the field reads its declared default, one malformed counts, and the +// parent reads on past L. +// +// A LENGTH IS A 64-BIT NUMBER (§3), so it arrives as one: a payload length is +// whatever the wire spelled, and narrowing it to a signed count would read +// 0xFFFFFFFFFFFFFFFF as an empty payload. +inline bool TableUtf8Valid( const uint8_t * bytes, uint64_t length ) +{ + uint64_t i = 0; + while ( i < length ) + { + const uint8_t lead = bytes[i]; + uint64_t continuations; + uint32_t code_point; + if ( lead == 0 ) { return false; } + if ( lead < 0x80 ) { i++; continue; } + else if ( ( lead & 0xE0 ) == 0xC0 ) { continuations = 1; code_point = lead & 0x1F; } + else if ( ( lead & 0xF0 ) == 0xE0 ) { continuations = 2; code_point = lead & 0x0F; } + else if ( ( lead & 0xF8 ) == 0xF0 ) { continuations = 3; code_point = lead & 0x07; } + else { return false; } + if ( i + continuations >= length ) { return false; } + for ( uint64_t k = 1; k <= continuations; k++ ) + { + if ( ( bytes[i + k] & 0xC0 ) != 0x80 ) { return false; } + code_point = ( code_point << 6 ) | uint32_t( bytes[i + k] & 0x3F ); + } + if ( continuations == 1 && code_point < 0x80 ) { return false; } + if ( continuations == 2 && ( code_point < 0x800 || ( code_point >= 0xD800 && code_point <= 0xDFFF ) ) ) { return false; } + if ( continuations == 3 && ( code_point < 0x10000 || code_point > 0x10FFFF ) ) { return false; } + i += 1 + continuations; + } + return true; +} + +// A CLAMP CUTS AT A CODE POINT BOUNDARY (§3, §16.2): the last whole code point +// that fits within the bound, over a payload the check above already accepted, +// so a clamp can never invent ill-formed storage. +// +// THE ANSWER IS NEVER ABOVE THE BOUND. The length arrives as the wire's own +// 64-bit number and the caller turns the answer back into the size of a copy, +// so a length no reader could have bounded has to leave here bounded: taken as +// a signed count, 0xFFFFFFFFFFFFFFFF is -1, -1 is under every bound, and the +// copy would run at SIZE_MAX. +inline int64_t TableUtf8Clamp( const uint8_t * bytes, uint64_t length, int64_t bound ) +{ + if ( length <= (uint64_t) bound ) { return (int64_t) length; } + int64_t cut = bound; + while ( cut > 0 && ( bytes[cut] & 0xC0 ) == 0x80 ) { cut--; } + return cut; +} + +// ONE CODE UNIT off the wire: two bytes LITTLE-ENDIAN, this wire's order for +// every fixed-width number (docs/SPEC-TABLES.md §3). No unit can exceed +// 0xFFFF, because two bytes cannot spell one. +inline uint16_t TableUtf16Unit( const uint8_t * bytes, int64_t index ) +{ + return uint16_t( uint16_t( bytes[index * 2] ) | ( uint16_t( bytes[index * 2 + 1] ) << 8 ) ); +} + +// ILL-FORMED WIDE TEXT IS DAMAGE (docs/SPEC-TABLES.md §3, §4): a kind 33 +// payload carrying an UNPAIRED SURROGATE or a ZERO CODE UNIT among its units, +// checked AS IT ARRIVES and before the reader's own bound, on the rule kind 12 +// takes for UTF-8. An ODD L is framing damage and the caller rejects it ahead +// of this, because units is L / 2. SPEC.md §4.12 refuses the same content +// TERMINALLY on the packet wire; here the field reads its declared default, +// one malformed counts, and the parent reads on past L. +inline bool TableUtf16Valid( const uint8_t * bytes, int64_t units ) +{ + int64_t i = 0; + while ( i < units ) + { + const uint16_t unit = TableUtf16Unit( bytes, i ); + if ( unit == 0 ) { return false; } + if ( unit >= 0xD800 && unit <= 0xDBFF ) + { + if ( i + 1 >= units ) { return false; } // a high surrogate with no low half + const uint16_t low = TableUtf16Unit( bytes, i + 1 ); + if ( low < 0xDC00 || low > 0xDFFF ) { return false; } + i += 2; + continue; + } + if ( unit >= 0xDC00 && unit <= 0xDFFF ) { return false; } // a low surrogate first + i++; + } + return true; +} + +// A CLAMP CUTS AT A CODE UNIT BOUNDARY AND NEVER SPLITS A PAIR (§3, §16.2): +// the first bound units of a payload the check above already accepted, and +// where the last kept unit is a HIGH SURROGATE whose low half did not fit, +// that unit is dropped with it. So a clamp can never invent an unpaired +// surrogate, exactly as kind 12's clamp can never invent a broken sequence. +inline int64_t TableUtf16Clamp( const uint8_t * bytes, int64_t units, int64_t bound ) +{ + if ( units <= bound ) { return units; } + int64_t cut = bound; + if ( cut > 0 ) + { + const uint16_t last = TableUtf16Unit( bytes, cut - 1 ); + if ( last >= 0xD800 && last <= 0xDBFF ) { cut--; } + } + return cut; +} + +// The RESERVED node-table id, the one id the language holds back +// (docs/SPEC-TABLES.md §3.1, §5). It rides in every unit, pointered or not, +// because every body has to know that a NESTED body claiming one is damaged. +static const uint64_t kTableNodeTableFieldId = 0xFFFFFFFFFFFFFFFFull; + +// TableWireForm is the FORM BYTE, and it is the whole header +// (docs/SPEC-TABLES.md §3). A reader that meets a byte it does not know +// refuses the wire by name and never reports damage. +const uint8_t kTableWireForm = 1; + +// TableOpen reads the form byte and the trailer, in that order, and hands back +// the ROOT BODY. It answers one of three verdicts, because five zero counters +// and a false flag are what a clean read prints too: +// +// TableOpenOk the form is known and the table read whole +// TableOpenRefused a FORM BYTE this reader does not carry: nothing is +// decoded, nothing is counted, and no damage is reported +// TableOpenDamaged a table that cannot be read whole — fewer than eight +// bytes, a count whose entries run past the front of the +// file, a count that leaves no room for the form byte, or +// ONE ID IN TWO ENTRIES. The whole wire is malformed, +// nothing is decoded, and one event is counted. +// TableOpenBodyStopped the form and the table were good and the ROOT BODY +// could not be walked to its own terminator. What it +// decoded before that is kept, as everywhere on this wire. +enum TableOpenVerdict { TableOpenOk, TableOpenRefused, TableOpenDamaged, TableOpenBodyStopped }; + +inline TableOpenVerdict TableOpen( const uint8_t * buffer, int64_t bytes, TableIdTable & table, int64_t & body_bytes ) +{ + if ( bytes < 1 ) { return TableOpenDamaged; } + if ( buffer[0] != kTableWireForm ) { return TableOpenRefused; } + if ( bytes < 9 ) { return TableOpenDamaged; } + const uint8_t * tail = buffer + bytes - 8; + uint64_t lo = uint64_t( tail[0] ) | uint64_t( tail[1] ) << 8 | uint64_t( tail[2] ) << 16 | uint64_t( tail[3] ) << 24; + uint64_t hi = uint64_t( tail[4] ) | uint64_t( tail[5] ) << 8 | uint64_t( tail[6] ) << 16 | uint64_t( tail[7] ) << 24; + uint64_t count = lo | ( hi << 32 ); + if ( count > (uint64_t) ( bytes / 8 ) ) { return TableOpenDamaged; } + const int64_t span = (int64_t) count * 8 + 8; + if ( span + 1 > bytes ) { return TableOpenDamaged; } + table.entries = buffer + bytes - span; + table.count = (int64_t) count; + // THE ENTRIES ARE DISTINCT: a table that carries one id twice is malformed + // for the whole wire, because no wire this schema writes carries a repeat + // and it would leave one more shape of table for a hostile writer to aim + // at (docs/SPEC-TABLES.md §3). + for ( int64_t i = 1; i < table.count; i++ ) + { + const uint64_t id = table.at( uint64_t( i ) + 1 ); + for ( int64_t j = 0; j < i; j++ ) + { + if ( table.at( uint64_t( j ) + 1 ) == id ) { return TableOpenDamaged; } + } + } + body_bytes = bytes - span - 1; + return TableOpenOk; +} + +// TableBodyExtent walks a body's framing to the zero reference that ends it, +// so a reader can tell a body that ENDED EARLY — leaving bytes no field claims +// — from one that is merely damaged. ANY BYTE BETWEEN THE ROOT'S TERMINATOR +// AND THE TABLE'S FIRST ENTRY IS MALFORMED, because no field claims it and the +// two ends of the file have met (docs/SPEC-TABLES.md §3). +inline bool TableBodyEndsEarly( const uint8_t * body, int64_t bytes, const TableIdTable & table ) +{ + TableReport ignored; + TableReader r( body, bytes, &ignored, &table ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.getleb( ref ) ) { return false; } + if ( ref == 0 ) { return r.offset != bytes; } + if ( ref > (uint64_t) table.count ) { return false; } + if ( !r.has( 1 ) ) { return false; } + if ( !r.skip( r.get8() ) ) { return false; } + } +} + +// THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3): a batch of BITPACKED bodies +// under one announced vocabulary. +// +// A form 2 wire is THREE PARTS: the form byte, the body count, and the bodies +// as one continuous bit stream, zero-padded to the next byte at the end and +// nowhere else. A body is a sequence of fields, each a REFERENCE followed by a +// PAYLOAD and nothing else: no kind byte and no length, because the +// announcement carries the kind and the shape of every entry. +const uint8_t kTableWireMessageForm = 2; + +// THE COUNT IS A RANGED INTEGER OVER [1, 256], eight bits carrying M - 1. 256 +// is a WIRE CONSTANT of this form rather than a receiver's policy, because the +// count's WIDTH depends on it and two peers that disagreed on the width would +// not be reading the same wire. A batch of zero is not spellable. +static const int64_t kTableMessageBatchMax = 256; + +// The RESERVED ids of the announcement's own two fields (§5, §11), beside the +// node table's. They are the announcement's transport, they never appear in a +// body, and they take no slot in the vocabulary. +static const uint64_t kTableBuildVersionFieldId = 0xFFFFFFFFFFFFFFFEull; +static const uint64_t kTableMessageVocabularyFieldId = 0xFFFFFFFFFFFFFFFDull; + +// THE WIDEST COUNT THIS FORM SPELLS, which is the count an UNBOUNDED array +// announces (§2.9): an unbounded array states no bound, so the announcement +// states the widest one a batch could carry. It is the ceiling an array's or a +// keyed entry's announced min and max are checked against. +static const uint64_t kTableMessageListMax = 0xFFFFFFFFull; + +// THIS UNIT'S OWN REFERENCE WIDTH: the bits a writer spends on every reference +// of every body it writes, which is a compile-time constant because the +// vocabulary is. A READER spends the width the SENDER's vocabulary settles. +static const int64_t kTableMessageRefBitsHere = 7; + +// THIS UNIT'S OWN ENTRY COUNT, which is the CAPACITY a receiver declares for +// its resolved vocabulary when it talks only to peers of this schema (§3.3). +// The vocabulary is a pure function of the build version, so a peer at this +// build announces exactly this many entries; a receiver that means to meet +// OTHER builds declares more, and an announcement above whatever it declared +// is refused as vocabulary_too_large. +static const int64_t kTableMessageEntriesHere = 75; + +// The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A +// pointered body names the node table through it, and the node table is the +// ROOT body's FIRST field because a pointer index's width is settled by the +// node count it carries. +static const uint64_t kTableNodeTableFieldSlot = 54; + +// THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own +// layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, +// so a value written here and a value written by a generated packet writer are +// the same bits in the same places. + +// EIGHT BYTES OF THE STREAM AS ONE WORD, and the word is LITTLE-END-FIRST +// whatever order this host is in, because the stream's own definition puts +// bit i in byte i/8: byte 0 of the run holds the word's low eight bits. That +// is what lets one value of any width move in one unaligned load or store +// instead of one touch a byte, and the BITS ON THE WIRE do not move. +inline uint64_t table_message_byteswap64( uint64_t v ) +{ + return ( v >> 56 ) | ( ( v >> 40 ) & 0xff00ull ) | ( ( v >> 24 ) & 0xff0000ull ) | ( ( v >> 8 ) & 0xff000000ull ) + | ( ( v << 8 ) & 0xff00000000ull ) | ( ( v << 24 ) & 0xff0000000000ull ) | ( ( v << 40 ) & 0xff000000000000ull ) + | ( v << 56 ); +} + +inline uint64_t table_message_load64( const uint8_t * p ) +{ + uint64_t v = 0; + memcpy( &v, p, 8 ); +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + v = table_message_byteswap64( v ); +#endif + return v; +} + +inline void table_message_store64( uint8_t * p, uint64_t v ) +{ +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + v = table_message_byteswap64( v ); +#endif + memcpy( p, &v, 8 ); +} + +struct TableBitWriter +{ + uint8_t * buffer; + int64_t capacity; // bytes + int64_t bits; + bool overflow; + + TableBitWriter() : buffer( NULL ), capacity( 0 ), bits( 0 ), overflow( false ) {} + TableBitWriter( uint8_t * to_buffer, int64_t to_capacity ) : buffer( to_buffer ), capacity( to_capacity ), bits( 0 ), overflow( false ) {} + + // ONE WORD AT A TIME, never one bit and never one byte of arithmetic: the + // value is shifted into place in a REGISTER once, and the bytes it + // occupies are stored from that register with no read back. A sixty-four + // bit field costs one shift rather than nine masked read-modify-writes. + // The word is assembled little-end-first, so the BITS ON THE WIRE are the + // same bits in the same places, bit i in byte i/8 at position i%8 with the + // low bit first, which is what the pinned goldens hold. IT WRITES EXACTLY + // THE BYTES THE VALUE OCCUPIES and never one past them, so a caller's + // buffer beyond the batch is its own. + void put( uint64_t value, int64_t n ) + { + if ( n <= 0 ) { return; } + if ( ( bits + n + 7 ) / 8 > capacity ) { overflow = true; bits += n; return; } + if ( n < 64 ) { value &= ( uint64_t( 1 ) << n ) - 1; } // a caller's high bits never leak + const int64_t index = bits >> 3; + const int64_t bit = bits & 7; + // the byte the write STARTS in keeps the bits already written to it, + // and every byte after it is this value's own + const uint64_t head = bit != 0 ? ( uint64_t( buffer[index] ) & ( ( uint64_t( 1 ) << bit ) - 1 ) ) : 0; + const uint64_t word = head | ( value << bit ); + const int64_t need = ( bit + n + 7 ) >> 3; // 1 to 9 bytes + if ( need >= 8 ) + { + table_message_store64( buffer + index, word ); + if ( need > 8 ) { buffer[index + 8] = uint8_t( value >> ( 64 - bit ) ); } + } + else + { + for ( int64_t i = 0; i < need; i++ ) { buffer[index + i] = uint8_t( word >> ( 8 * i ) ); } + } + bits += n; + } + + // THE ALIGN IS WHAT BUYS THIS (docs/SPEC-TABLES.md §3.3): a string(N), a + // bytes(N) and a blob record align before their bytes precisely so the + // largest payload on the wire moves as ONE memcpy. Off a boundary there is + // nothing to memcpy and the bytes go through put. + void putbytes( const uint8_t * data, int64_t n ) + { + if ( n <= 0 ) { return; } + if ( ( bits & 7 ) == 0 ) + { + if ( ( bits >> 3 ) + n > capacity ) { overflow = true; bits += n * 8; return; } + memcpy( buffer + ( bits >> 3 ), data, (size_t) n ); + bits += n * 8; + return; + } + for ( int64_t i = 0; i < n; i++ ) { put( (uint64_t) data[i], 8 ); } + } + + // a string's or a bytes' payload ALIGNS before its bytes, and a batch + // aligns once at its end. Both are zero fill, spent in one call. + void align() { put( 0, ( 8 - ( bits & 7 ) ) & 7 ); } +}; + +// TableAlignBits is what an align costs from a bit position, which a measure +// spends exactly where a save does. +inline int64_t TableAlignBits( int64_t bits ) { return ( 8 - ( bits % 8 ) ) % 8; } + +struct TableBitReader +{ + const uint8_t * buffer; + int64_t bits; // the stream's extent, in bits + int64_t offset; // bits consumed + + TableBitReader() : buffer( NULL ), bits( 0 ), offset( 0 ) {} + TableBitReader( const uint8_t * from_buffer, int64_t from_bytes ) : buffer( from_buffer ), bits( from_bytes * 8 ), offset( 0 ) {} + + bool has( int64_t n ) const { return n >= 0 && offset + n <= bits; } + + // the primitive is sixty-four bits, and a width above it is refused + // here as well as at the announcement: no field on any body can ask this + // reader to move more bits than it holds + // ONE WORD OF THE BUFFER AT A TIME, the mirror of the writer's put: the + // eight bytes the value starts in load as one little-end-first word and a + // ninth byte carries the spill a value that straddles the word needs. + // Within nine bytes of the stream's end there is no room for a word load + // and the bytes come one at a time, by the same arithmetic. + bool get( uint64_t & value, int64_t n ) + { + if ( n > 64 || !has( n ) ) { return false; } + value = 0; + if ( n == 0 ) { return true; } + const int64_t index = offset >> 3; + const int64_t bit = offset & 7; + const int64_t bytes = ( bits + 7 ) >> 3; + if ( index + 9 <= bytes ) + { + uint64_t v = table_message_load64( buffer + index ) >> bit; + if ( bit != 0 && bit + n > 64 ) { v |= uint64_t( buffer[index + 8] ) << ( 64 - bit ); } + value = n == 64 ? v : ( v & ( ( uint64_t( 1 ) << n ) - 1 ) ); + offset += n; + return true; + } + int64_t got = 0; + while ( got < n ) + { + const int64_t byte = offset >> 3; + const int64_t off = offset & 7; + const int64_t room = 8 - off; + const int64_t take = ( n - got ) < room ? ( n - got ) : room; + const uint64_t chunk = ( uint64_t( buffer[byte] ) >> off ) & ( ( uint64_t( 1 ) << take ) - 1 ); + value |= chunk << got; + offset += take; + got += take; + } + return true; + } + + // the bytes of an ALIGNED payload, which is the read side of the memcpy + // the align buys (docs/SPEC-TABLES.md §3.3) + bool getbytes( uint8_t * out, int64_t n ) + { + if ( n < 0 || !has( n * 8 ) ) { return false; } + if ( ( offset & 7 ) == 0 ) + { + memcpy( out, buffer + ( offset >> 3 ), (size_t) n ); + offset += n * 8; + return true; + } + for ( int64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !get( by, 8 ) ) { return false; } + out[i] = (uint8_t) by; + } + return true; + } + + bool skip( int64_t n ) { if ( !has( n ) ) { return false; } offset += n; return true; } + + // the pad to the next byte boundary is VERIFIED ZERO, which is the packet + // wire's rule for the same reason (SPEC.md §4.3) + bool align() + { + const int64_t pad = ( 8 - ( offset & 7 ) ) & 7; + if ( pad == 0 ) { return true; } + uint64_t bits_read = 0; + return get( bits_read, pad ) && bits_read == 0; + } +}; + +// TableBitsRequired is bits_required( min, max ): the bit length of max - min, +// and zero where the two are equal, which is a value that spends no bit at all. +inline int64_t TableBitsRequired( int64_t min, int64_t max ) +{ + if ( max <= min ) { return 0; } + uint64_t span = (uint64_t) ( max - min ); + int64_t n = 0; + while ( span > 0 ) { n++; span >>= 1; } + return n; +} + +// THE ANNOUNCED ENTRY (§3.3): an id, a kind, and a SHAPE, which is the width +// and range facts a reader needs to SKIP a field exactly and to DECODE one +// whose own declaration has moved. One name may take TWO entries, at two kinds or two +// shapes, and a body names the one it means. +// +// The ELEMENT's own facts ride beside the field's because an array's element +// is the one nesting this wire has: an array of arrays is not a table-wire +// construct, so one level is every level. +// +// IT IS THE RESOLVED ENTRY AND THE CALLER SIZES AN ARRAY OF THEM, so it +// carries what a DECODE takes and nothing a decode does not: qmin, qdelta and +// qcount are what SPEC.md §4.3's rule leaves behind, and the qmax and qres +// that rule CONSUMES are locals of the parse. The widths are int16 because a +// width is bounded by the kind it came under and no kind holds more than 128 +// bits. +struct TableMessageEntry +{ + uint64_t id = 0; + int64_t min = 0; // an array's minimum count + int64_t max = 0; // an array's maximum count, a string's capacity, a keyed array's slots + int64_t base_lo = 0; // the ranged base, low half: a signed kind's sign-extends, an unsigned kind's is whole + int64_t base_hi = 0; // its high half, for a 128-bit kind + int64_t elem_max = 0; + int64_t elem_base_lo = 0; + int64_t elem_base_hi = 0; + // what SPEC.md §4.3's derivation leaves: the base, the step and the count + float qmin = 0.0f; + float qdelta = 0.0f; + uint32_t qcount = 0; + float elem_qmin = 0.0f; + float elem_qdelta = 0.0f; + uint32_t elem_qcount = 0; + // THE PAYLOAD'S WIDTH, RESOLVED: what the kind, the packing and the + // announced bits together say, computed once at AnnounceRead, and -1 + // where the payload is not a fixed-width value at all + int16_t value_bits = -1; + int16_t elem_value_bits = -1; + uint8_t kind = 0; + uint8_t packing = 0; + uint8_t elem_kind = 0; + uint8_t elem_packing = 0; +}; + +// TableMessageEntrySame reports whether two RESOLVED entries carry the same +// shape, which is every fact of the entry but its id and its kind. It is what +// the announcement's duplicate rule is asked in: two entries that agree on all +// three parts are malformed (§3.3). +inline bool TableMessageEntrySame( const TableMessageEntry & a, const TableMessageEntry & b ) +{ + return a.min == b.min && a.max == b.max && a.base_lo == b.base_lo && a.base_hi == b.base_hi + && a.elem_max == b.elem_max && a.elem_base_lo == b.elem_base_lo && a.elem_base_hi == b.elem_base_hi + && a.qmin == b.qmin && a.qdelta == b.qdelta && a.qcount == b.qcount + && a.elem_qmin == b.elem_qmin && a.elem_qdelta == b.elem_qdelta && a.elem_qcount == b.elem_qcount + && a.value_bits == b.value_bits && a.elem_value_bits == b.elem_value_bits + && a.packing == b.packing && a.elem_kind == b.elem_kind && a.elem_packing == b.elem_packing; +} + +// TableMessageKindBits is the widest RANGED value a kind can carry, its own +// storage width: a width above it is a hostile width on the announcement. +inline int64_t TableMessageKindBits( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 6: case 20: case 25: return 8; + case 3: case 7: case 21: case 26: return 16; + case 4: case 8: case 22: case 27: return 32; + case 5: case 9: case 23: case 28: return 64; + default: return 128; + } +} + +// TableMessageQuantization is SPEC.md §4.3's derivation over an announced +// triple, in float32 and by nothing else: delta, the step count and the +// width. False is a triple SPEC.md calls non-conforming, which on the +// announcement is a hostile width like any other (§3.3). +inline bool TableMessageQuantization( float qmin, float qmax, float qres, float & delta, uint32_t & count, int64_t & bits ) +{ + if ( !( qmin < qmax ) || !( qres > 0.0f ) ) { return false; } + delta = qmax - qmin; + float values = delta / qres; + if ( !( delta - delta == 0.0f ) || !( values - values == 0.0f ) ) { return false; } // Inf - Inf is NaN + if ( !( values >= 1.0f ) ) { values = 1.0f; } + else if ( values > 4294967040.0f ) { values = 4294967040.0f; } // the largest float below 2^32 + count = (uint32_t) values; + if ( (float) count < values ) { count++; } // ceil, on a value the cast holds exactly + bits = TableBitsRequired( 0, (int64_t) count ); + return true; +} + +// The two roundings on each side of the rule (SPEC.md §7.2): the product +// rounds to float32 BEFORE the add, which a compiler permitted to contract +// would otherwise fuse into one rounding and move the wire. +#if ( defined( __GNUC__ ) || defined( __clang__ ) ) && ( defined( __aarch64__ ) || defined( _M_ARM64 ) ) +#define TABLE_FLOAT_FORCE_ROUND( x ) __asm__ ( "" : "+w" ( x ) ) +#elif ( defined( __GNUC__ ) || defined( __clang__ ) ) && ( defined( __x86_64__ ) || defined( __i386__ ) ) +#define TABLE_FLOAT_FORCE_ROUND( x ) __asm__ ( "" : "+x" ( x ) ) +#else +#define TABLE_FLOAT_FORCE_ROUND( x ) do { volatile float table_float_force_round_slot = ( x ); ( x ) = table_float_force_round_slot; } while ( 0 ) +#endif + +// TableMessageQuantize is the writer's half: the index a value takes. +inline uint32_t TableMessageQuantize( float value, float qmin, float delta, uint32_t count ) +{ + float normalized = ( value - qmin ) / delta; + if ( !( normalized >= 0.0f ) ) { normalized = 0.0f; } + else if ( !( normalized <= 1.0f ) ) { normalized = 1.0f; } + float scaled = normalized * (float) count; + TABLE_FLOAT_FORCE_ROUND( scaled ); + uint32_t index = (uint32_t) ( scaled + 0.5f ); // floor of a non-negative value + if ( index > count ) { index = count; } + return index; +} + +// TableMessageDequantize is the reader's half: the float an index names. +inline float TableMessageDequantize( uint32_t index, float qmin, float delta, uint32_t count ) +{ + if ( index > count ) { index = count; } + const float normalized = index / (float) count; + float scaled = normalized * delta; + TABLE_FLOAT_FORCE_ROUND( scaled ); + return scaled + qmin; +} + +inline bool TableMessageIntegerKind( uint8_t kind ) +{ + return ( kind >= 2 && kind <= 9 ) || kind == 18 || kind == 19; +} + +inline bool TableMessageFixedKind( uint8_t kind ) { return kind >= 20 && kind <= 29; } + +inline bool TableMessageKnownKind( uint8_t kind ) +{ + return kind == 0 || ( kind >= 1 && kind <= 17 ) || ( kind >= 18 && kind <= 29 ) || ( kind >= 30 && kind <= 33 ); +} + +// A CANONICAL LEB128, which is the announcement's own integer: the +// announcement is a form 1 FILE and takes §3's rule. +inline bool TableMessageLeb( const uint8_t * in, int64_t size, int64_t & at, uint64_t & value ) +{ + value = 0; + for ( int64_t shift = 0; at < size; shift += 7 ) + { + if ( shift >= 64 ) { return false; } + const uint8_t by = in[ at++ ]; + value |= uint64_t( by & 0x7F ) << shift; + if ( ( by & 0x80 ) == 0 ) { return !( shift > 0 && by == 0 ); } + } + return false; +} + +// TableMessageShapeFacts is where one shape's facts land: the field's own, +// or its element's, which is the one nesting this wire has. +struct TableMessageShapeFacts +{ + uint8_t & packing; int64_t & value_bits; int64_t & base_lo; int64_t & base_hi; + float & qmin; float & qmax; float & qres; float & qdelta; uint32_t & qcount; + int64_t & min; int64_t & max; uint8_t & elem_kind; +}; + +inline bool TableMessageShapeRead( const uint8_t * in, int64_t size, int64_t & at, uint8_t kind, TableMessageShapeFacts f ); +inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t value_bits ); + +// TableMessageEntryRead parses ONE entry, and answers false for a HOSTILE +// SHAPE: bits above the kind's own domain, an array whose min exceeds its +// max, an element kind outside the closed set, a quantized triple SPEC.md +// calls non-conforming, or a shape running past the vocabulary's own bytes. +inline bool TableMessageEntryRead( const uint8_t * in, int64_t size, int64_t & at, TableMessageEntry & entry ) +{ + if ( at + 9 > size ) { return false; } + entry = TableMessageEntry(); + for ( int i = 0; i < 8; i++ ) { entry.id |= uint64_t( in[ at + i ] ) << ( 8 * i ); } + entry.kind = in[ at + 8 ]; + at += 9; + if ( !TableMessageKnownKind( entry.kind ) ) { return false; } + // The parse lands in LOCALS and the entry keeps what a decode reads: the + // quantized max and res are the derivation's inputs and never a field's. + uint8_t packing = 0, elem_kind = 0; + int64_t bits = 0, base_lo = 0, base_hi = 0, min = 0, max = 0; + float qmin = 0.0f, qmax = 0.0f, qres = 0.0f, qdelta = 0.0f; + uint32_t qcount = 0; + TableMessageShapeFacts own = { packing, bits, base_lo, base_hi, + qmin, qmax, qres, qdelta, qcount, + min, max, elem_kind }; + if ( !TableMessageShapeRead( in, size, at, entry.kind, own ) ) { return false; } + entry.packing = packing; + entry.value_bits = (int16_t) TableMessageValueBits( entry.kind, packing, bits ); + entry.base_lo = base_lo; + entry.base_hi = base_hi; + entry.qmin = qmin; + entry.qdelta = qdelta; + entry.qcount = qcount; + entry.min = min; + entry.max = max; + entry.elem_kind = elem_kind; + if ( entry.kind == 14 || entry.kind == 16 ) + { + uint8_t elem_packing = 0, inner_kind = 0; + int64_t elem_bits = 0, elem_base_lo = 0, elem_base_hi = 0, elem_min = 0, elem_max = 0; + float elem_qmin = 0.0f, elem_qmax = 0.0f, elem_qres = 0.0f, elem_qdelta = 0.0f; + uint32_t elem_qcount = 0; + TableMessageShapeFacts elem = { elem_packing, elem_bits, elem_base_lo, elem_base_hi, + elem_qmin, elem_qmax, elem_qres, elem_qdelta, elem_qcount, + elem_min, elem_max, inner_kind }; + if ( !TableMessageShapeRead( in, size, at, entry.elem_kind, elem ) ) { return false; } + entry.elem_packing = elem_packing; + entry.elem_value_bits = (int16_t) TableMessageValueBits( entry.elem_kind, elem_packing, elem_bits ); + entry.elem_base_lo = elem_base_lo; + entry.elem_base_hi = elem_base_hi; + entry.elem_qmin = elem_qmin; + entry.elem_qdelta = elem_qdelta; + entry.elem_qcount = elem_qcount; + entry.elem_max = elem_max; + } + return true; +} + +// TableMessageShapeRead is one shape, by the kind that names it (§3.3's shape +// table). Every number in it is a canonical LEB128 except where the row says +// otherwise: a RANGED BASE IS ENCODED BY ITS KIND'S SIGNEDNESS, zigzag for the +// signed kinds, unsigned for the unsigned kinds and sixteen bytes for the +// 128-bit and fixed-point kinds, and a QUANTIZED f32 carries min, max and res +// as float32, from which the step count and the width derive by SPEC.md +// §4.3's rule and by nothing else. +inline bool TableMessageShapeRead( const uint8_t * in, int64_t size, int64_t & at, uint8_t kind, TableMessageShapeFacts f ) +{ + uint64_t v = 0; + if ( TableMessageIntegerKind( kind ) || TableMessageFixedKind( kind ) || kind == 10 ) + { + if ( at >= size ) { return false; } + f.packing = in[ at++ ]; + if ( f.packing == 0 ) { return true; } + if ( f.packing == 1 && kind != 10 ) + { + if ( !TableMessageLeb( in, size, at, v ) || (int64_t) v > TableMessageKindBits( kind ) ) { return false; } + f.value_bits = (int64_t) v; + if ( kind == 18 || kind == 19 || TableMessageFixedKind( kind ) ) + { + if ( at + 16 > size ) { return false; } + uint64_t lo = 0, hi = 0; + for ( int i = 0; i < 8; i++ ) { lo |= uint64_t( in[ at + i ] ) << ( 8 * i ); } + for ( int i = 0; i < 8; i++ ) { hi |= uint64_t( in[ at + 8 + i ] ) << ( 8 * i ); } + f.base_lo = (int64_t) lo; f.base_hi = (int64_t) hi; + at += 16; + return true; + } + if ( !TableMessageLeb( in, size, at, v ) ) { return false; } + if ( kind >= 2 && kind <= 5 ) { f.base_lo = (int64_t) ( v >> 1 ) ^ -(int64_t) ( v & 1 ); } // zigzag + else { f.base_lo = (int64_t) v; } // the unsigned domain, whole + return true; + } + if ( f.packing == 2 && kind == 10 ) + { + if ( at + 12 > size ) { return false; } + uint32_t raw[3] = { 0, 0, 0 }; + for ( int k = 0; k < 3; k++ ) { for ( int i = 0; i < 4; i++ ) { raw[k] |= uint32_t( in[ at + 4 * k + i ] ) << ( 8 * i ); } } + at += 12; + memcpy( &f.qmin, &raw[0], 4 ); + memcpy( &f.qmax, &raw[1], 4 ); + memcpy( &f.qres, &raw[2], 4 ); + return TableMessageQuantization( f.qmin, f.qmax, f.qres, f.qdelta, f.qcount, f.value_bits ); + } + return false; // a packing outside the closed set + } + // A MAX ABOVE WHAT THE KIND CAN HOLD IS A HOSTILE WIDTH (§3.3). A string + // and a wide string are bounded by the int32 storage cap the checker + // applies to every N (SPEC §4.3, §6.1), and an array and a keyed entry by + // the 32-bit count an unbounded array announces (§2.9), which is the + // widest count this form spells. A larger bound is a shape no conforming + // declaration can produce, and a reader that carried it would do its + // length arithmetic in a range that overflows. + if ( kind == 12 || kind == 33 ) + { + if ( !TableMessageLeb( in, size, at, v ) || v > (uint64_t) INT32_MAX ) { return false; } + f.max = (int64_t) v; + return true; + } + if ( kind == 14 || kind == 16 ) + { + if ( kind == 14 ) + { + if ( !TableMessageLeb( in, size, at, v ) || v > kTableMessageListMax ) { return false; } + f.min = (int64_t) v; + } + if ( !TableMessageLeb( in, size, at, v ) || v > kTableMessageListMax ) { return false; } + if ( (int64_t) v < f.min ) { return false; } + f.max = (int64_t) v; + if ( at >= size ) { return false; } + f.elem_kind = in[ at++ ]; + if ( !TableMessageKnownKind( f.elem_kind ) ) { return false; } + // AND AN ELEMENT KIND OF 12 OR 33 IS REFUSED HERE, at the + // announcement, rather than at the skip that would meet it (§3.3): no + // declaration this language accepts is an array of string(N) or of + // wstring(N), so a shape announcing one is one rule's business and not + // two. + if ( f.elem_kind == 12 || f.elem_kind == 33 ) { return false; } + return true; + } + return true; +} + +// TableMessageValueBits is one value's width under a shape, and -1 where the +// kind's payload is not a fixed-width value at all. +inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t value_bits ) +{ + if ( kind == 1 ) { return 1; } + if ( kind == 11 ) { return 64; } + if ( kind == 10 ) { return packing == 2 ? value_bits : 32; } + if ( TableMessageIntegerKind( kind ) || TableMessageFixedKind( kind ) ) + { + if ( packing == 1 ) { return value_bits; } + switch ( kind ) + { + case 2: case 6: case 20: case 25: return 8; + case 3: case 7: case 21: case 26: return 16; + case 4: case 8: case 22: case 27: return 32; + case 5: case 9: case 23: case 28: return 64; + default: return 128; + } + } + return -1; +} + +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an +// ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under +// the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 +// over element kind 6, and a trailer of those two reserved ids. +// +// THE VOCABULARY IS A FIELD AND NOT THE TRAILER, and that buys three things: +// §3's writer rule that an id no body references is never written is restored +// unbroken, an entry can carry a KIND and a SHAPE which a trailer of bare ids +// cannot, and one NAME can appear at two shapes. +// +// The order is the COOK PROJECTION's (§20.2): each record in the order the +// projection renders it and each record's fields in the order the projection +// renders them, then each enum's variants and each union's arms. Then comes +// the tail the projection does not name: the reserved node-table id, the three +// blob type ids as bytes, string and wstring, and every table's own name id in +// the projection's sorted record order. The tail is UNCONDITIONAL, so an +// ordinary edit only ever grows it at its end and never moves a slot a +// generated field header carries as a literal. +static const int64_t kTableAnnounceBytes = 901; +static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, + 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, + 0xe4, 0x7c, 0x11, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, + 0x20, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0d, 0xec, 0x10, + 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, + 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, + 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, + 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, + 0x07, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x21, 0x06, + 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0c, 0x10, 0x38, 0x81, + 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, + 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, +}; + +// TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries +// an announcement carried, RESOLVED ONCE, under one numbering. +// +// THE RECEIVER RESOLVES ONCE (§3.3), so this holds the entries themselves and +// not the announcement's bytes: every entry is parsed at AnnounceRead, and +// every body after it dispatches through ONE ARRAY INDEX with nothing to +// re-read and nothing to decide. The announcement is free the moment +// AnnounceRead returns. +// +// THE STORAGE IS THE CALLER'S and this library never allocates. The caller +// declares an array of entries wherever it wants it, static, on a heap, in an +// arena or beside its connection, and hands it here with its CAPACITY. The +// announcement holds for the life of the connection (§3.3), so the array does +// too, and a peer holds TWO for a connection, the one it writes with and the +// one it reads with. A restart opens a fresh connection with an empty +// vocabulary and nothing is cached across connections. +// +// kTableMessageEntriesHere is the capacity a receiver that talks only to peers +// of THIS schema declares, and a receiver meeting other builds declares more. +struct TableVocabulary +{ + // THE CONFORMING DEFAULT BYTE BOUND (§3.3). The ENTRY bound has no default + // because it IS the caller's capacity: an announcement naming more entries + // than the caller made room for is refused as vocabulary_too_large before + // an entry is touched, and the byte bound is read off the vocabulary + // field's own length before that. + static const int64_t kDefaultMaxBytes = 64 * 1024; + + TableVocabulary( TableMessageEntry * storage, int64_t capacity ) + : entries( storage ), max_entries( capacity ) {} + + TableMessageEntry * entries; // THE CALLER'S, capacity max_entries + int64_t max_entries; + int64_t count = 0; + int64_t ref_bits = 0; + uint64_t build_version = 0; + bool announced = false; + // REFUSAL IS TERMINAL (§3.3): a connection whose first announcement was + // refused, for any reason, carries no vocabulary for its life, and every + // announcement after it is refused as second_announcement + bool refused = false; + int64_t max_bytes = kDefaultMaxBytes; +}; + +// TableVocabularyEntryAt is the entry a reference names, counted from 1: ONE +// ARRAY INDEX into the caller's resolved storage, no parse and no branch. +inline const TableMessageEntry & TableVocabularyEntryAt( const TableVocabulary & vocabulary, uint64_t slot ) +{ + return vocabulary.entries[ slot - 1 ]; +} + +// AnnounceRead reads an announcement into one direction's vocabulary (§3.3). +// +// The announcement IS a file, so every malformed rule of §3 already covers it. +// Over its body there are EXACTLY TWO STRICT CHECKS: the BUILD VERSION +// present, exactly once, under kind 9, eight bytes wide, and the VOCABULARY +// present, exactly once, under kind 14 over element kind 6. Everything else is +// ordinary and tolerant, so an unknown field is skipped and counted and the +// announcement can GAIN a field in a later minor without a lockstep redeploy. +// +// The FIRST announcement sets the vocabulary and it is the only one that can. +// A SECOND is refused by name: it does not replace it, does not amend it and +// changes nothing. A refused announcement sets NO VOCABULARY, and the refusal +// is TERMINAL: every announcement after it, whether or not the first set +// anything, is second_announcement, so a peer holds no retry on the +// connection and cannot buy a second resolve by having its first refused. +inline bool AnnounceReadOnce( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * to ); + +inline bool AnnounceRead( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * to = report != NULL ? report : &ignored; + if ( vocabulary.announced || vocabulary.refused ) + { + to->refused = true; + to->reason = second_announcement; + return false; + } + const bool set = AnnounceReadOnce( vocabulary, buffer, bytes, to ); + if ( !set ) { vocabulary.refused = true; } + return set; +} + +inline bool AnnounceReadOnce( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * to ) +{ + if ( bytes < 1 ) { to->malformed = true; return false; } + if ( buffer[0] != kTableWireForm ) + { + to->refused = true; + to->reason = buffer[0] == kTableWireMessageForm ? message_form_as_file : newer_form; + return false; + } + if ( bytes < 9 ) { to->malformed = true; return false; } + TableIdTable table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( buffer, bytes, table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { to->malformed = true; } + else { to->refused = true; to->reason = newer_form; } + return false; + } + if ( TableBodyEndsEarly( buffer + 1, body_bytes, table ) ) { to->malformed = true; return false; } + TableReader r( buffer + 1, body_bytes, to, &table ); + uint64_t version = 0; + const uint8_t * words = NULL; + int64_t words_bytes = 0; + int32_t seen_version = 0, seen_vocabulary = 0; + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.getleb( ref ) ) { to->malformed = true; return false; } + if ( ref == 0 ) { break; } + if ( ref > (uint64_t) table.count || !r.has( 1 ) ) { to->malformed = true; return false; } + const uint64_t id = table.at( ref ); + const uint8_t kind = r.get8(); + if ( id == kTableBuildVersionFieldId ) + { + if ( kind != 9 || !r.has( 8 ) ) { to->malformed = true; return false; } + version = r.get64(); + // THE BUILD VERSION IS KEPT THE MOMENT IT IS READ, refusal or not, so + // that a refusal on this connection NAMES IT (§3.3). It is not the + // vocabulary, and a refused announcement still sets none. + vocabulary.build_version = version; + seen_version++; + continue; + } + if ( id == kTableMessageVocabularyFieldId ) + { + // kind 14 over element kind 6, which is §3's spelling for an + // opaque run of bytes + uint64_t framed = 0; + if ( kind != 14 || !r.getleb( framed ) || !r.has( (int64_t) framed ) ) { to->malformed = true; return false; } + const int64_t begin = r.offset, end = r.offset + (int64_t) framed; + r.offset = end; + if ( begin >= end || r.buffer[ begin ] != 6 ) { to->malformed = true; return false; } + int64_t at = begin + 1; + uint64_t length = 0; + if ( !TableMessageLeb( r.buffer, end, at, length ) || at + (int64_t) length != end ) { to->malformed = true; return false; } + if ( (int64_t) length > vocabulary.max_bytes ) { to->refused = true; to->reason = vocabulary_too_large; return false; } + words = r.buffer + at; + words_bytes = (int64_t) length; + seen_vocabulary++; + continue; + } + to->unknown++; + if ( !r.skip( kind ) ) { to->malformed = true; return false; } + } + if ( seen_version != 1 || seen_vocabulary != 1 ) { to->malformed = true; return false; } + + // THE ENTRIES, RESOLVED ONCE into the caller's storage (§3.3): every width + // is checked here and never again, and no body after this parses a byte of + // an announcement. An entry count above the caller's CAPACITY is refused + // by name before the entry is touched. + int64_t at = 0, count = 0, node_table_slots = 0; + while ( at < words_bytes ) + { + if ( count >= vocabulary.max_entries ) { to->refused = true; to->reason = vocabulary_too_large; return false; } + TableMessageEntry & parsed = vocabulary.entries[ count ]; + if ( !TableMessageEntryRead( words, words_bytes, at, parsed ) ) { to->malformed = true; return false; } + // THE RESERVED IDS WHERE THEY DO NOT BELONG (§3.3): the announcement's + // own two never take a slot, and the node-table id takes exactly one, + // so a vocabulary carrying either of the first or a SECOND node-table + // id is malformed whole and sets nothing + if ( parsed.id == kTableBuildVersionFieldId || parsed.id == kTableMessageVocabularyFieldId ) { to->malformed = true; return false; } + if ( parsed.id == kTableNodeTableFieldId ) { if ( node_table_slots++ > 0 ) { to->malformed = true; return false; } } + // A TRIPLE ALREADY PLACED IS NEVER PLACED TWICE, so two entries that + // agree on the id, the kind and every fact of the shape are malformed + // (§3.3): no writer this wire has produces one, and a reader that took + // it would carry two slots naming one thing. The scan is quadratic in + // the entry count, and the entry count is bounded above at 4096, so it + // is at most eight million compares on a path that runs ONCE a + // connection and never again. + for ( int64_t seen = 0; seen < count; seen++ ) + { + const TableMessageEntry & other = vocabulary.entries[ seen ]; + if ( other.id == parsed.id && other.kind == parsed.kind && TableMessageEntrySame( other, parsed ) ) { to->malformed = true; return false; } + } + count++; + } + vocabulary.count = count; + vocabulary.ref_bits = TableBitsRequired( 0, count ); + vocabulary.build_version = version; + vocabulary.announced = true; + return true; +} + +// TableMessageReserved is one of the three ids the language holds back (§3.1, +// §3.3, §5): each is malformed anywhere but its own transport, and the rule +// OUTRANKS the wrong-sort rule below. +inline bool TableMessageReserved( uint64_t id ) +{ + // THE THREE ARE THE TOP THREE VALUES a uint64 holds, so the test is ONE + // comparison: 0xFFFFFFFFFFFFFFFD, FE and FF and nothing else is at or + // above the vocabulary's own id, and a declaration hashing to any of them + // is refused by name (§11) + return id >= kTableMessageVocabularyFieldId; +} + +// TableMessageNameEntry resolves a reference used as a VALUE, which is an +// enum's variant, a keyed array's slot key or a node record's type id, and +// which must name a kind-0 entry (§3.3). A reference of 0 where an entry is required, one +// above E, one naming a reserved id and one naming an entry that carries a +// payload are each damage: the reader RESOLVED the entry and it contradicts +// the position it was used in, so the next bit's meaning is what is in doubt. +inline bool TableMessageNameEntry( const TableVocabulary & vocabulary, uint64_t ref, TableMessageEntry & entry ) +{ + if ( ref == 0 || ref > (uint64_t) vocabulary.count ) { return false; } + entry = TableVocabularyEntryAt( vocabulary, ref ); + return !TableMessageReserved( entry.id ) && entry.kind == 0; +} + +// TableMessageArmEntry resolves a UNION's arm reference, which must name an +// entry carrying the arm's own kind and shape: a kind-0 entry frames nothing, +// and a reserved id belongs to no arm (§3.3). +inline bool TableMessageArmEntry( const TableVocabulary & vocabulary, uint64_t ref, TableMessageEntry & entry ) +{ + if ( ref == 0 || ref > (uint64_t) vocabulary.count ) { return false; } + entry = TableVocabularyEntryAt( vocabulary, ref ); + return !TableMessageReserved( entry.id ) && entry.kind != 0; +} + +// TableMessageSkipVariant steps over an ENUM's variant reference on a SKIP +// path and RESOLVES it while it is there: 0 is None and the whole payload, and +// every other reference must name a kind-0 entry, because every reference +// above E is damage and one naming an entry that carries a payload +// contradicts the position it was used in, whether or not this reader was +// going to keep the value (§3.3). +inline bool TableMessageSkipVariant( TableBitReader & r, const TableVocabulary & vocabulary ) +{ + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + TableMessageEntry named; + return TableMessageNameEntry( vocabulary, ref, named ); +} +// TableMessageSkip steps over one field's payload without decoding it, using +// the announced ENTRY alone (§3.3). It is what makes an unknown entry +// skippable on a body with no kind byte, and it is ONE function over every +// table, because a shape says everything a skipper needs. +inline bool TableMessageSkipBody( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits ); +inline bool TableMessageSkip( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ); + +// TableMessageSkipElement steps over ONE element of an array or keyed entry +// by the element's own announced shape: a nested body to its zero reference, +// a variant or a node index at its reference width, a union arm by its own +// entry, and a fixed-width value at its bits. +inline bool TableMessageSkipElement( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ) +{ + switch ( entry.elem_kind ) + { + case 13: return TableMessageSkipBody( r, vocabulary, index_bits ); + case 30: return TableMessageSkipVariant( r, vocabulary ); + case 17: return index_bits > 0 && r.skip( index_bits ); + case 15: + { + TableMessageEntry inner; + inner.kind = 15; + return TableMessageSkip( r, vocabulary, index_bits, inner ); + } + default: + { + const int64_t elem = entry.elem_value_bits; + return elem >= 0 && r.skip( elem ); + } + } +} + +// TableMessageElementRunBits is the bits ONE element of an array or a keyed +// entry occupies on the SKIP path, where nothing is resolved and a run of them +// is one multiplication, and -1 where the element's width is its own +// content's. A ZERO is a real answer, and it is why this exists: a ranged +// element whose min equals its max rides no bits at all (§3.3). +inline int64_t TableMessageElementRunBits( const TableVocabulary & vocabulary, const TableMessageEntry & entry ) +{ + int64_t elem = 0; + switch ( entry.elem_kind ) + { + // a nested body, a union arm, an enum's variant and a node index each + // RESOLVE something, and a resolve that contradicts its position is + // damage this reader must still find, so they are walked + case 13: case 15: case 30: case 17: return -1; + default: elem = entry.elem_value_bits; break; + } + if ( elem < 0 ) { return -1; } + if ( entry.kind == 16 ) { elem += vocabulary.ref_bits; } // a keyed slot's own key reference + return elem; +} + +// TableMessageSkipRun steps over n elements of one fixed width in a single +// arithmetic step. A FIXED-WIDTH ELEMENT IS ARITHMETIC (§3.3), and a loop here +// would be the one superlinear thing in this form: a zero-width element under +// a count of 2^31 is six bytes of wire. +inline bool TableMessageSkipRun( TableBitReader & r, uint64_t n, int64_t width ) +{ + if ( width < 0 ) { return false; } + if ( width == 0 ) { return true; } + if ( n > (uint64_t) ( INT64_MAX / width ) ) { return false; } + return r.skip( (int64_t) ( n * (uint64_t) width ) ); +} + +inline bool TableMessageSkip( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ) +{ + switch ( entry.kind ) + { + case 0: case 32: return true; // a name, and a payload-free arm + case 30: return TableMessageSkipVariant( r, vocabulary ); + case 13: return TableMessageSkipBody( r, vocabulary, index_bits ); + case 17: return index_bits > 0 && r.skip( index_bits ); // a node index, at the width the body's node count settled + case 15: + { + uint64_t arm = 0; + if ( !r.get( arm, vocabulary.ref_bits ) ) { return false; } + if ( arm == 0 ) { return true; } + TableMessageEntry arm_entry; + if ( !TableMessageArmEntry( vocabulary, arm, arm_entry ) ) { return false; } + return TableMessageSkip( r, vocabulary, index_bits, arm_entry ); + } + case 12: + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() ) { return false; } + return r.skip( (int64_t) n * 8 ); + } + case 33: + { + // the length, NO align, then SIXTEEN bits a code unit (§3.3) + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + return r.skip( (int64_t) n * 16 ); + } + case 31: + { + // THE ESCAPE: align, a thirty-two bit L, then L bytes, opaque. It is + // the one path a later-major writer has on this form (§3.3) + uint64_t n = 0; + if ( !r.align() || !r.get( n, 32 ) ) { return false; } + return r.skip( (int64_t) n * 8 ); + } + case 14: case 16: + { + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, width ) ) { return false; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !r.align() ) { return false; } + // A RUN OF FIXED-WIDTH ELEMENTS IS ONE MULTIPLICATION (§3.3), and + // only an element whose width is its own content's is walked + const int64_t run = TableMessageElementRunBits( vocabulary, entry ); + if ( run >= 0 ) { return TableMessageSkipRun( r, n, run ); } + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 && !r.skip( vocabulary.ref_bits ) ) { return false; } + if ( !TableMessageSkipElement( r, vocabulary, index_bits, entry ) ) { return false; } + } + return true; + } + } + const int64_t width = entry.value_bits; + return width >= 0 && r.skip( width ); +} + +inline bool TableMessageSkipBody( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + if ( !TableMessageSkip( r, vocabulary, index_bits, TableVocabularyEntryAt( vocabulary, ref ) ) ) { return false; } + } +} + +// TableMessageNodeTableOpen reads the node table's opening when a body has +// one: the reserved id's reference and the count at thirty-two raw bits. A +// body whose first reference is anything else has no node table, and the +// reader is left where it was. False is damage: a reference past E, or bits +// that run out. +inline bool TableMessageNodeTableOpen( TableBitReader & r, const TableVocabulary & vocabulary, int64_t & count ) +{ + count = 0; + const int64_t at = r.offset; + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { r.offset = at; return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + if ( TableVocabularyEntryAt( vocabulary, ref ).id != kTableNodeTableFieldId ) { r.offset = at; return true; } + uint64_t n = 0; + if ( !r.get( n, 32 ) ) { return false; } + count = (int64_t) n; + return true; +} +// AnnounceMeasure is the announcement's byte count, which is a constant of the +// unit and not a walk. +inline int64_t AnnounceMeasure() { return kTableAnnounceBytes; } + +// Announce writes the announcement into the caller's buffer and answers the +// bytes written, which is exactly AnnounceMeasure's answer, or -1 when the +// buffer is too small. It allocates nothing and walks nothing. +inline int64_t Announce( uint8_t * buffer, int64_t capacity ) +{ + if ( buffer == NULL || capacity < kTableAnnounceBytes ) { return -1; } + memcpy( buffer, kTableAnnounce, (size_t) kTableAnnounceBytes ); + return kTableAnnounceBytes; +} + +// THE PRIMITIVE IS A BATCH (§3.3): a number of bodies of ONE ROOT in one +// buffer, one count and one continuous bit stream with no alignment between +// them. A single message is the batch of one. +// +// The count rides ahead of the bodies, so a writer declares it at Begin and +// End refuses a batch that wrote a different number: a count the bodies do not +// match is not a wire this writer will hand anyone. +struct TableMessageBatch +{ + TableBitWriter w; + int64_t declared = 0; + int64_t written = 0; +}; + +inline bool TableMessageBatchBegin( TableMessageBatch & batch, uint8_t * buffer, int64_t capacity, int64_t bodies ) +{ + if ( buffer == NULL || capacity < 1 || bodies < 1 || bodies > kTableMessageBatchMax ) { return false; } + buffer[0] = kTableWireMessageForm; // the FORM BYTE is read first, always + batch.w = TableBitWriter( buffer + 1, capacity - 1 ); + batch.declared = bodies; + batch.written = 0; + batch.w.put( (uint64_t) ( bodies - 1 ), 8 ); // a ranged integer over [1, 256] + return true; +} + +// TableMessageBatchEnd zero-fills to the next byte, the one alignment a batch +// spends at its end, and answers the whole batch's byte count, or -1. +inline int64_t TableMessageBatchEnd( TableMessageBatch & batch ) +{ + if ( batch.written != batch.declared || batch.w.overflow ) { return -1; } + batch.w.align(); + if ( batch.w.overflow ) { return -1; } + return 1 + batch.w.bits / 8; +} + +// TableMessageBatchBytes is a batch's byte count from its bodies' BIT count, +// which is what every MeasureMessages answers. +inline int64_t TableMessageBatchBytes( int64_t body_bits ) +{ + if ( body_bits < 0 ) { return -1; } + return 1 + ( 8 + body_bits + 7 ) / 8; +} + +// The reading half. A batch is opened once and its bodies are then read in +// order into the storage the caller sized for them: which root a batch carries +// is the APPLICATION's and never this wire's. +struct TableMessageBatchReader +{ + TableBitReader r; + const TableVocabulary * vocabulary = NULL; + TableReport * report = NULL; + int64_t remaining = 0; + // THE SINK A CALLER THAT PASSED NO REPORT WRITES INTO IS THE READER'S OWN, + // not a static: a static is shared mutable state, and two threads reading + // two batches without reports would be writing one object. LoadMessages + // already keeps its sink locally, for the same reason. + TableReport ignored; +}; + +// TableMessageBatchOpen answers the batch's body count, or -1 with the refusal +// on the report: a form byte this reader does not carry, or a body from a peer +// that never announced. +inline int64_t TableMessageBatchOpen( TableMessageBatchReader & br, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + br.report = report != NULL ? report : &br.ignored; + br.vocabulary = &vocabulary; + if ( bytes < 1 ) { br.report->malformed = true; return -1; } + if ( buffer[0] != kTableWireMessageForm ) { br.report->refused = true; br.report->reason = newer_form; return -1; } + if ( !vocabulary.announced ) { br.report->refused = true; br.report->reason = no_vocabulary; return -1; } + br.r = TableBitReader( buffer + 1, bytes - 1 ); + uint64_t count = 0; + if ( !br.r.get( count, 8 ) ) { br.report->malformed = true; return -1; } + br.remaining = (int64_t) count + 1; + return br.remaining; +} + +// TableMessageRefuseBatch is the batch's own refusal (§3.3): M above 256 on the +// write side, or above the caller's capacity on the read side. Nothing is +// written or decoded, no counter moves, and the reason names it. +inline void TableMessageRefuseBatch( TableReport * report ) +{ + if ( report == NULL ) { return; } + report->refused = true; + report->reason = batch_too_large; +} + +// TableMessageBatchClose verifies the trailing pad, and that NOTHING FOLLOWS +// IT: the batch ends at the pad to the byte boundary, and a buffer with bytes +// left over describes no batch this reader can name (§3.3). +inline bool TableMessageBatchClose( TableMessageBatchReader & br ) +{ + if ( br.remaining != 0 || !br.r.align() || br.r.offset != br.r.bits ) { br.report->malformed = true; return false; } + return true; +} + + +// An ENUM-KEYED array's storage: E.Max slots, ONE PER NAMED VARIANT, with the +// key k at index k-1 — the storage SHIFTS LEFT and nothing is stored for None. +// +// NOTHING OUTSIDE THE ARRAY NAMES ITS SIZE: the extent is derived from E::Max +// here and nowhere else, so there is no size parameter to spell and no count a +// consumer could put one out of step with. +// +// NONE IS THE NULL KEY: it names no slot, it never rides on the wire, a stored +// key of 0 is malformed, and INDEXING BY IT IS A PROGRAM ERROR IN EVERY +// CONFIGURATION — caught by operator[], which cannot see a runtime key any +// earlier, and REFUSED UNCONDITIONALLY. A KEY PAST Max IS THE SAME ERROR for +// the same reason — it names a variant this enum does not have — so the +// accessor refuses BOTH ENDS. NDEBUG does not remove the compare: +// there is NO UB PATH here in any build. ITERATION is still the surface a +// consumer of the whole array wants: begin()/end() walk every stored slot and +// yield the KEY, 1..E.Max, so a call site writes no bound, no cast, no shift +// and no None question. +template +struct TableKeyed +{ + // the extent is the enum's, derived here and named nowhere else + static constexpr int32_t kSlots = (int32_t) E::Max; + + T slots[kSlots] = {}; + + T & operator[]( E key ) + { + RefuseKey( key ); + return slots[ (int32_t) key - 1 ]; + } + const T & operator[]( E key ) const + { + RefuseKey( key ); + return slots[ (int32_t) key - 1 ]; + } + + // THE REFUSAL, and it stands in EVERY BUILD, AT BOTH ENDS. The storage + // holds one slot per NAMED variant: nothing for None below it and nothing + // above Max, so a build that skipped this compare would index one element + // BEFORE the array or past its end — undefined behavior in the + // configuration a game ships. Either key is a program error, so the + // accessor ends the program rather than reading something. The assert + // carries the message where a debugger can read it and NDEBUG removes + // that; the fatal is what stands after it. BOTH GO THROUGH THE HOOKS — + // define schema_assert and schema_fatal and this refusal lands in your + // own handler. + // + // ONE UNSIGNED COMPARE COVERS BOTH ENDS: the storage index is key - 1, and + // None's is -1, which wraps above kSlots unsigned. The cost is one + // perfectly-predicted compare, on a path that reads config. + static void RefuseKey( E key ) + { + if ( (uint32_t) ( (int32_t) key - 1 ) >= (uint32_t) kSlots ) + { + schema_assert( false && "an enum-keyed array holds one slot per named variant: None keys none, and neither does a key past Max" ); + schema_fatal(); + } + } + + // ---- iteration: keys 1..E.Max over storage 0..E.Max-1, key beside element ---- + // + // The entry is a key and a REFERENCE, handed out BY VALUE the way any + // proxy is: for ( auto [ key, element ] : keyed ) binds element to the + // reference member, so iterating fills the array as well as reads it. + // auto & [ key, element ] does NOT compile, and that is by design — a + // non-const lvalue reference cannot bind to the proxy. Write + // auto [ ... ], or auto && [ ... ] if you prefer the reference form. + // + // THE ITERATORS CARRY NO iterator_traits TYPEDEFS. They bought std::distance + // and the forward-pass algorithms for an audience that does not call them, + // and the they need is the single most expensive include the + // generated corpus had: 536 headers and 986 KB, in a header whose whole + // remaining set is 123. begin(), end() and size() need none of it. + + struct Entry { E key; T & element; }; + struct ConstEntry { E key; const T & element; }; + + struct Iterator + { + T * slots; + int32_t index; // the STORAGE index; the key it holds is index + 1 + Entry operator*() const { return Entry{ (E) ( index + 1 ), slots[index] }; } + Iterator & operator++() { index++; return *this; } + bool operator==( const Iterator & other ) const { return index == other.index; } + bool operator!=( const Iterator & other ) const { return index != other.index; } + }; + + struct ConstIterator + { + const T * slots; + int32_t index; // the STORAGE index; the key it holds is index + 1 + ConstEntry operator*() const { return ConstEntry{ (E) ( index + 1 ), slots[index] }; } + ConstIterator & operator++() { index++; return *this; } + bool operator==( const ConstIterator & other ) const { return index == other.index; } + bool operator!=( const ConstIterator & other ) const { return index != other.index; } + }; + + Iterator begin() { return Iterator{ slots, 0 }; } + Iterator end() { return Iterator{ slots, kSlots }; } + ConstIterator begin() const { return ConstIterator{ slots, 0 }; } + ConstIterator end() const { return ConstIterator{ slots, kSlots }; } +}; + +inline float table_bits_to_float( uint32_t bits ) { float f; memcpy( &f, &bits, 4 ); return f; } +inline uint32_t table_float_to_bits( float f ) { uint32_t b; memcpy( &b, &f, 4 ); return b; } +inline double table_bits_to_double( uint64_t bits ) { double d; memcpy( &d, &bits, 8 ); return d; } +inline uint64_t table_double_to_bits( double d ) { uint64_t b; memcpy( &b, &d, 8 ); return b; } + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_PRIMITIVES + +#ifndef MAPDEMO_SCHEMA_TABLE_ARENA +#define MAPDEMO_SCHEMA_TABLE_ARENA + +namespace mapdemo { + +// ---- variable-length tables: tuning constants (docs/SPEC-TABLES.md) ---- +// +// The segment size and the count multiply to exactly 2^32: the u32 reference +// is the arena's hard ceiling, and these constants saturate it rather than +// leaving address space unreachable. Slab handout costs one atomic per slab, +// so per-node allocation costs no synchronization at all. + +static const uint32_t kTableSegmentBits = 22; // 4 MiB segments +static const uint32_t kTableSegmentSize = 1u << kTableSegmentBits; +static const uint32_t kTableSegmentMask = kTableSegmentSize - 1u; +static const uint32_t kTableMaxSegments = 1u << ( 32 - kTableSegmentBits ); // 1024 -> 4 GiB +static const uint32_t kTableSlabBytes = 64u * 1024u; // one atomic per slab +static const uint32_t kTableAlign = 8; // every node starts 8-aligned +static const uint32_t kTableAllocFailed = 0xFFFFFFFFu; + +// ---- THE CALLER'S ALLOCATOR (docs/SPEC-TABLES.md §6.5) ---- +// +// Every allocation the variable-length runtime makes goes through one of +// these — the arena's segments, the pack walk's identity map, the numbering's +// entry array, the packed region, and the tool path's node directory. There is +// no other call to the C library on this path, so a counting allocator sees +// every byte and a game's own heap can own all of it. +// +// It is the shape TableBlockAllocator already has (§19.1): two function +// pointers and a context the caller carries. What it adds is a CONTRACT ON +// alloc — the bytes come back ZEROED. Lock copies whole nodes, PADDING +// INCLUDED, so anything left uninitialized reaches a packed region; the default +// pair reaches that through calloc, which costs nothing measurable because a +// fresh segment is untouched pages either way. +struct TableAllocator +{ + void * ( *alloc )( void * context, int64_t bytes ); // ZEROED bytes, NULL on failure + void ( *free )( void * context, void * pointer ); + void * context; +}; + +// The default pair, and it is the one every entry point takes when the caller +// names none. It calls schema_allocate / schema_release, so a program with its +// own C-library replacement can move the floor without writing a struct at all. +inline void * table_default_alloc( void * context, int64_t bytes ) { (void) context; return schema_allocate( bytes ); } +inline void table_default_free( void * context, void * pointer ) { (void) context; schema_release( pointer ); } + +inline TableAllocator TableDefaultAllocator() +{ + TableAllocator allocator; + allocator.alloc = table_default_alloc; + allocator.free = table_default_free; + allocator.context = NULL; + return allocator; +} + +// ---- TableRef: a relocatable reference (never a machine pointer) ---- +// +// Two encodings, one slot, and the FORM says which is in force: +// +// in the arena — the node's arena offset (segment index in the high bits) +// in a region — the SELF-RELATIVE byte delta from this slot's own address, +// so a deref is one add, needs no base pointer, and a whole +// region relocates by memcpy with zero fix-up +// +// 0 is null in both, and a slot can never name the node that contains it, so +// zero names nothing real in either form. +// +// A REGION DELTA HAS NO REQUIRED SIGN (§6.3). A region is packed depth-first, +// so a node's FIRST reference points forward; every LATER reference to that +// same node points BACK at the one body it already has, which is exactly what +// makes one node one node in a region. Sharing and a back-reference are the +// same fact, and nothing validates a reference by its sign. +// +// IT IS EIGHT BYTES, SIGNED, so ONE REGION REACHES EVERYTHING (§6.3, §7): a +// four-byte slot bounded a region at 2 GiB, and the scale a cook exists for is +// *"100mbs or many gigabytes of data in Assets.bin"*. +struct TableRef +{ + int64_t value = 0; + bool null() const { return value == 0; } +}; + +// TableSlot is what Alloc hands back: usable as the node pointer (write +// fields through it) AND as the reference to store in a pointer field. +template struct TableSlot +{ + T * ptr = NULL; + TableRef ref; + T * operator->() const { return ptr; } + T & operator*() const { return *ptr; } + operator T *() const { return ptr; } + operator TableRef() const { return ref; } + bool null() const { return ptr == NULL; } +}; + +inline uint32_t TableAlignUp( uint32_t bytes ) { return ( bytes + kTableAlign - 1 ) & ~( kTableAlign - 1 ); } +inline int64_t TableAlignUp64( int64_t bytes ) { return ( bytes + kTableAlign - 1 ) & ~( int64_t( kTableAlign ) - 1 ); } + +// ---- a BYTE BUFFER's node (docs/SPEC-TABLES.md §2.5, §6.3) ---- +// +// A *bytes or *string slot is a TableRef like every pointer slot, and it names +// a BLOB NODE: this eight-byte header and then the bytes, at offset eight so +// the data is eight-aligned. A *string blob carries one more zero byte after +// its data, so a region hands back a C string with no copy. The node's extent +// is the header plus its bytes, rounded to the arena's alignment like every +// node's; on the wire it is a record whose body is the bytes (§3.1). +struct TableBlob +{ + uint32_t length; + uint32_t zero; +}; + +static const int64_t kTableBlobHeader = 8; // length (u32), then four zero bytes +static const int64_t kTableBlobMaxLength = 0xFFFFFFFF; // a record's length is a u32 (§3.1) + +// the node's storage: the header, the bytes, a string's terminator, rounded +// to the arena's alignment like every node +inline int64_t TableBlobStorage( int64_t length, bool terminated ) +{ + return TableAlignUp64( kTableBlobHeader + length + ( terminated ? 1 : 0 ) ); +} + +// What a read answers: a pointer INTO the region and the length, NULL and +// zero for a null slot. Off a locked region, a loaded one or an opened cook +// the pointer is one add from the slot, and nothing is copied. +struct TableBytesView +{ + const uint8_t * data; + int64_t length; +}; + +struct TableStringView +{ + const char * data; // zero-terminated + int64_t length; +}; + +// What AllocBytes and AllocString hand back: the bytes to write through, the +// length asked for, and the reference to store in the slot — the three +// answers TableSlot gives for a table node. +struct TableBytesSlot +{ + uint8_t * data = NULL; + int64_t length = 0; + TableRef ref; + bool null() const { return data == NULL; } + operator TableRef() const { return ref; } +}; + +struct TableStringSlot +{ + char * data = NULL; // room for length bytes and the terminator, already zero + int64_t length = 0; + TableRef ref; + bool null() const { return data == NULL; } + operator TableRef() const { return ref; } +}; + +// ---- the arena: segmented, slab-handed, lock-free by ownership ---- +// +// Allocation is thread-local inside a worker's slab — no atomics on the node +// path. A worker takes its next slab with ONE compare-exchange, and a new +// segment is published with one more. Nothing ever moves: a segment, once +// allocated, lives untouched until the arena is torn down, so a T* obtained +// from Alloc stays valid while other workers allocate, and an offset stays +// correct while the arena grows. +// +// The model this DELIBERATELY refuses: one buffer under a lock, grown by +// realloc. A realloc moves the buffer under workers mid-write; offsets fix +// identity but not the raw references already resolved from them, and the +// resulting corruption is invisible until much later. Segments never move, so +// that bug class cannot be written here. +// +// Slack: at most one slab tail per worker plus one slab per segment (a slab +// that will not fit is skipped rather than split), i.e. under 2% of a segment +// plus threads x 64 KiB. That is the price of never synchronizing per node. +struct TableArena +{ + std::atomic segments[ kTableMaxSegments ]; + std::atomic cursor; // (segment << kTableSegmentBits) | bytes handed out + bool locked = false; // MONOTONIC: Lock() is one-way, there is no unlock + // THE ARENA CARRIES ITS OWN, so everything downstream of a builder — + // segments, pack map, numbering, region, node directory — allocates through + // the one pair the caller named, with nothing to thread by hand. + TableAllocator allocator; +}; + +inline void TableArenaInit( TableArena & arena, TableAllocator allocator ) +{ + for ( uint32_t i = 0; i < kTableMaxSegments; i++ ) + { + arena.segments[i].store( NULL, std::memory_order_relaxed ); + } + arena.cursor.store( 0, std::memory_order_relaxed ); + arena.locked = false; + arena.allocator = allocator; +} + +inline void TableArenaShutdown( TableArena & arena ) +{ + for ( uint32_t i = 0; i < kTableMaxSegments; i++ ) + { + uint8_t * segment = arena.segments[i].exchange( NULL, std::memory_order_acq_rel ); + if ( segment != NULL ) { arena.allocator.free( arena.allocator.context, segment ); } + } + arena.cursor.store( 0, std::memory_order_relaxed ); +} + +// one L1 load plus an add: the segment table is 8 KiB and stays hot +inline uint8_t * TableArenaAt( const TableArena & arena, uint32_t offset ) +{ + return arena.segments[ offset >> kTableSegmentBits ].load( std::memory_order_relaxed ) + ( offset & kTableSegmentMask ); +} + +// TableArenaGrabSlab hands one worker its next private slab. Returns +// kTableAllocFailed when the arena's address space or the allocator is +// exhausted — a loud refusal, never a silent smaller slab. +inline uint32_t TableArenaGrabSlab( TableArena & arena ) +{ + for ( ;; ) + { + uint32_t cursor = arena.cursor.load( std::memory_order_acquire ); + uint32_t segment = cursor >> kTableSegmentBits; + uint32_t used = cursor & kTableSegmentMask; + // strictly less: a slab is never split across segments, and the tail + // is the documented slack + if ( used + kTableSlabBytes < kTableSegmentSize ) + { + if ( arena.segments[segment].load( std::memory_order_acquire ) == NULL ) + { + // THE SEGMENT COMES BACK ZEROED, which is the allocator's + // contract and not an extra pass here: Lock copies whole nodes, + // PADDING INCLUDED, so anything uninitialized reaches a packed + // region. Value-initializing a node with placement new zeroes + // its MEMBERS and not its padding, so the zeroing has to happen + // at the segment or not at all. It costs nothing measurable: a + // fresh segment is untouched pages either way, and the default + // pair's calloc has the kernel hand them over zeroed. + uint8_t * memory = (uint8_t *) arena.allocator.alloc( arena.allocator.context, (int64_t) kTableSegmentSize ); + if ( memory == NULL ) { return kTableAllocFailed; } + uint8_t * expected = NULL; + if ( !arena.segments[segment].compare_exchange_strong( expected, memory, std::memory_order_acq_rel ) ) + { + // another worker published this segment first + arena.allocator.free( arena.allocator.context, memory ); + } + } + if ( arena.cursor.compare_exchange_weak( cursor, cursor + kTableSlabBytes, std::memory_order_acq_rel ) ) + { + return ( segment << kTableSegmentBits ) | used; + } + continue; + } + uint32_t next_segment = segment + 1; + if ( next_segment >= kTableMaxSegments ) { return kTableAllocFailed; } // 4 GiB: the u32 reference's ceiling + arena.cursor.compare_exchange_weak( cursor, next_segment << kTableSegmentBits, std::memory_order_acq_rel ); + } +} + +// TableArenaGrabSpan reserves a SPAN of the arena's address space for one node +// larger than a slab — a BYTE BUFFER of any size (docs/SPEC-TABLES.md §2.5) — +// and allocates it as one contiguous block. It takes whole segment indices +// from the cursor, starting at the index after the cursor's so nothing else +// is ever handed out inside the span, and publishes the block under the first +// of them; the indices the span covers past that one stay NULL, which is +// enough, because only a node's START is ever resolved through the segment +// table and a blob's bytes follow its header inside the one allocation. The +// unused tail of the segment the cursor was in is slack, like a slab tail. +// Returns kTableAllocFailed when the address space or the allocator is +// exhausted — a loud refusal, never a smaller blob. +inline uint32_t TableArenaGrabSpan( TableArena & arena, int64_t bytes ) +{ + if ( bytes <= 0 || bytes > ( (int64_t) kTableMaxSegments - 2 ) * (int64_t) kTableSegmentSize ) { return kTableAllocFailed; } + const uint32_t spanned = (uint32_t) ( ( bytes + kTableSegmentSize - 1 ) >> kTableSegmentBits ); + for ( ;; ) + { + uint32_t cursor = arena.cursor.load( std::memory_order_acquire ); + uint32_t start = ( cursor >> kTableSegmentBits ) + 1; + if ( start + spanned >= kTableMaxSegments ) { return kTableAllocFailed; } // 4 GiB: the u32 reference's ceiling + uint32_t next = ( start + spanned ) << kTableSegmentBits; + if ( !arena.cursor.compare_exchange_weak( cursor, next, std::memory_order_acq_rel ) ) { continue; } + // the span is this worker's now: nothing else can publish under its + // first index, so a plain store suffices, and the block comes back + // ZEROED like every segment — the blob's bytes and its tail are zeros + // until written + uint8_t * memory = (uint8_t *) arena.allocator.alloc( arena.allocator.context, bytes ); + if ( memory == NULL ) { return kTableAllocFailed; } + arena.segments[start].store( memory, std::memory_order_release ); + return start << kTableSegmentBits; + } +} + +// ---- TableWorker: one thread's allocation front ---- +// +// The threading contract, stated plainly: +// * Alloc on YOUR OWN worker is safe concurrently with any other worker's. +// No locks, no atomics per node. +// * Writing fields of a node ANOTHER worker allocated is your own +// synchronization problem — this runtime does not arbitrate it. +// * Lock and Save are single-threaded: call them after the workers have +// joined. +struct TableWorker +{ + TableArena * arena = NULL; + uint32_t next = 0; + uint32_t end = 0; + + template TableSlot Alloc() + { + static_assert( alignof( T ) <= kTableAlign, "a table node's alignment must fit the arena's" ); + TableSlot slot; + if ( arena == NULL || arena->locked ) { return slot; } + uint32_t bytes = TableAlignUp( (uint32_t) sizeof( T ) ); + if ( bytes > kTableSlabBytes ) { return slot; } // a node larger than a slab: refused, never split + if ( end == 0 || next + bytes > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return slot; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + uint32_t at = next; + next += bytes; + // A NODE IS BORN IN TWO HALVES: start its lifetime in the raw + // storage, then write the declared defaults ONE MEMBER AT A TIME. + // + // It is "T", not "T{}". Value-initialising the whole aggregate says + // the same thing and costs cl O(BYTES) TO COMPILE — it expands element + // by element in its front end — while both halves here cost + // O(declarations). The slab cap below refuses a large node at RUN + // TIME and bounds nothing at compile time: the cost is paid by + // whatever T a caller instantiates this with. + // Padding is not the difference: value-initialisation zeroes MEMBERS + // and not padding either way, which is why the segment is calloc'd. + // + // TableReset is an OVERLOAD SET, one per closure member, reached from + // this template by argument-dependent lookup on T's own namespace — + // Alloc is a template and cannot spell Reset. + // + // The reset is here because ONE DEFINITION SAYS WHAT THE DECLARED + // DEFAULTS ARE, and it is Reset. Default-initialisation lands on + // the same values today, because a member with a non-zero default + // carries a member initializer that says so — but that is the class + // definition agreeing with Reset, not the arena reading it, and #320's + // fix was itself a pass that MOVED initialisation between the two. + // The arena reads the definition. + slot.ptr = new ( TableArenaAt( *arena, at ) ) T; + TableReset( *slot.ptr ); + slot.ref.value = at; + return slot; + } + + // Alloc a BYTE BUFFER's node of exactly length bytes (docs/SPEC-TABLES.md + // §2.5): the blob header and its bytes, zeroed, in this thread's slab when + // it fits and in a span of the arena's own when it does not. NULL is the + // arena locked, a length below zero or past a record's u32, or the + // allocator refusing. The offset comes back for the reference. + TableBlob * AllocBlob( int64_t length, bool terminated, uint32_t & at ) + { + at = 0; + if ( arena == NULL || arena->locked ) { return NULL; } + if ( length < 0 || length > kTableBlobMaxLength ) { return NULL; } + const int64_t bytes = TableBlobStorage( length, terminated ); + if ( bytes > (int64_t) kTableSlabBytes ) + { + at = TableArenaGrabSpan( *arena, bytes ); + if ( at == kTableAllocFailed ) { at = 0; return NULL; } + } + else + { + if ( end == 0 || next + (uint32_t) bytes > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return NULL; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + at = next; + next += (uint32_t) bytes; + } + TableBlob * blob = (TableBlob *) TableArenaAt( *arena, at ); + blob->length = (uint32_t) length; // the bytes after it are the segment's zeros + blob->zero = 0; + return blob; + } + + // RAW, ZEROED storage of the bytes asked for, at the alignment asked for: a MAP's or a LIST's builder + // head and its segments (docs/SPEC-TABLES.md §2.8, §2.9). It is not a node: it carries + // no type id, takes no index and has no Reset, so it goes through the same + // slab and span the blob path uses rather than through Alloc. + uint8_t * AllocRaw( int64_t bytes, int64_t align, uint32_t & at ) + { + at = 0; + if ( arena == NULL || arena->locked ) { return NULL; } + if ( bytes <= 0 || align > (int64_t) kTableAlign ) { return NULL; } + const int64_t rounded = TableAlignUp64( bytes ); + if ( rounded > (int64_t) kTableSlabBytes ) + { + at = TableArenaGrabSpan( *arena, rounded ); + if ( at == kTableAllocFailed ) { at = 0; return NULL; } + return TableArenaAt( *arena, at ); + } + if ( end == 0 || next + (uint32_t) rounded > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return NULL; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + at = next; + next += (uint32_t) rounded; + return TableArenaAt( *arena, at ); // the segment came back zeroed + } + // a *bytes node: the bytes to write through, and the reference to store + TableBytesSlot AllocBytes( int64_t length ) + { + TableBytesSlot slot; + uint32_t at = 0; + TableBlob * blob = AllocBlob( length, false, at ); + if ( blob == NULL ) { return slot; } + slot.data = (uint8_t *) ( blob + 1 ); + slot.length = length; + slot.ref.value = at; + return slot; + } + + // a *string node: room for length bytes and the zero byte after them + TableStringSlot AllocString( int64_t length ) + { + TableStringSlot slot; + uint32_t at = 0; + TableBlob * blob = AllocBlob( length, true, at ); + if ( blob == NULL ) { return slot; } + slot.data = (char *) ( blob + 1 ); + slot.length = length; + slot.ref.value = at; + return slot; + } +}; + +// ---- TablePackMap: the pack walk's identity map (docs/SPEC-TABLES.md §3.1, §6.2) ---- +// +// ONE ENTRY PER REACHABLE NODE, and that map IS identity: a node must know +// where it landed to be named a second time, so Lock packs a shared node ONCE +// and every later reference resolves to the one body it already has. That is +// the same first-visit numbering the wire uses, so the pack order and the node +// order are one order. +// +// COLOURING AN ENTRY WHILE ITS DESCENT IS OPEN COSTS ONE BIT, and it is what +// makes a data cycle free to refuse: a reference to an entry still open is a +// cycle, and Lock returns failure rather than recursing away. The ROOT's entry +// is open for the whole walk. +// +// The map is proportional to NODES, never to bytes, and it lives on the +// AUTHORING side, where §6.5 licenses allocation. Nothing on the reading path +// ever builds one. +struct TablePackEntry +{ + const void * key; // the node's address in the graph being packed + int64_t offset; // where that node landed in the region + uint8_t open; // its descent is still open: a reference here is a cycle +}; + +struct TablePackMap +{ + TablePackEntry * entries = NULL; + int64_t capacity = 0; // a power of two, or zero while empty + int64_t count = 0; + TableAllocator allocator; // the caller's, carried from the walk that built it +}; + +inline void TablePackMapInit( TablePackMap & map, TableAllocator allocator ) +{ + map.entries = NULL; + map.capacity = 0; + map.count = 0; + map.allocator = allocator; +} + +inline void TablePackMapShutdown( TablePackMap & map ) +{ + map.allocator.free( map.allocator.context, map.entries ); + TablePackMapInit( map, map.allocator ); +} + +// The two walks behind Lock re-derive the SAME map from the same graph — the +// numbering is never carried between them (§3.1) — so the second starts from +// an empty map and keeps the capacity the first paid for. +inline void TablePackMapReset( TablePackMap & map ) +{ + if ( map.entries != NULL ) { memset( map.entries, 0, (size_t) map.capacity * sizeof( TablePackEntry ) ); } + map.count = 0; +} + +// open addressing, linear probing, a multiply-shift hash over the address: a +// node key is a pointer and its low bits are alignment, so the low bits alone +// would collide on every node of one type +inline int64_t TablePackMapSlot( const TablePackMap & map, const void * key ) +{ + uint64_t hash = (uint64_t) (uintptr_t) key; + hash *= 0x9E3779B97F4A7C15ull; + hash ^= hash >> 29; + int64_t mask = map.capacity - 1; + int64_t at = (int64_t) ( hash & (uint64_t) mask ); + while ( map.entries[at].key != NULL && map.entries[at].key != key ) + { + at = ( at + 1 ) & mask; + } + return at; +} + +inline TablePackEntry * TablePackMapFind( TablePackMap & map, const void * key ) +{ + if ( map.capacity == 0 ) { return NULL; } + TablePackEntry * entry = &map.entries[ TablePackMapSlot( map, key ) ]; + return entry->key == key ? entry : NULL; +} + +// QUADRUPLING, not doubling, and the reason is measured: growth rehashes every +// entry, and on a graph of 131,071 nodes the doubling schedule spent 45% of +// Lock in rehashing alone. Quadrupling from 1024 buys 1.35x on that graph and +// keeps the map NODE-proportional (§6.2) — under 128 bytes a node at its +// worst, right after a grow, and about 64 on average. +inline bool TablePackMapGrow( TablePackMap & map ) +{ + TablePackMap grown; + grown.allocator = map.allocator; + grown.capacity = map.capacity != 0 ? map.capacity * 4 : 1024; + grown.entries = (TablePackEntry *) map.allocator.alloc( map.allocator.context, grown.capacity * (int64_t) sizeof( TablePackEntry ) ); + if ( grown.entries == NULL ) { return false; } + for ( int64_t i = 0; i < map.capacity; i++ ) + { + if ( map.entries[i].key == NULL ) { continue; } + grown.entries[ TablePackMapSlot( grown, map.entries[i].key ) ] = map.entries[i]; + grown.count++; + } + map.allocator.free( map.allocator.context, map.entries ); + map = grown; + return true; +} + +// REACH a node: one probe answers both questions the walk has. A true "taken" +// says this is a FIRST visit, and the entry is now the node's, coloured open +// at "offset"; otherwise the entry is the one the node already has, and its +// open bit says cycle or sharing. NULL is an allocation failure, and it is a +// refusal like any other: Lock fails rather than packing a graph it cannot +// track. +// +// It is one call and not a find followed by an insert because the walk asks +// this question twice per node — once to measure, once to pack — and every +// probe is a miss into a table larger than L2. +inline TablePackEntry * TablePackMapReach( TablePackMap & map, const void * key, int64_t offset, bool & taken, int64_t & slot ) +{ + if ( ( map.count + 1 ) * 4 >= map.capacity * 3 ) // keep the load factor under three quarters + { + if ( !TablePackMapGrow( map ) ) { return NULL; } + } + slot = TablePackMapSlot( map, key ); + TablePackEntry * entry = &map.entries[slot]; + taken = entry->key != key; // an empty slot is a first visit; the key is never NULL + if ( taken ) + { + entry->key = key; + entry->offset = offset; + entry->open = 1; + map.count++; + } + return entry; +} + +// The descent finished: the node keeps its entry — identity outlives the +// descent — and stops being a cycle. The "hint" is the slot Reach returned, and it +// is checked against the key rather than trusted, so a rehash between the two +// costs a second probe instead of correctness. +inline void TablePackMapClose( TablePackMap & map, const void * key, int64_t hint ) +{ + if ( hint >= 0 && hint < map.capacity && map.entries[hint].key == key ) + { + map.entries[hint].open = 0; + return; + } + TablePackEntry * entry = TablePackMapFind( map, key ); + if ( entry != NULL ) { entry->open = 0; } +} + +// ---- resolution contexts: which encoding a walk is reading ---- + +struct TableArenaCtx { const TableArena * arena; }; +struct TableRegionCtx {}; + +// ---- a BYTE BUFFER's resolution (docs/SPEC-TABLES.md §2.5, §6.3) ---- +// +// The same two encodings a table pointer has, resolved the same way: a +// self-relative delta in a region — one add, no base — and an arena offset +// while the builder is mutable. The blob is reached through its header, and a +// view is the header plus eight and the header's first word. Nothing here +// allocates and nothing copies: off a locked region, a loaded one or an +// opened cook the view points INTO the region. +inline const TableBlob * TableBlobAt( const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) ( (const uint8_t *) &ref + ref.value ) : NULL; +} +inline const TableBlob * TableBlobAt( const TableRegionCtx &, const TableRef & ref ) { return TableBlobAt( ref ); } +inline const TableBlob * TableBlobAt( const TableArenaCtx & ctx, const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) TableArenaAt( *ctx.arena, (uint32_t) ref.value ) : NULL; +} +inline const TableBlob * TableBlobAt( const TableArena & arena, const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) TableArenaAt( arena, (uint32_t) ref.value ) : NULL; +} + +inline TableBytesView TableBytesViewOf( const TableBlob * blob ) +{ + TableBytesView view = { NULL, 0 }; + if ( blob != NULL ) { view.data = (const uint8_t *) ( blob + 1 ); view.length = (int64_t) blob->length; } + return view; +} +inline TableStringView TableStringViewOf( const TableBlob * blob ) +{ + TableStringView view = { NULL, 0 }; + if ( blob != NULL ) { view.data = (const char *) ( blob + 1 ); view.length = (int64_t) blob->length; } + return view; +} + +// the const form's hot path: one add, no base +inline TableBytesView TableBytesAt( const TableRef & ref ) { return TableBytesViewOf( TableBlobAt( ref ) ); } +inline TableStringView TableStringAt( const TableRef & ref ) { return TableStringViewOf( TableBlobAt( ref ) ); } +// and the context forms a walk uses: a region context, an arena context, or +// the arena itself while the builder is mutable +template inline TableBytesView TableBytesAt( const Ctx & ctx, const TableRef & ref ) { return TableBytesViewOf( TableBlobAt( ctx, ref ) ); } +template inline TableStringView TableStringAt( const Ctx & ctx, const TableRef & ref ) { return TableStringViewOf( TableBlobAt( ctx, ref ) ); } + +// allocate a blob in the arena and point the slot at it; the slot holds the +// arena offset, as every slot does while the builder is mutable +inline uint8_t * TableBytesEmplace( TableWorker & worker, TableRef & slot, int64_t length ) +{ + TableBytesSlot allocated = worker.AllocBytes( length ); + slot = allocated.ref; + return allocated.data; +} +// the text is copied in when one is given; a NULL text leaves the zeros for +// the caller to fill +inline char * TableStringEmplace( TableWorker & worker, TableRef & slot, const char * text, int64_t length ) +{ + TableStringSlot allocated = worker.AllocString( length ); + slot = allocated.ref; + if ( allocated.data != NULL && text != NULL && length > 0 ) { memcpy( allocated.data, text, (size_t) length ); } + return allocated.data; +} + +// ---- the FLAT NODE TABLE (docs/SPEC-TABLES.md §3.1) ---- +// +// A pointered save writes every reachable node ONCE, into a node table, and a +// pointer field rides as an INDEX into it under kind 17. The encoding is +// flat: no pointer edge is a nesting level, so a chain's length is not a depth, +// and two references to one node are one node. +// +// THE FIELD RIDES ONCE: an L with sixty-four bits of capability frames a +// numbering of any size, so the whole numbering is one contiguous payload and a +// save's node bodies have no aggregate ceiling. + +static const uint64_t kTableNodeIndexNull = 0; // absence and null are one value +static const uint64_t kTableNodeIndexRoot = 1; // the body that hosts the table + +// The not-materialized sentinel (§6.3): a record whose type id this build could +// not name. Distinct from every real offset including the root's 0, so an index +// resolving through it yields NULL and can never fabricate the root. +static const uint64_t kTableNodeAbsent = 0xFFFFFFFFFFFFFFFFull; + +// What a node's storage answers when the FRAMING ITSELF is refused rather than +// merely unnameable: a count its L cannot carry, one above the int32 cap, or a +// blob past the size cap (docs/SPEC-TABLES.md §3.1, §6.5). An unnameable type +// id commands no storage and keeps its index. This one makes the whole measure +// answer -1 with its reason. +static const int64_t kTableNodeRefused = -2; + +// ---- the numbering, on the SAVE side ---- +// +// One entry per reachable node in FIRST-VISIT order, so entry k is node index +// k + 2. The two thunks are what let one loop write a table of mixed types: the +// numbering walk knows each target's type STATICALLY at the site it numbers it, +// so it stores the instantiation there and the loop never asks what a node is. +struct TableNumbering; + +struct TableNodeEntry +{ + const void * node; + uint64_t type_id; + // the type id's MESSAGE-FORM SLOT (docs/SPEC-TABLES.md §3.3), stored where + // the numbering walk stores the id itself and for the same reason: the + // target's type is known STATICALLY at the site that numbers it, so a + // form 2 save reads the slot out of the entry instead of looking an id up. + // Every pointer target's type id is an entry of the announcement, which is + // what makes the slot a compile-time fact of a POINTERED message too. + uint64_t type_slot; + int64_t ( * measure )( const void * ctx, const TableNumbering & numbering, TableIds & ids, const void * node ); + bool ( * save )( const void * ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const void * node ); + // the same two over the MESSAGE FORM (docs/SPEC-TABLES.md §3.3): a bitpacked + // body at a bit position, its pointer indices at the width the node count + // settled + int64_t ( * message_measure )( const void * ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const void * node ); + bool ( * message_save )( const void * ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const void * node ); +}; + +struct TableNumbering +{ + TablePackMap seen; // node -> index; the ROOT is index 1, open for the whole walk + TableNodeEntry * entries = NULL; + int64_t count = 0; + int64_t capacity = 0; +}; + +// The numbering allocates through the map's pair rather than carrying a second +// copy of it: one numbering is one walk, and a walk has one allocator. +inline void TableNumberingInit( TableNumbering & n, TableAllocator allocator ) +{ + TablePackMapInit( n.seen, allocator ); + n.entries = NULL; + n.count = 0; + n.capacity = 0; +} + +inline void TableNumberingShutdown( TableNumbering & n ) +{ + TableAllocator allocator = n.seen.allocator; + TablePackMapShutdown( n.seen ); + allocator.free( allocator.context, n.entries ); + n.entries = NULL; + n.count = 0; + n.capacity = 0; +} + +// The index a numbered node was given, for the save that writes it into a +// pointer slot. False means the two walks disagree about the graph, which is a +// refusal and never a guess. +inline bool TableNumberingIndex( const TableNumbering & n, const void * node, uint64_t & index ) +{ + if ( n.seen.capacity == 0 ) { return false; } + const TablePackEntry & entry = n.seen.entries[ TablePackMapSlot( n.seen, node ) ]; + if ( entry.key != node ) { return false; } + index = (uint64_t) entry.offset; + return true; +} + +inline bool TableNumberingAppend( TableNumbering & n, const TableNodeEntry & entry ) +{ + if ( n.count == n.capacity ) + { + // GROW BY COPY, never by realloc: the allocator hook is a PAIR, and a + // game's heap is not required to have a resize primitive at all. The + // schedule quadruples, so the copying is amortized to a constant per + // entry and the growth is the same growth it always was. + int64_t capacity = n.capacity != 0 ? n.capacity * 4 : 256; + TableAllocator allocator = n.seen.allocator; + TableNodeEntry * grown = (TableNodeEntry *) allocator.alloc( allocator.context, capacity * (int64_t) sizeof( TableNodeEntry ) ); + if ( grown == NULL ) { return false; } + if ( n.entries != NULL ) + { + memcpy( grown, n.entries, (size_t) n.count * sizeof( TableNodeEntry ) ); + allocator.free( allocator.context, n.entries ); + } + n.entries = grown; + n.capacity = capacity; + } + n.entries[n.count++] = entry; + return true; +} + +// The thunks the numbering stores. Each resolves to the closure member's own +// MeasureBody / SaveBodyFields through an overload set in the member's DECLARING +// file, reached by argument-dependent lookup at instantiation — the same bridge +// the arena's TableReset uses, and the reason a numbering may span the files of +// one unit without any file naming another's members. +template +inline int64_t TableNodeMeasureThunk( const void * ctx, const TableNumbering & numbering, TableIds & ids, const void * node ) +{ + return TableNodeMeasure( *(const Ctx *) ctx, numbering, ids, *(const T *) node ); +} + +template +inline bool TableNodeSaveThunk( const void * ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const void * node ) +{ + return TableNodeSave( *(const Ctx *) ctx, numbering, w, ids, *(const T *) node ); +} + +template +inline int64_t TableNodeMessageMeasureThunk( const void * ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const void * node ) +{ + return TableNodeMessageMeasure( *(const Ctx *) ctx, numbering, index_bits, at, *(const T *) node ); +} + +template +inline bool TableNodeMessageSaveThunk( const void * ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const void * node ) +{ + return TableNodeMessageSave( *(const Ctx *) ctx, numbering, index_bits, w, *(const T *) node ); +} +// ---- a BYTE BUFFER's record (docs/SPEC-TABLES.md §2.5, §3.1) ---- +// +// A blob rides as a node record under one of two RESERVED type ids — the fold +// a table's name takes, over the keywords "bytes" and "string", which no table +// can be named — with the bytes as its body and nothing framed inside. These +// two thunks are what the numbering stores for a blob, as it stores a +// member's codec for a table: the length, and the bytes verbatim. +static const uint64_t kTableBytesTypeId = 0x2f2ec0474f1c4fe4ull; // fnv1a64( "bytes" ) +static const uint64_t kTableStringTypeId = 0x704be0d8faaffc58ull; // fnv1a64( "string" ) + +template +inline int64_t TableBlobMeasureThunk( const void *, const TableNumbering &, TableIds &, const void * node ) +{ + return (int64_t) ( (const TableBlob *) node )->length; +} + +template +inline bool TableBlobSaveThunk( const void *, const TableNumbering &, TableWriter & w, TableIds &, const void * node ) +{ + const TableBlob * blob = (const TableBlob *) node; + w.raw( (const void *) ( blob + 1 ), (int64_t) blob->length ); + return true; +} + +// and the same two on the MESSAGE FORM (§3.3): a blob record is its length at +// thirty-two raw bits, an ALIGN, then the bytes verbatim +template +inline int64_t TableBlobMessageMeasureThunk( const void *, const TableNumbering &, int64_t, int64_t at, const void * node ) +{ + const int64_t length = (int64_t) ( (const TableBlob *) node )->length; + return 32 + TableAlignBits( at + 32 ) + length * 8; +} + +template +inline bool TableBlobMessageSaveThunk( const void *, const TableNumbering &, int64_t, TableBitWriter & w, const void * node ) +{ + const TableBlob * blob = (const TableBlob *) node; + w.put( (uint64_t) blob->length, 32 ); + w.align(); + w.putbytes( (const uint8_t *) ( blob + 1 ), (int64_t) blob->length ); + return !w.overflow; +} +// TableNodeTableMeasure and TableNodeTableSave are the framing, and they are +// ONE fill rule written twice — measure derives it from the graph and save +// derives the same one, which is what makes measure == save hold across a +// pointer graph (§3.1). +// +// The field rides ONCE, under the reserved id, kind 12: the payload opens with +// the count and then carries the records back to back, each a type id +// REFERENCE, a length and a body. The reserved id is interned BEFORE the +// records, and a record's type id before its body, which is the first-use order +// the trailer is written in (§3). +template +inline int64_t TableNodeTablePayload( const Ctx & ctx, TableIds & ids, const TableNumbering & n ) +{ + int64_t payload = TableLebBytes( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + payload += TableLebBytes( ids.ref( n.entries[k].type_id ) ); + const int64_t body = n.entries[k].measure( (const void *) &ctx, n, ids, n.entries[k].node ); + if ( body < 0 ) { return -1; } + payload += TableLebBytes( (uint64_t) body ) + body; + } + return payload; +} + +template +inline int64_t TableNodeTableMeasure( const Ctx & ctx, TableIds & ids, const TableNumbering & n ) +{ + if ( n.count == 0 ) { return 0; } // a root that reaches no nodes writes none of them + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayload( ctx, ids, n ); + if ( payload < 0 ) { return -1; } + return TableLebBytes( ref ) + 1 + TableLebBytes( (uint64_t) payload ) + payload; +} + +template +inline bool TableNodeTableSave( const Ctx & ctx, TableWriter & w, TableIds & ids, const TableNumbering & n ) +{ + if ( n.count == 0 ) { return true; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayload( ctx, ids, n ); + if ( payload < 0 ) { return false; } + w.putleb( ref ); + w.put8( 12 ); // kind 12 is the opaque byte payload: a reader that cannot name the id skips by L + w.putleb( (uint64_t) payload ); + w.putleb( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.putleb( ids.ref( n.entries[k].type_id ) ); + const int64_t body = n.entries[k].measure( (const void *) &ctx, n, ids, n.entries[k].node ); + if ( body < 0 ) { return false; } + w.putleb( (uint64_t) body ); + if ( !n.entries[k].save( (const void *) &ctx, n, w, ids, n.entries[k].node ) ) { return false; } + } + return true; +} + +// ---- the numbering, on the LOAD side: a region's NODE DIRECTORY (§6.3) ---- +// +// The wire's numbering made resident: one entry per numbered node, in index +// order, position i describing node index i + 1 — so position 0 is the ROOT at +// offset 0. It is ATTRIBUTION, and attribution is separable: nothing that reads +// a structure touches it, a deref is one add on a self-relative offset, and a +// caller may release it once Load returns. +struct TableNodeDirEntry +{ + uint64_t offset; + uint64_t type_id; +}; + +// the node's extent cursor, defined with the extent runtime (docs/SPEC-TABLES.md +// §2.8, §2.9); the node map names it only through a pointer. +struct TableExtentCarve; + +// TableNodeMap is what a pointer slot resolves through while a body decodes. +struct TableNodeMap +{ + uint8_t * base = NULL; + const TableNodeDirEntry * entries = NULL; + int64_t count = 0; // the ROOT's entry included, so it is records + 1 + bool good = false; // the node table read whole; a numbering that failed resolves nothing + // WHERE THE NODES LIVE, and therefore what a resolved slot holds: a region + // takes the SELF-RELATIVE delta so a deref is one add, and the tool's + // builder path takes the node's ARENA OFFSET (§6.3). + bool arena = false; + // WHERE A MAP'S ENTRIES AND A LIST'S ELEMENTS LAND while this node's body + // decodes (docs/SPEC-TABLES.md §2.8, §2.9): the node's own extent on the + // region path and the builder's arena on the tool's. It is MUTABLE + // because the cursor belongs to ONE node's decode and the dispatch that + // owns that node holds the map by const reference, exactly as it did + // before either construct existed. The decoder's signature does not + // move for a construct it may not carry. + mutable TableExtentCarve * carve = NULL; + // and the TOOL's path's allocation front, set once: there the arrays + // are the builder's arena's rather than a node's extent. + TableWorker * worker = NULL; + // THE TOOL PATH'S REFUSAL (docs/SPEC-TABLES.md §2.9): a count above the + // int32 cap met while a body decoded. LoadBuilder answers NULL for it + // and moves no counter; mutable for the reason the cursor is. + mutable bool refused = false; +}; + +// TableNodeResolve places one node index in a pointer slot, and every failure +// is one of §4's events with the pointer left null. The declared TARGET type id +// is checked at every index, the root's included: the root carries no record +// and therefore no wire type id, so the READER'S OWN root type is what the +// claim is checked against. +inline void TableNodeResolve( const TableNodeMap & map, TableRef & slot, uint64_t index, uint64_t target, TableReport * report ) +{ + slot.value = 0; + if ( index == kTableNodeIndexNull || !map.good ) { return; } + if ( index - 1 >= (uint64_t) map.count ) + { + report->malformed = true; // an index above node_count + 1 + return; + } + const TableNodeDirEntry & entry = map.entries[index - 1]; + if ( entry.offset == kTableNodeAbsent ) + { + // a node whose type id this build could not name KEEPS ITS INDEX, and + // every pointer naming it reads null. The unknown was counted once, at + // the node, not once per pointer. + return; + } + if ( entry.type_id != target ) + { + report->kind_mismatch++; + return; + } + slot.value = map.arena ? (int64_t) entry.offset + : (int64_t) ( ( map.base + entry.offset ) - (const uint8_t *) &slot ); +} + +// ---- the record SCAN, and it is the whole of load's bound (§3.1) ---- +// +// Reading follows no reference. The scan walks the root body's top-level fields, +// finds the ONE under the reserved id, and reads records out of its payload in +// order — the field rides once, so nothing is copied to make a body contiguous +// and the generated body decoder never learns the transport exists. +struct TableNodeScan +{ + TableReader fields; // over the ROOT body, skipping past everything else + const uint8_t * payload; // the node-table field's payload + int64_t payload_size; + int64_t payload_offset; + bool opened; // the root body has been walked for the field + uint64_t declared; + int64_t records; + bool present; // the root body carries a node table at all + bool malformed; + const TableIdTable * ids; +}; + +inline TableNodeScan TableNodeScanBegin( const uint8_t * body, int64_t size, TableReport * report, const TableIdTable * ids ) +{ + TableNodeScan s = { TableReader( body, size, report, ids ), NULL, 0, 0, false, 0, 0, false, false, ids }; + return s; +} + +// find the node-table field, or answer false when the root body has none. A +// body carrying an id more than once is legal input and THE LAST OCCURRENCE +// WINS (docs/SPEC-TABLES.md §3), so the walk runs to the terminator and keeps +// the last rather than stopping at the first. +inline bool TableNodeScanOpen( TableNodeScan & s ) +{ + if ( s.opened ) { return false; } + s.opened = true; + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.fields.getleb( ref ) ) { break; } + if ( ref == 0 ) { break; } // the terminator + if ( s.ids == NULL || ref > (uint64_t) s.ids->count ) { break; } + const uint64_t id = s.ids->at( ref ); + if ( !s.fields.has( 1 ) ) { break; } + const uint8_t kind = s.fields.get8(); + if ( id == kTableNodeTableFieldId ) + { + s.present = true; + if ( kind != 12 ) { s.malformed = true; return false; } + uint64_t length = 0; + if ( !s.fields.getleb( length ) || !s.fields.room( length ) ) { s.malformed = true; return false; } + s.payload = s.fields.buffer + s.fields.offset; + s.payload_size = (int64_t) length; + s.fields.offset += (int64_t) length; + continue; + } + if ( !s.fields.skip( kind ) ) { break; } + } + if ( s.payload == NULL ) { return false; } + TableReader head( s.payload, s.payload_size, s.fields.report, s.ids ); + if ( !head.getleb( s.declared ) ) { s.malformed = true; return false; } + s.payload_offset = head.offset; + return true; +} + +// the next record, or false at the end of the table — s.malformed says whether +// the end was the end or the framing giving out +inline bool TableNodeScanNext( TableNodeScan & s, uint64_t & type_id, const uint8_t * & body, int64_t & length ) +{ + if ( !s.opened && !TableNodeScanOpen( s ) ) { return false; } + if ( s.payload == NULL || s.payload_offset >= s.payload_size ) { return false; } + TableReader rec( s.payload, s.payload_size, s.fields.report, s.ids ); + rec.offset = s.payload_offset; + uint64_t ref = 0; + if ( !rec.getleb( ref ) || ref == 0 || s.ids == NULL || ref > (uint64_t) s.ids->count ) + { + s.malformed = true; // a type id reference of 0, or one past the table + return false; + } + type_id = s.ids->at( ref ); + uint64_t declared_length = 0; + if ( !rec.getleb( declared_length ) ) + { + s.malformed = true; // a record whose length is damaged + return false; + } + if ( declared_length > (uint64_t) ( s.payload_size - rec.offset ) ) + { + s.malformed = true; // a record whose length runs past its field + return false; + } + body = s.payload + rec.offset; + length = (int64_t) declared_length; + s.payload_offset = rec.offset + length; + s.records++; + return true; +} + +// The record scan is AUTHORITATIVE: node_count is data from the wire, and a +// count that disagrees with the scan is malformed. Nothing is sized from it +// before the scan has confirmed it. +inline bool TableNodeScanWhole( TableNodeScan & s ) +{ + if ( s.malformed ) { return false; } + if ( !s.present ) { return true; } // no node table at all is not a broken one + return s.declared == (uint64_t) s.records; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_ARENA + +#ifndef MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES +#define MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES + +namespace mapdemo { + +// ---- the NODE TABLE on the message wire (docs/SPEC-TABLES.md §3.1, §3.3) ---- +// +// THE NODE TABLE, WHEN A BODY HAS ONE, IS THE FIRST FIELD OF THE ROOT BODY: the +// reserved id as a reference, the node count at THIRTY-TWO RAW BITS, then the +// records back to back, each a type id reference and a body: a table's fields +// end at their own zero reference, and a blob's body is a length, an align and +// its bytes. A root +// that reaches no node elides the field, like every other empty thing. +// +// Measure derives the numbering from the graph and save derives the same one, +// and the two thunks stored at numbering time are what let one loop write a +// table of mixed types. +template +inline int64_t TableMessageNodeTableMeasure( const Ctx & ctx, const TableNumbering & n, int64_t index_bits, int64_t at ) +{ + if ( n.count == 0 ) { return 0; } // a root that reaches no nodes writes none of them + int64_t bits = kTableMessageRefBitsHere + 32; + for ( int64_t k = 0; k < n.count; k++ ) + { + bits += kTableMessageRefBitsHere; + const int64_t body = n.entries[k].message_measure( (const void *) &ctx, n, index_bits, at + bits, n.entries[k].node ); + if ( body < 0 ) { return -1; } + bits += body; + } + return bits; +} + +template +inline bool TableMessageNodeTableSave( const Ctx & ctx, const TableNumbering & n, int64_t index_bits, TableBitWriter & w ) +{ + if ( n.count == 0 ) { return true; } + w.put( kTableNodeTableFieldSlot, kTableMessageRefBitsHere ); + w.put( (uint64_t) n.count, 32 ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.put( n.entries[k].type_slot, kTableMessageRefBitsHere ); + if ( !n.entries[k].message_save( (const void *) &ctx, n, index_bits, w, n.entries[k].node ) ) { return false; } + } + return !w.overflow; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES + +#ifndef MAPDEMO_SCHEMA_TABLE_RETAIN +#define MAPDEMO_SCHEMA_TABLE_RETAIN + +namespace mapdemo { + +// ---- RETAIN-UNKNOWN (docs/SPEC-TABLES.md §6.6) ---- +// +// A REGION ROUND TRIP AND ONLY THAT: LoadRetain is Load's path into a region +// and SaveRetain saves from that same region. The builder path carries no +// retention, because a builder has no node directory to anchor a record on and +// re-derives its numbering from the reader's declaration order. +// +// Nothing here allocates. The record bytes and the retained-id list are the +// caller's storage, declared with their capacities, and a record that does not +// fit whole is dropped with one retain_lost. + +// THE PATH NAMES THE BODY, and it is the REGION's own address (§6.6). Step one +// is the node's index in the region's node directory, 1 for the root body and +// k for the node at directory position k - 1. Every further step is the PAIR: +// the field ordinal in the body the step descends from, in the READER's own +// declaration order, and the element index inside that field: zero for a +// scalar body, the element's index for an array of any of the four kinds, the +// ARM's OWN ORDINAL for a union, and the key's slot for a map. +static const int32_t kTableRetainDepthMax = 5; + +struct TableRetainStep +{ + uint32_t ordinal; + uint32_t index; +}; + +// at is the node's own address, which is what the SAVE side matches on: a +// record carries the directory INDEX and the directory answers the address in +// one add, so neither side ever searches a numbering. +struct TableRetainPath +{ + const void * at; + uint32_t node; + int32_t depth; + TableRetainStep steps[ kTableRetainDepthMax ]; +}; + +inline TableRetainPath TableRetainPathRoot( const void * at, uint32_t node ) +{ + TableRetainPath path; + path.at = at; + path.node = node; + path.depth = 0; + return path; +} + +// A STEP IS COMPUTED LOCALLY, at the moment the walk descends (§6.6), and it +// is taken by VALUE so that a descent is an expression: both sides walk the +// same declaration order, so neither numbers a tree and neither pops. +inline TableRetainPath TableRetainStepInto( const TableRetainPath & path, uint32_t ordinal, uint32_t index ) +{ + TableRetainPath out = path; + if ( out.depth < kTableRetainDepthMax ) + { + out.steps[ out.depth ].ordinal = ordinal; + out.steps[ out.depth ].index = index; + } + out.depth++; + return out; +} + +// THE CALLER'S TWO STORES (§6.6): the record bytes and the retained ids, each +// a pointer, a capacity and what has been used of it. A retention buffer +// belongs to ONE loaded region, and the next LoadRetain into it resets both. +struct TableRetain +{ + // AN ENTRY IS THE ID AND ITS SLOT IN THE TRAILER BEING WRITTEN. The two + // stores are numbered into ONE trailer in merged first-use order, so an + // index into this list is not the number a second reference wants and the + // slot rides beside the id. The layout is this port's own. + struct Id + { + uint64_t id; + int32_t slot; + }; + + uint8_t * bytes = NULL; + int64_t capacity = 0; + int64_t used = 0; + Id * ids = NULL; + int32_t id_capacity = 0; + int32_t id_used = 0; + int32_t count = 0; // records held + + // the REGION this buffer belongs to: a record carries a directory index + // and the save resolves it here, so nothing searches and nothing allocates + const uint8_t * base = NULL; + const TableNodeDirEntry * directory = NULL; + int64_t directory_count = 0; +}; + +// A RETAINED RECORD IS READER-PRIVATE (§6.6). It is not a wire form: no form +// byte, no version, no declared byte order, and nothing ever writes one to +// disk or hands one to another process. What it must CARRY is the body it +// belongs to, and the field's identity and bytes with every reference +// resolved. This layout is one sound way to carry them and nothing compares +// two ports' buffers. +// +// u32 record bytes, this header included +// u32 node the path's first step +// u32 depth the step pairs that follow +// u32 payload bytes +// u64 field id +// u8 kind +// u8 placed the save's own mark, cleared before every save +// depth x { u32 ordinal, u32 index } +// payload the field's payload with every reference resolved +static const int64_t kTableRetainRecordHeader = 26; + +inline uint32_t TableRetainRead32( const uint8_t * p ) +{ + return uint32_t( p[0] ) | uint32_t( p[1] ) << 8 | uint32_t( p[2] ) << 16 | uint32_t( p[3] ) << 24; +} + +inline uint64_t TableRetainRead64( const uint8_t * p ) +{ + return uint64_t( TableRetainRead32( p ) ) | ( uint64_t( TableRetainRead32( p + 4 ) ) << 32 ); +} + +inline void TableRetainWrite32( uint8_t * p, uint32_t v ) +{ + p[0] = uint8_t( v ); p[1] = uint8_t( v >> 8 ); p[2] = uint8_t( v >> 16 ); p[3] = uint8_t( v >> 24 ); +} + +inline void TableRetainWrite64( uint8_t * p, uint64_t v ) +{ + TableRetainWrite32( p, uint32_t( v ) ); + TableRetainWrite32( p + 4, uint32_t( v >> 32 ) ); +} + +inline int64_t TableRetainRecordBytes( const uint8_t * record ) { return (int64_t) TableRetainRead32( record ); } +inline uint32_t TableRetainRecordNode( const uint8_t * record ) { return TableRetainRead32( record + 4 ); } +inline int32_t TableRetainRecordDepth( const uint8_t * record ) { return (int32_t) TableRetainRead32( record + 8 ); } +inline int64_t TableRetainRecordPayloadBytes( const uint8_t * record ) { return (int64_t) TableRetainRead32( record + 12 ); } +inline uint64_t TableRetainRecordId( const uint8_t * record ) { return TableRetainRead64( record + 16 ); } +inline uint8_t TableRetainRecordKind( const uint8_t * record ) { return record[24]; } +inline bool TableRetainRecordPlaced( const uint8_t * record ) { return record[25] != 0; } +inline const uint8_t * TableRetainRecordSteps( const uint8_t * record ) { return record + kTableRetainRecordHeader; } +inline const uint8_t * TableRetainRecordPayload( const uint8_t * record ) +{ + return record + kTableRetainRecordHeader + 8 * (int64_t) TableRetainRecordDepth( record ); +} +inline uint8_t * TableRetainRecordPayload( uint8_t * record ) +{ + return record + kTableRetainRecordHeader + 8 * (int64_t) TableRetainRecordDepth( record ); +} + +// THE RECORD'S OWN BODY, resolved through the directory the buffer holds. A +// node index names one node for the life of the region, so this is one add. +inline const void * TableRetainRecordAt( const TableRetain & retain, const uint8_t * record ) +{ + const uint32_t node = TableRetainRecordNode( record ); + if ( retain.directory == NULL || node == 0 || (int64_t) node > retain.directory_count ) { return NULL; } + return (const void *) ( retain.base + retain.directory[ node - 1 ].offset ); +} + +// Does this record belong to the body the walk is standing in? The node first, +// which rejects almost everything in one compare, then the step pairs. +inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * record, const TableRetainPath & path ) +{ + if ( TableRetainRecordDepth( record ) != path.depth ) { return false; } + if ( TableRetainRecordAt( retain, record ) != path.at ) { return false; } + const uint8_t * steps = TableRetainRecordSteps( record ); + for ( int32_t i = 0; i < path.depth; i++ ) + { + if ( TableRetainRead32( steps + 8 * i ) != path.steps[i].ordinal ) { return false; } + if ( TableRetainRead32( steps + 8 * i + 4 ) != path.steps[i].index ) { return false; } + } + return true; +} + +// EVERY ID THIS BUILD CAN NAME, ascending: the set TableIds's capacity is +// derived from. An id inside a retained record takes its trailer entry from +// the GENERATED table when it is here and from the CALLER's list otherwise, so +// no retained id ever enters the generated table and no id is written twice. +static const int32_t kTableRetainKnownIds = 76; +static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, + 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, + 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, + 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, + 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, +}; + +inline bool TableRetainNameable( uint64_t id ) +{ + int32_t low = 0, high = kTableRetainKnownIds - 1; + while ( low <= high ) + { + const int32_t mid = low + ( high - low ) / 2; + if ( kTableRetainKnown[mid] == id ) { return true; } + if ( kTableRetainKnown[mid] < id ) { low = mid + 1; } else { high = mid - 1; } + } + return false; +} + +// THE TWO STORES, NUMBERED INTO ONE TRAILER in merged first-use order (§6.6). +// It answers the surface TableIds answers, ref, count, truncate and +// overflow, so the retain family's codec is the plain one with its names +// changed, and +// the GENERATED TABLE IS UNTOUCHED: its capacity, its overflow rule and its +// -1 stand exactly as they are for every save. +struct TableRetainIds +{ + TableIds known; + int32_t known_slot[ TableIds::kCapacity ]; + TableRetain * retain; + int32_t count; + bool overflow; + bool lost; // a retained id past the caller's capacity: the record is dropped + + TableRetainIds( TableRetain * to_retain ) : retain( to_retain ), count( 0 ), overflow( false ), lost( false ) {} + + // an id this build CAN name, which is every id the generated codec writes + uint64_t ref( uint64_t id ) + { + const int32_t before = known.count; + const uint64_t k = known.ref( id ); + if ( known.overflow ) { overflow = true; return 1; } + if ( known.count != before ) { known_slot[ (int32_t) k - 1 ] = ++count; } + return (uint64_t) known_slot[ (int32_t) k - 1 ]; + } + + // an id from INSIDE a retained record. A retained id takes its entry from + // the caller's list, and one past the capacity sets lost: the record is + // dropped, nothing else about the save changes, and the save is never + // refused (§6.6). + uint64_t record_ref( uint64_t id ) + { + if ( TableRetainNameable( id ) ) { return ref( id ); } + if ( retain == NULL ) { lost = true; return 0; } + for ( int32_t i = 0; i < retain->id_used; i++ ) + { + if ( retain->ids[i].id == id ) { return (uint64_t) retain->ids[i].slot; } + } + if ( retain->id_used >= retain->id_capacity ) { lost = true; return 0; } + retain->ids[ retain->id_used ].id = id; + retain->ids[ retain->id_used ].slot = ++count; + retain->id_used++; + return (uint64_t) count; + } + + // undo every entry taken since mark, in either store. Both are appended in + // slot order, so an entry removed is the last one of its store. + void truncate( int32_t mark ) + { + while ( known.count > 0 && known_slot[ known.count - 1 ] > mark ) { known.truncate( known.count - 1 ); } + while ( retain != NULL && retain->id_used > 0 && retain->ids[ retain->id_used - 1 ].slot > mark ) { retain->id_used--; } + count = mark; + } +}; + +// THE FILE STILL CARRIES ONE ID TABLE (§3): the split is the writer's storage +// rather than the wire's, and the trailer is one merge of two slot-ordered +// stores. +inline int64_t TableRetainIdsBytes( const TableRetainIds & ids ) { return int64_t( ids.count ) * 8 + 8; } + +// A TWO-WAY MERGE over the stores' own slot order, and not a scan for each +// slot. Every entry either store holds took its slot from the same counter, so +// the two runs interleave to exactly the slots 1 to count and the merge has no +// case for a slot neither store took. +inline void TableRetainIdsWrite( TableWriter & w, const TableRetainIds & ids ) +{ + const int32_t retained = ids.retain != NULL ? ids.retain->id_used : 0; + int32_t i = 0, j = 0; + while ( i < ids.known.count || j < retained ) + { + if ( j >= retained || ( i < ids.known.count && ids.known_slot[i] < ids.retain->ids[j].slot ) ) + { + w.put64( ids.known.ids[i] ); + i++; + continue; + } + w.put64( ids.retain->ids[j].id ); + j++; + } + w.put64( uint64_t( ids.count ) ); +} + +// ---- THE RESOLVING WALK (§6.6) ---- +// +// A reference names a SLOT of the file's id table, so a verbatim copy +// re-emitted into a file whose table is ordered differently would point at +// other names in silence. A retained record therefore holds the field with +// every reference replaced by the sixty-four-bit id it names, and every length +// that frames a rewritten reference recomputed. +// +// THE WALK IS AN INTERPRETATION, AND ITS VERDICT IS STATED: it reads kind +// bytes, lengths and references and nothing else. No value is decoded, no +// bound is checked, no branch is taken on a payload byte, and anything it +// cannot frame DROPS THE RECORD, counts one retain_lost, and never raises +// malformed on the plain read. +// +// THE WALK IS ONE PASS EACH WAY, and its cost is linear in the record's own +// bytes. Every length that frames a content in the resolved form is a fixed +// slot rather than a canonical LEB128, so the capture reserves it, writes the +// content, and fills the slot in behind it. A spelling that had to know the +// resolved size before writing it would have to walk each content twice, once +// at every level, and the file chooses the nesting. +// +// A retained record's inner nesting is the WRITER's and not this build's, so +// it is the one depth on this path a file can drive. The cap counts NESTED +// BODIES, and a record past it is dropped on the same rule as any other shape +// the walk cannot take. Time no longer rests on it: it is a small stated +// constant and nothing more. +static const int32_t kTableRetainWalkDepthMax = 64; + +// the three RESERVED ids (§3.1, §3.3). One inside a retained record's payload +// would be re-emitted into a nested body, where it is malformed, so meeting +// one drops the record. +inline bool TableRetainReservedId( uint64_t id ) +{ + return id == kTableNodeTableFieldId || id == kTableBuildVersionFieldId || id == kTableMessageVocabularyFieldId; +} + +struct TableRetainIn +{ + const uint8_t * in; + int64_t size; + int64_t at; + const TableIdTable * ids; + uint8_t * out; // NULL: measuring, and nothing is written + int64_t out_at; +}; + +inline void TableRetainInRaw( TableRetainIn & s, const uint8_t * from, int64_t bytes ) +{ + if ( s.out != NULL ) { memcpy( s.out + s.out_at, from, (size_t) bytes ); } + s.out_at += bytes; +} + +inline void TableRetainInLeb( TableRetainIn & s, uint64_t v ) +{ + uint8_t b[10]; + int64_t n = 0; + while ( v >= 0x80 ) { b[n++] = uint8_t( v ) | 0x80; v >>= 7; } + b[n++] = uint8_t( v ); + TableRetainInRaw( s, b, n ); +} + +inline void TableRetainInId( TableRetainIn & s, uint64_t id ) +{ + uint8_t b[8]; + TableRetainWrite64( b, id ); + TableRetainInRaw( s, b, 8 ); +} + +inline bool TableRetainInLebRead( TableRetainIn & s, uint64_t & value ) +{ + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( s.at >= s.size ) { return false; } + const uint8_t b = s.in[ s.at++ ]; + if ( i == 9 && b > 1 ) { return false; } + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) { return i == 0 || b != 0; } + shift += 7; + } + return false; +} + +// one REFERENCE resolved to the id it names. A zero reference is the wire's +// own "no id", the enum's None and the union's empty arm, and rides as the +// id zero. A reference above the entry count, a reference at an id-table entry +// of zero, and a reference at a reserved id are each damage the plain read +// never looked at, and each drops the record. +inline bool TableRetainInRef( TableRetainIn & s, bool zero_allowed ) +{ + uint64_t ref = 0; + if ( !TableRetainInLebRead( s, ref ) ) { return false; } + if ( ref == 0 ) + { + if ( !zero_allowed ) { return false; } + TableRetainInId( s, 0 ); + return true; + } + if ( s.ids == NULL || ref > (uint64_t) s.ids->count ) { return false; } + const uint64_t id = s.ids->at( ref ); + if ( id == 0 || TableRetainReservedId( id ) ) { return false; } + TableRetainInId( s, id ); + return true; +} + +inline int64_t TableRetainInPayload( TableRetainIn & s, uint8_t kind, int32_t depth ); +inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ); + +// ONE FRAMED LENGTH in the resolved form: a pair of fixed u32. The first is +// the RESOLVED byte count of the content it frames, reserved here and written +// once the content is out. The second is the SAVE's scratch, left zero by the +// capture and filled by the walk that emits. +// +// The record is the reader's own storage and nothing outside this family ever +// reads it, so a length may be written after the bytes it measures. That is +// the whole of what makes the walk one pass. +static const int64_t kTableRetainSlotBytes = 8; + +inline int64_t TableRetainInSlot( TableRetainIn & s ) +{ + const int64_t at = s.out_at; + const uint8_t zero[ kTableRetainSlotBytes ] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + TableRetainInRaw( s, zero, kTableRetainSlotBytes ); + return at; +} + +inline void TableRetainInPatch( TableRetainIn & s, int64_t slot, int64_t resolved ) +{ + if ( s.out != NULL ) { TableRetainWrite32( s.out + slot, (uint32_t) resolved ); } +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it. +// The measuring pass takes the same path and reserves the same fixed width, so +// the size it answers is the size the writing pass lays down. +inline int64_t TableRetainInFramed( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s ); + const int64_t resolved = TableRetainInContent( s, kind, length, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s, slot, resolved ); + return resolved; +} + +inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + if ( length < 0 || s.at + length > s.size ) { return -1; } + const int64_t end = s.at + length; + const int64_t began = s.out_at; + switch ( kind ) + { + case 13: // a table BODY: fields, then the zero reference + { + for ( ;; ) + { + uint64_t ref = 0; + const int64_t mark = s.at; + if ( !TableRetainInLebRead( s, ref ) ) { return -1; } + // THE TERMINATOR IS A REFERENCE, and a reference in the + // resolved form is a fixed eight-byte id: the zero that ends a + // body rides at the width every other one does. + if ( ref == 0 ) { TableRetainInId( s, 0 ); break; } + s.at = mark; + if ( !TableRetainInRef( s, false ) ) { return -1; } + if ( s.at >= end ) { return -1; } + const uint8_t field_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &field_kind, 1 ); + if ( TableRetainInPayload( s, field_kind, depth ) < 0 ) { return -1; } + if ( s.at > end ) { return -1; } + } + break; + } + case 14: // an ARRAY body: the element kind, the count, then the elements + { + if ( s.at >= end ) { return -1; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainInLebRead( s, n ) ) { return -1; } + TableRetainInLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( TableRetainInPayload( s, elem_kind, depth ) < 0 ) { return -1; } + if ( s.at > end ) { return -1; } + } + break; + } + case 16: // an ENUM-KEYED body: N triples of a KEY REFERENCE, an L and the element + { + if ( s.at >= end ) { return -1; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainInLebRead( s, n ) ) { return -1; } + TableRetainInLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + // A KEYED BODY'S KEYS RESOLVE AT EVERY ELEMENT KIND (§6.6, §3.2) + if ( !TableRetainInRef( s, false ) ) { return -1; } + uint64_t slot_bytes = 0; + if ( !TableRetainInLebRead( s, slot_bytes ) ) { return -1; } + if ( slot_bytes > (uint64_t) ( end - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, elem_kind, (int64_t) slot_bytes, depth + 1 ) < 0 ) { return -1; } + } + break; + } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + default: + // every other content is bytes: a string, wide text, an escape, a + // payload-free kind, a scalar under a keyed slot's own length + TableRetainInRaw( s, s.in + s.at, length ); + s.at += length; + break; + } + if ( s.at != end ) { return -1; } + return s.out_at - began; +} + +// the depth a payload carries is its enclosing body's: only a framed CONTENT +// is a level, and TableRetainInContent is the one place the cap is read. +inline int64_t TableRetainInPayload( TableRetainIn & s, uint8_t kind, int32_t depth ) +{ + const int64_t began = s.out_at; + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: // the fixed-width kinds, by width + case 3: case 7: case 21: case 26: + case 4: case 8: case 10: case 22: case 27: + case 5: case 9: case 11: case 23: case 28: + case 18: case 19: case 24: case 29: + { + int64_t width = 1; + switch ( kind ) + { + case 3: case 7: case 21: case 26: width = 2; break; + case 4: case 8: case 10: case 22: case 27: width = 4; break; + case 5: case 9: case 11: case 23: case 28: width = 8; break; + case 18: case 19: case 24: case 29: width = 16; break; + default: width = 1; break; + } + if ( s.at + width > s.size ) { return -1; } + TableRetainInRaw( s, s.in + s.at, width ); + s.at += width; + break; + } + case 12: case 31: case 32: case 33: // L, then L bytes, nothing framed inside + { + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + TableRetainInLeb( s, length ); + TableRetainInRaw( s, s.in + s.at, (int64_t) length ); + s.at += (int64_t) length; + break; + } + case 13: case 14: case 16: // L, then a body the walk resolves + { + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, kind, (int64_t) length, depth + 1 ) < 0 ) { return -1; } + break; + } + case 15: // a UNION: the arm id reference, and when it is not zero its kind, L and payload + { + const int64_t mark = s.at; + uint64_t arm = 0; + if ( !TableRetainInLebRead( s, arm ) ) { return -1; } + s.at = mark; + if ( !TableRetainInRef( s, true ) ) { return -1; } + if ( arm == 0 ) { break; } + if ( s.at >= s.size ) { return -1; } + const uint8_t arm_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &arm_kind, 1 ); + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, arm_kind, (int64_t) length, depth + 1 ) < 0 ) { return -1; } + break; + } + case 30: // an ENUM's variant reference, zero for None + { + if ( !TableRetainInRef( s, true ) ) { return -1; } + break; + } + case 17: return -1; // A NODE INDEX (§3.1): the whole record goes with it + default: return -1; // a kind this walk cannot frame + } + return s.out_at - began; +} + +// ---- CAPTURE: the load side (§6.6) ---- +// +// The field is skipped by its framing exactly as it always was and counted +// unknown exactly as it always was, so a full buffer degrades to the default +// behavior one field at a time. False is what r.skip( kind ) answers false +// for, and nothing else: retention can lose a field, it can never turn a good +// read into a bad one. +inline bool TableRetainCapture( TableRetain * retain, TableReader & r, const TableRetainPath & path, + uint64_t field_id, uint8_t kind ) +{ + const int64_t start = r.offset; + if ( !r.skip( kind ) ) { return false; } + if ( retain == NULL ) { return true; } + const int64_t wire_bytes = r.offset - start; + + TableRetainIn probe; + probe.in = r.buffer + start; + probe.size = wire_bytes; + probe.at = 0; + probe.ids = r.ids; + probe.out = NULL; + probe.out_at = 0; + const int64_t payload = TableRetainInPayload( probe, kind, 0 ); + if ( payload < 0 || probe.at != wire_bytes ) { r.report->retain_lost++; return true; } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL: the buffer never holds a + // truncated field, and the read continues (§6.6) + r.report->retain_lost++; + return true; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, field_id ); + record[24] = kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableRetainIn write; + write.in = r.buffer + start; + write.size = wire_bytes; + write.at = 0; + write.ids = r.ids; + write.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out_at = 0; + if ( TableRetainInPayload( write, kind, 0 ) < 0 ) { r.report->retain_lost++; return true; } + retain->used += need; + retain->count++; + r.report->retained++; + return true; +} + +// LoadRetain RESETS BOTH STORES and writes into neither list (§6.6): a +// retained record carries its field's identity in the record itself, with +// every reference resolved. +inline void TableRetainReset( TableRetain * retain, const TableNodeMap & nodes, const uint8_t * region ) +{ + if ( retain == NULL ) { return; } + retain->used = 0; + retain->id_used = 0; + retain->count = 0; + retain->base = region; + retain->directory = nodes.entries; + retain->directory_count = nodes.count; +} + +// ---- RECORD LIFETIME (docs/SPEC-TABLES.md §6.6) ---- +// +// A RETAINED RECORD BELONGS TO THE BODY OCCURRENCE THAT CARRIED IT, AND DIES +// WITH IT. Legal input can carry a known child body twice, and the later +// occurrence resets the child and wins whole (§3, §4): the records retained +// under the earlier occurrence go with the values it held. The discard moves +// NEITHER counter. The writer superseded the data, so nothing was lost that +// the load could have kept, and retained counted the record when its bytes +// were kept and does not fall when they are let go. +// +// The occurrences are four, and each is a body the wire lets a writer put down +// again: a repeated TABLE field, by value or under ?, a UNION whose arm is +// written again, a MAP's duplicate key, and a KEYED-ARRAY slot written again. +// The FIELD form covers the three where the field itself is read again, arm +// switches and shrinking arrays included; the BODY form covers a duplicate key +// inside one occurrence of a map, where the field is read once and the entry +// twice. +inline bool TableRetainUnder( const uint8_t * record, const void * at, const TableRetain & retain, + const TableRetainPath & path, bool field, uint32_t ordinal ) +{ + if ( TableRetainRecordAt( retain, record ) != at ) { return false; } + const int32_t depth = TableRetainRecordDepth( record ); + if ( field ) + { + if ( depth <= path.depth ) { return false; } + } + else if ( depth < path.depth ) { return false; } + const uint8_t * steps = TableRetainRecordSteps( record ); + for ( int32_t i = 0; i < path.depth; i++ ) + { + if ( TableRetainRead32( steps + 8 * i ) != path.steps[i].ordinal ) { return false; } + if ( TableRetainRead32( steps + 8 * i + 4 ) != path.steps[i].index ) { return false; } + } + if ( field && TableRetainRead32( steps + 8 * path.depth ) != ordinal ) { return false; } + return true; +} + +inline void TableRetainDiscard( TableRetain * retain, const TableRetainPath & path, bool field, uint32_t ordinal ) +{ + if ( retain == NULL || retain->count == 0 ) { return; } + int64_t read = 0, write = 0; + int32_t kept = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + const int64_t bytes = TableRetainRecordBytes( retain->bytes + read ); + if ( !TableRetainUnder( retain->bytes + read, path.at, *retain, path, field, ordinal ) ) + { + if ( write != read ) { memmove( retain->bytes + write, retain->bytes + read, (size_t) bytes ); } + write += bytes; + kept++; + } + read += bytes; + } + retain->used = write; + retain->count = kept; +} + +inline void TableRetainDiscardBody( TableRetain * retain, const TableRetainPath & path ) +{ + TableRetainDiscard( retain, path, false, 0 ); +} + +inline void TableRetainDiscardField( TableRetain * retain, const TableRetainPath & path, uint32_t ordinal ) +{ + TableRetainDiscard( retain, path, true, ordinal ); +} + +// ---- EMIT: the save side (§6.6) ---- +// +// The record read back the other way: every resolved id becomes the reference +// the trailer being written gives it, and every length is recomputed against +// the references' new widths. The walk is the capture's mirror and the same +// damage rules apply, except that damage cannot be met: these bytes are the +// reader's own. +// +// A WIRE LENGTH IS CANONICAL LEB128 AND RIDES BEFORE ITS CONTENT, so this side +// cannot fill a slot in behind the bytes the way the capture does. It takes +// one POST-ORDER pass instead: measuring computes each content's wire size and +// leaves it in that content's own scratch slot, and the emit reads the size +// there rather than walking for it. Measuring runs immediately before the +// emit, on the same record and the same id table, which is what makes the two +// readings one walk. +struct TableRetainOut +{ + uint8_t * in; // the record: only a framed length's scratch half is written + int64_t size; + int64_t at; + TableRetainIds * ids; + TableWriter * w; // NULL: measuring, and the scratch slots are being filled + int64_t bytes; +}; + +inline void TableRetainOutRaw( TableRetainOut & s, const uint8_t * from, int64_t bytes ) +{ + if ( s.w != NULL ) { s.w->raw( from, bytes ); } + s.bytes += bytes; +} + +inline void TableRetainOutLeb( TableRetainOut & s, uint64_t v ) +{ + if ( s.w != NULL ) { s.w->putleb( v ); } + s.bytes += TableLebBytes( v ); +} + +inline bool TableRetainOutLebRead( TableRetainOut & s, uint64_t & value ) +{ + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( s.at >= s.size ) { return false; } + const uint8_t b = s.in[ s.at++ ]; + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) { return true; } + shift += 7; + } + return false; +} + +inline bool TableRetainOutRef( TableRetainOut & s ) +{ + if ( s.at + 8 > s.size ) { return false; } + const uint64_t id = TableRetainRead64( s.in + s.at ); + s.at += 8; + if ( id == 0 ) { TableRetainOutLeb( s, 0 ); return true; } // the wire's own no-id + const uint64_t ref = s.ids->record_ref( id ); + if ( s.ids->lost || s.ids->overflow ) { return false; } + TableRetainOutLeb( s, ref ); + return true; +} + +inline bool TableRetainOutPayload( TableRetainOut & s, uint8_t kind, int32_t depth ); +inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t length, int32_t depth ); + +// ONE FRAMED CONTENT, read out of the record's fixed slot and written with the +// canonical LEB128 length this wire wants. The ids it names are interned on +// the way past, which is what makes measure and save one walk in two readings, +// exactly as every other body on this wire is. +// +// MEASURING walks the content, then leaves the wire size it found in the +// scratch half of the slot. EMITTING reads that size, writes it, and CHECKS +// the content against it: a size no measure of this save left there is a +// record refused rather than a length that does not frame what follows. +inline bool TableRetainOutFramed( TableRetainOut & s, uint8_t kind, int32_t depth ) +{ + if ( s.at + kTableRetainSlotBytes > s.size ) { return false; } + uint8_t * const slot = s.in + s.at; + const int64_t resolved = (int64_t) TableRetainRead32( slot ); + s.at += kTableRetainSlotBytes; + if ( s.w == NULL ) + { + const int64_t began = s.bytes; + if ( !TableRetainOutContent( s, kind, resolved, depth ) ) { return false; } + const int64_t wire = s.bytes - began; + if ( wire > 0xFFFFFFFFll ) { return false; } + TableRetainWrite32( slot + 4, (uint32_t) wire ); + s.bytes += TableLebBytes( (uint64_t) wire ); + return true; + } + const int64_t wire = (int64_t) TableRetainRead32( slot + 4 ); + TableRetainOutLeb( s, (uint64_t) wire ); + const int64_t began = s.bytes; + if ( !TableRetainOutContent( s, kind, resolved, depth ) ) { return false; } + return s.bytes - began == wire; +} + +inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t length, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return false; } + if ( length < 0 || s.at + length > s.size ) { return false; } + const int64_t end = s.at + length; + switch ( kind ) + { + case 13: + { + for ( ;; ) + { + if ( s.at + 8 > end ) { return false; } + const uint64_t id = TableRetainRead64( s.in + s.at ); + if ( id == 0 ) { s.at += 8; TableRetainOutLeb( s, 0 ); break; } + if ( !TableRetainOutRef( s ) ) { return false; } + if ( s.at >= end ) { return false; } + const uint8_t field_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &field_kind, 1 ); + if ( !TableRetainOutPayload( s, field_kind, depth ) ) { return false; } + } + break; + } + case 14: + { + if ( s.at >= end ) { return false; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainOutLebRead( s, n ) ) { return false; } + TableRetainOutLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( !TableRetainOutPayload( s, elem_kind, depth ) ) { return false; } + } + break; + } + case 16: + { + if ( s.at >= end ) { return false; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainOutLebRead( s, n ) ) { return false; } + TableRetainOutLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( !TableRetainOutRef( s ) ) { return false; } + if ( !TableRetainOutFramed( s, elem_kind, depth + 1 ) ) { return false; } + } + break; + } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; + default: + TableRetainOutRaw( s, s.in + s.at, length ); + s.at += length; + break; + } + return s.at == end; +} + +// the depth a payload carries is its enclosing body's, exactly as on the +// capture side: only a framed CONTENT is a level. +inline bool TableRetainOutPayload( TableRetainOut & s, uint8_t kind, int32_t depth ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: + case 3: case 7: case 21: case 26: + case 4: case 8: case 10: case 22: case 27: + case 5: case 9: case 11: case 23: case 28: + case 18: case 19: case 24: case 29: + { + int64_t width = 1; + switch ( kind ) + { + case 3: case 7: case 21: case 26: width = 2; break; + case 4: case 8: case 10: case 22: case 27: width = 4; break; + case 5: case 9: case 11: case 23: case 28: width = 8; break; + case 18: case 19: case 24: case 29: width = 16; break; + default: width = 1; break; + } + if ( s.at + width > s.size ) { return false; } + TableRetainOutRaw( s, s.in + s.at, width ); + s.at += width; + break; + } + case 12: case 31: case 32: case 33: + { + uint64_t length = 0; + if ( !TableRetainOutLebRead( s, length ) ) { return false; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return false; } + TableRetainOutLeb( s, length ); + TableRetainOutRaw( s, s.in + s.at, (int64_t) length ); + s.at += (int64_t) length; + break; + } + case 13: case 14: case 16: + { + if ( !TableRetainOutFramed( s, kind, depth + 1 ) ) { return false; } + break; + } + case 15: + { + if ( s.at + 8 > s.size ) { return false; } + const uint64_t arm = TableRetainRead64( s.in + s.at ); + if ( !TableRetainOutRef( s ) ) { return false; } + if ( arm == 0 ) { break; } + if ( s.at >= s.size ) { return false; } + const uint8_t arm_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &arm_kind, 1 ); + if ( !TableRetainOutFramed( s, arm_kind, depth + 1 ) ) { return false; } + break; + } + case 30: + { + if ( !TableRetainOutRef( s ) ) { return false; } + break; + } + default: return false; + } + return true; +} + +// ---- THE RETAINED TAIL: where the records go back (§6.6) ---- +// +// AT THE END OF THEIR OWN BODY, IN THE ORDER RETAINED. Position carries +// nothing on this wire, so appending is chosen for three properties: it is a +// write with no splice, the retained order is preserved, and the result is +// IDEMPOTENT after the first save. +// +// A RETAINED ID PAST THE CAPACITY COUNTS ONE retain_lost AND ITS RECORD IS +// DROPPED, and the save is never refused. MeasureRetain and SaveRetain drop +// the same records under the same walk, so the measure sees the same overflow +// and its answer is the size the save writes. + +// one record's WIRE bytes under the trailer being written, and -1 for a record +// this save cannot place: an id the caller's list had no room for, or a +// resolved form the walk cannot read back. The ids it names are interned on +// the way past, which is what makes measure and save one rule read twice. +// +// THIS IS THE MEASURING PASS, and it leaves every framed content's wire size +// in that content's own scratch slot. The record is the caller's buffer and +// the pass writes nothing else into it. +inline int64_t TableRetainRecordWire( uint8_t * record, TableRetainIds & ids, uint64_t & ref ) +{ + const int32_t mark = ids.count; + ids.lost = false; + ref = ids.record_ref( TableRetainRecordId( record ) ); + if ( !ids.lost && !ids.overflow ) + { + TableRetainOut s; + s.in = TableRetainRecordPayload( record ); + s.size = TableRetainRecordPayloadBytes( record ); + s.at = 0; + s.ids = &ids; + s.w = NULL; + s.bytes = 0; + if ( TableRetainOutPayload( s, TableRetainRecordKind( record ), 0 ) && s.at == s.size ) + { + return TableLebBytes( ref ) + 1 + s.bytes; + } + } + // the record is not written at all, and nothing else about the save + // changes: a full id list degrades to the default behavior one record at a + // time, and the entries this attempt took are given back + ids.truncate( mark ); + ids.lost = false; + return -1; +} + +inline int64_t TableRetainTailMeasure( TableRetain * retain, TableRetainIds & ids, const TableRetainPath & path ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return 0; } + int64_t bytes = 0; + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordHere( *retain, record, path ) ) { continue; } + uint64_t ref = 0; + const int64_t wire = TableRetainRecordWire( record, ids, ref ); + if ( wire < 0 ) { continue; } + bytes += wire; + } + return bytes; +} + +inline bool TableRetainTailSave( TableRetain * retain, TableRetainIds & ids, TableWriter & w, const TableRetainPath & path ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return true; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordHere( *retain, record, path ) ) { continue; } + uint64_t ref = 0; + if ( TableRetainRecordWire( record, ids, ref ) < 0 ) { continue; } + w.putleb( ref ); + w.put8( TableRetainRecordKind( record ) ); + TableRetainOut s; + s.in = TableRetainRecordPayload( record ); + s.size = TableRetainRecordPayloadBytes( record ); + s.at = 0; + s.ids = &ids; + s.w = &w; + s.bytes = 0; + if ( !TableRetainOutPayload( s, TableRetainRecordKind( record ), 0 ) ) { return false; } + record[25] = 1; // PLACED: the one mark the save leaves on the buffer + } + return !w.overflow; +} + +// THE SAVE'S OWN SHARE OF retain_lost, counted ONCE and read after the save +// (§6.6): every record the walk did not place. A record whose path no longer +// names a body, one the caller's id list had no room for, and one the walk +// could not read back are one number here, because the check a caller reads is +// one number. A record is marked as it is written, so this cannot double-count +// a body measured twice. +inline void TableRetainClearPlaced( TableRetain * retain ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + retain->bytes[ at + 25 ] = 0; + at += TableRetainRecordBytes( retain->bytes + at ); + } +} + +inline void TableRetainCountLost( const TableRetain * retain, TableReport * report ) +{ + if ( retain == NULL || retain->bytes == NULL || report == NULL ) { return; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + const uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordPlaced( record ) ) { report->retain_lost++; } + } +} + +// THE NODE TABLE under retention (§3.1, §6.6): the same fill rule the plain +// pair derives, with the retain family's ids and each record's own body +// reached through a dispatch the CALL supplies rather than a second pair of +// thunks on the numbering. A store per node on the PLAIN save path would be a +// cost this feature is not allowed to have. +template +inline int64_t TableNodeTablePayloadRetain( const Ctx & ctx, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure ) +{ + int64_t payload = TableLebBytes( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + payload += TableLebBytes( ids.ref( n.entries[k].type_id ) ); + const int64_t body = measure( ctx, n, ids, n.entries[k].type_id, n.entries[k].node, retain ); + if ( body < 0 ) { return -1; } + payload += TableLebBytes( (uint64_t) body ) + body; + } + return payload; +} + +template +inline int64_t TableNodeTableMeasureRetain( const Ctx & ctx, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure ) +{ + if ( n.count == 0 ) { return 0; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayloadRetain( ctx, ids, n, retain, measure ); + if ( payload < 0 ) { return -1; } + return TableLebBytes( ref ) + 1 + TableLebBytes( (uint64_t) payload ) + payload; +} + +template +inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure, Save save ) +{ + if ( n.count == 0 ) { return true; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayloadRetain( ctx, ids, n, retain, measure ); + if ( payload < 0 ) { return false; } + w.putleb( ref ); + w.put8( 12 ); // kind 12 is the opaque byte payload, exactly as the plain save writes it + w.putleb( (uint64_t) payload ); + w.putleb( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.putleb( ids.ref( n.entries[k].type_id ) ); + const int64_t body = measure( ctx, n, ids, n.entries[k].type_id, n.entries[k].node, retain ); + if ( body < 0 ) { return false; } + w.putleb( (uint64_t) body ); + if ( !save( ctx, n, w, ids, n.entries[k].type_id, n.entries[k].node, retain ) ) { return false; } + } + return true; +} +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_RETAIN + +#ifndef MAPDEMO_SCHEMA_TABLE_EXTENT +#define MAPDEMO_SCHEMA_TABLE_EXTENT + +namespace mapdemo { + +// ---- the NODE EXTENT: where a map's entries and a list's elements live (§2.8, §2.9) ---- + +// TableExtentCarve is a node's extent cursor, PRE-ORDER: a container's whole +// array first, then, element by element in the container's own order, the +// arrays of any list or map an element holds by value. The cursor is the node +// map's, because the generated decoder is threaded with that and not with a +// region. +struct TableExtentCarve +{ + uint8_t * at = NULL; // the region path: the node's extent, unspent + int64_t left = 0; + TableWorker * worker = NULL; // the TOOL's path: the arrays come from the arena +}; + +// AN UNREACHED SLOT MUST HOLD NO LIST OR MAP WITH ELEMENTS IN IT (§2.8, §2.9, +// §7.6). An empty one takes no bytes, so a record whose extent measures ZERO is +// a record whose every by-value list and map is empty. A measure that REFUSED +// answers non-zero here too, and refusing on it is the same answer one level up. +inline bool TableExtentUnreachedEmpty( int64_t extent ) { return extent == 0; } + +// ---- LoadMeasure's framing walk (§6.5) ---- +// +// The measure reads no field value: it walks each record's field headers, +// skipping every payload by its framing, to reach each N at every depth. A +// false is a REFUSAL, and it carries its reason (§6.5). +typedef bool ( * TableWireExtentFn )( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ); + +// the framing walk over an ARRAY OF TABLES held by value: its elements' own +// lists and maps are part of this node's extent too +inline bool TableWireExtentElements( const uint8_t * body, int64_t length, int64_t & at, TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } + if ( r.get8() != 13 ) { return true; } + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +// and over an ENUM-KEYED array, whose triples carry a key REFERENCE before each +// length-prefixed element (docs/SPEC-TABLES.md §3.2) +inline bool TableWireExtentKeyed( const uint8_t * body, int64_t length, int64_t & at, TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } + if ( r.get8() != 13 ) { return true; } + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t key = 0; + if ( !r.getleb( key ) ) { return true; } + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_EXTENT + +#ifndef MAPDEMO_SCHEMA_TABLE_MAP +#define MAPDEMO_SCHEMA_TABLE_MAP + +namespace mapdemo { + +// ---- a MAP: a sorted entry array, and the lookup over it (§2.8) ---- +// +// On the wire, in a region and in a cook a map is an array of one generated +// ENTRY table held in ascending key order. What this adds is Find — a binary +// search over that array where it lies — and a builder that inserts, replaces +// and erases by key. Nothing here is stored: a region and a cook carry the +// array and the count, and not one byte about a hash or a probe. + +// entries carved from ONE call to the allocator pair; a new segment is +// appended when the current one fills, and nothing ever moves (§6.4) +static const int32_t kTableMapSegmentEntries = 32; + +// TableDeclRef names a type in an unevaluated context and is never defined — +// what 's declval is for, without the include the generated corpus +// refuses to pay for (the iterator_traits note, §13.9). +template T & TableDeclRef(); + +// THE ORDER IS TOTAL, AND IT IS THE SAME IN NINE LANGUAGES (§2.8). Integers +// compare by VALUE, signed for the signed kinds and unsigned for the unsigned. +// Strings compare by BYTES, unsigned, a shorter string that is a prefix of a +// longer one first: memcmp over the common length, then the lengths. Never a +// locale, never a code point, never a case fold. +inline int TableKeyOrder( uint64_t a, uint64_t b ) { return a < b ? -1 : ( a > b ? 1 : 0 ); } +inline int TableKeyOrder( int64_t a, int64_t b ) { return a < b ? -1 : ( a > b ? 1 : 0 ); } +inline int TableKeyOrder( const char * a, int32_t a_length, const char * b, int32_t b_length ) +{ + const int32_t common = a_length < b_length ? a_length : b_length; + if ( common > 0 ) + { + const int order = memcmp( (const void *) a, (const void *) b, (size_t) common ); + if ( order != 0 ) { return order < 0 ? -1 : 1; } + } + return a_length < b_length ? -1 : ( a_length > b_length ? 1 : 0 ); +} + +// the length of a NUL-terminated key at a call site, bounded by the storage it +// has to fit: a key one byte longer than the bound is refused, never truncated +inline int32_t TableKeyLength( const char * key, int32_t bound ) +{ + if ( key == NULL ) { return 0; } + for ( int32_t i = 0; i <= bound; i++ ) { if ( key[i] == 0 ) { return i; } } + return bound + 1; // longer than the bound: the caller refuses it +} + +// A KEY IS DATA AND A LENGTH, and the length is CARRIED, never recomputed +// (§2.8, §3). A string(N) key holds any byte a wire or a text can spell, +// U+0000 included, so a lookup that measures to the first NUL answers that "a" +// and "a", 0, "b" are the same key: the first entry is found, RESET, and +// relabeled with the second key, which deletes an entry the report never +// mentions. Every internal lookup and every insertion takes this pair, and the +// public const char * surface builds one and is a wrapper over it. +struct TableMapKeyRef +{ + const char * data; + int32_t length; +}; + +// ---- the storage: SIXTEEN BYTES in the holder's record (§2.8, §7.2) ---- +// +// An int64 self-relative reference to the entry array and an int32 count, then +// padding to eight. The reference is a TableRef like a pointer's: in the arena +// it names the builder's HEAD, in a region it is the delta from the slot to +// the first entry, and 0 is the empty map in both. +template struct TableMap +{ + TableRef entries; + int32_t count = 0; // the LIVE count, in both forms + int32_t padding = 0; // named, so the record has no unwritten byte in it + + // ---- the CONST form: a locked region, a loaded one, an opened cook ---- + // + // One surface over one encoding (§6.3). A region reference resolves from + // the slot's own address, so every one of these is a member and needs no + // base and no context. + const Entry * Entries() const + { + return entries.value != 0 ? (const Entry *) ( (const uint8_t *) &entries + entries.value ) : NULL; + } + int32_t size() const { return count; } + + // FIND: floor( log2 n ) + 1 key compares, in place, no allocation. NULL + // when absent, and on a map[K]*T the RESOLVED pointer, which is what a + // pointer field's accessor answers. + template const Entry * FindEntry( Key key ) const + { + const Entry * base = Entries(); + int32_t low = 0, high = count; + while ( low < high ) + { + const int32_t mid = low + ( high - low ) / 2; + const int order = TableEntryOrder( base[mid], key ); + if ( order == 0 ) { return base + mid; } + if ( order < 0 ) { low = mid + 1; } else { high = mid; } + } + return NULL; + } + // the return type is DEDUCED, so it is worked out when a call site + // instantiates Find and not when the holder's record declares the slot — + // which is what lets the entry's own overloads be declared after it + template auto Find( Key key ) const + { + return TableEntryFound( FindEntry( key ) ); + } + + // ---- iteration: ASCENDING key order, the key beside the value ---- + // + // A proxy BY VALUE, the keyed array's shape (§2.4): for ( auto [ key, + // value ] : map ). It carries no iterator_traits, for the reason + // TableKeyed's does not (§13.9). + struct ConstEntry + { + decltype( TableEntryKey( TableDeclRef() ) ) key; + decltype( TableEntryFound( (const Entry *) NULL ) ) value; + }; + + struct ConstIterator + { + const Entry * at; + ConstEntry operator*() const { return ConstEntry{ TableEntryKey( *at ), TableEntryFound( at ) }; } + ConstIterator & operator++() { at++; return *this; } + bool operator==( const ConstIterator & other ) const { return at == other.at; } + bool operator!=( const ConstIterator & other ) const { return at != other.at; } + }; + + ConstIterator begin() const { return ConstIterator{ Entries() }; } + ConstIterator end() const { return ConstIterator{ Entries() + count }; } +}; + +// ---- the BUILDER's side: a head, and segments that never move (§2.8, §6.4) ---- +// +// The head is a small node in the arena holding the segment chain, the live +// count and the dead count, allocated when the first entry is inserted. Each +// segment is a fixed number of entries carved from one call to the allocator +// pair. An entry's address is stable for the arena's life, so a value handed +// back by an insert stays valid while other entries arrive. +struct TableMapHead +{ + TableRef first; // the arena offset of the first segment + TableRef last; // and of the one an insert appends into + int32_t live; + int32_t dead; +}; + +template struct TableMapSegment +{ + TableRef next; + int32_t used; // entries carved from this segment + int32_t padding; + uint32_t dead[ ( kTableMapSegmentEntries + 31 ) / 32 ]; // Erase marks one bit, never the entry + Entry entries[ kTableMapSegmentEntries ]; +}; + +inline bool TableMapSegmentDead( const uint32_t * dead, int32_t index ) +{ + return ( dead[ index / 32 ] & ( 1u << ( index % 32 ) ) ) != 0; +} + +// ---- the ORDERED CURSOR the four writing walks read (§2.8) ---- +// +// Measure, Save, Lock and Cook each write a map's entries in ascending key +// order with no key twice, deriving the order from the builder's entries as +// each walk derives the numbering (§3.1). Nothing passes between them, so +// measure == save over a map is a real check on two sorts agreeing. +// +// A REGION is already sorted, so its cursor is the array in place and +// allocates nothing. The BUILDER's is the sort: an array of entry pointers +// allocated through the pair and released before the walk returns, because +// sorting the segments themselves would move entries whose addresses a caller +// holds. +template struct TableMapCursor +{ + const Entry * const * order = NULL; // the builder's form: sorted pointers + const Entry * entries = NULL; // the region's form: the array in place + int32_t count = 0; + TableAllocator allocator; + bool ok = false; + const Entry * operator[]( int32_t index ) const + { + return order != NULL ? order[index] : entries + index; + } +}; + +// heapsort: O( n log n ) once per map, no recursion, no allocation past the +// pointer array the caller already paid for +template inline void TableMapSort( const Entry ** order, int32_t count ) +{ + for ( int32_t start = count / 2 - 1; start >= 0; start-- ) + { + int32_t root = start; + for ( ;; ) + { + int32_t child = 2 * root + 1; + if ( child >= count ) { break; } + if ( child + 1 < count && TableEntryOrder( *order[child], *order[child + 1] ) < 0 ) { child++; } + if ( TableEntryOrder( *order[root], *order[child] ) >= 0 ) { break; } + const Entry * swap = order[root]; order[root] = order[child]; order[child] = swap; + root = child; + } + } + for ( int32_t end = count - 1; end > 0; end-- ) + { + const Entry * swap = order[0]; order[0] = order[end]; order[end] = swap; + int32_t root = 0; + for ( ;; ) + { + int32_t child = 2 * root + 1; + if ( child >= end ) { break; } + if ( child + 1 < end && TableEntryOrder( *order[child], *order[child + 1] ) < 0 ) { child++; } + if ( TableEntryOrder( *order[root], *order[child] ) >= 0 ) { break; } + const Entry * hold = order[root]; order[root] = order[child]; order[child] = hold; + root = child; + } + } +} + +// the REGION form: the array is already sorted, so the cursor is the array +template +inline TableMapCursor TableMapOrder( const TableRegionCtx &, const TableMap & map ) +{ + TableMapCursor cursor; + cursor.entries = map.Entries(); + cursor.count = map.count; + cursor.ok = true; + return cursor; +} + +// the BUILDER's form: gather the LIVE entries out of the segment chain in +// insertion order, then sort. A dead entry costs nothing on any wire (§2.8). +template +inline TableMapCursor TableMapOrder( const TableArena & arena, const TableMap & map ) +{ + TableMapCursor cursor; + cursor.allocator = arena.allocator; + cursor.count = map.count; + if ( map.entries.value == 0 || map.count <= 0 ) { cursor.ok = map.count == 0; cursor.count = 0; return cursor; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + if ( head->live != map.count ) { return cursor; } // the slot and the head disagree: refused, never guessed + const Entry ** order = (const Entry **) arena.allocator.alloc( arena.allocator.context, (int64_t) map.count * (int64_t) sizeof( const Entry * ) ); + if ( order == NULL ) { return cursor; } + int32_t at = 0; + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 && at < map.count ) + { + const TableMapSegment * segment = (const TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used && at < map.count; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + order[at++] = segment->entries + i; + } + segment_ref = segment->next; + } + if ( at != map.count ) + { + arena.allocator.free( arena.allocator.context, order ); + return cursor; + } + TableMapSort( order, map.count ); + cursor.order = order; + cursor.ok = true; + return cursor; +} + +template +inline TableMapCursor TableMapOrder( const TableArenaCtx & ctx, const TableMap & map ) +{ + return TableMapOrder( *ctx.arena, map ); +} + +template inline void TableMapRelease( TableMapCursor & cursor ) +{ + if ( cursor.order != NULL ) { cursor.allocator.free( cursor.allocator.context, (void *) cursor.order ); } + cursor.order = NULL; +} + +// ---- the builder's five (§2.8) ---- +// +// Insert APPENDS after one LINEAR SCAN of the live entries for the key it may +// replace, Find is that same scan, and Erase is the scan and one bit. The +// builder builds NO INDEX, and that is a rule: the sort happens once, at Lock, +// Save or Cook, and every lookup that matters runs over the sorted region. + +// the head, allocated when the first entry is inserted +template +inline TableMapHead * TableMapReach( TableWorker & worker, TableMap & map ) +{ + if ( worker.arena == NULL || worker.arena->locked ) { return NULL; } + if ( map.entries.value != 0 ) { return (TableMapHead *) TableArenaAt( *worker.arena, (uint32_t) map.entries.value ); } + uint32_t at = 0; + TableMapHead * head = (TableMapHead *) worker.AllocRaw( (int64_t) sizeof( TableMapHead ), (int64_t) alignof( TableMapHead ), at ); + if ( head == NULL ) { return NULL; } + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + map.entries.value = (int64_t) at; + return head; +} + +// one entry's storage, appended: the current segment when it has room, a new +// one carved from one call to the pair when it does not +template +inline Entry * TableMapAppend( TableWorker & worker, TableMapHead * head, TableMap & map ) +{ + TableMapSegment * segment = NULL; + if ( head->last.value != 0 ) + { + segment = (TableMapSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + if ( segment->used >= kTableMapSegmentEntries ) { segment = NULL; } + } + if ( segment == NULL ) + { + uint32_t at = 0; + segment = (TableMapSegment *) worker.AllocRaw( (int64_t) sizeof( TableMapSegment ), (int64_t) alignof( TableMapSegment ), at ); + if ( segment == NULL ) { return NULL; } // the arena could not carve another segment + segment->next.value = 0; + segment->used = 0; + segment->padding = 0; + for ( int32_t i = 0; i < (int32_t) ( sizeof( segment->dead ) / sizeof( segment->dead[0] ) ); i++ ) { segment->dead[i] = 0; } + if ( head->last.value != 0 ) + { + TableMapSegment * previous = (TableMapSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + previous->next.value = (int64_t) at; + } + else + { + head->first.value = (int64_t) at; + } + head->last.value = (int64_t) at; + } + Entry * entry = segment->entries + segment->used; + segment->used++; + head->live++; + map.count++; + return entry; +} + +// the LINEAR SCAN: the live entries in insertion order, O( n ) key compares +template +inline Entry * TableMapScan( const TableArena & arena, const TableMap & map, Key key ) +{ + if ( map.entries.value == 0 ) { return NULL; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( TableEntryOrder( segment->entries[i], key ) == 0 ) { return segment->entries + i; } + } + segment_ref = segment->next; + } + return NULL; +} + +// ERASE marks the entry DEAD, one bit in the segment's slot and not in the +// entry table, and decrements the live count. Its storage is reclaimed at +// RESET and never reused mid-build, because reusing a slot would make "an +// entry's address is stable" false for exactly one case. +template +inline bool TableMapErase( TableArena & arena, TableMap & map, Key key ) +{ + if ( map.entries.value == 0 ) { return false; } + TableMapHead * head = (TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( TableEntryOrder( segment->entries[i], key ) != 0 ) { continue; } + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + map.count--; + return true; + } + segment_ref = segment->next; + } + return false; +} + +// ---- iterate on the BUILDER: INSERTION order, live entries only (§2.8) ---- +template struct TableMapEach +{ + const TableArena * arena; + TableRef first; + + struct Iterator + { + const TableArena * arena; + TableMapSegment * segment; + int32_t index; + + void Skip() + { + for ( ;; ) + { + if ( segment == NULL ) { return; } + if ( index >= segment->used ) + { + segment = segment->next.value != 0 ? (TableMapSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + index = 0; + continue; + } + if ( TableMapSegmentDead( segment->dead, index ) ) { index++; continue; } + return; + } + } + auto operator*() const { return TableEntryEach( segment->entries + index ); } + Iterator & operator++() { index++; Skip(); return *this; } + bool operator==( const Iterator & other ) const { return segment == other.segment && index == other.index; } + bool operator!=( const Iterator & other ) const { return !( *this == other ); } + }; + + Iterator begin() const + { + Iterator it = { arena, first.value != 0 ? (TableMapSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL, 0 }; + it.Skip(); + return it; + } + Iterator end() const { Iterator it = { arena, NULL, 0 }; return it; } +}; + +template +inline TableMapEach TableMapEachOf( const TableArena & arena, const TableMap & map ) +{ + TableMapEach each = { &arena, TableRef() }; + if ( map.entries.value != 0 ) + { + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + each.first = head->first; + } + return each; +} + +// ---- the LOAD side: where a decoded entry lands (§2.8) ---- +// +// THE READER TRUSTS NOTHING and spends one compare per entry. Every load path +// applies the same rules and produces one report (§4), so the region load of +// §6.5 and LoadBuilder never disagree about a wire. These two shapes are what +// makes that true with one generated decoder: a REGION carves the entry array +// out of the holder node's own extent, and the TOOL's path appends into the +// builder's arena, and the decoder above them cannot tell which it has. + +// The node's extent cursor is TableExtentCarve, the extent runtime's (§2.8, +// §2.9): a map's whole entry array is carved first, then, entry by entry in +// key order, the arrays of any list or map an entry's value holds by value. + +// TableMapFill is one map field being decoded: where the next entry lands, and +// the entry that last LANDED, which is what the ascending check compares +// against. +template struct TableMapFill +{ + TableMap * map = NULL; + Entry * array = NULL; // the region path: the carved array + int32_t capacity = 0; + TableWorker * worker = NULL; // the TOOL's path + bool ok = false; +}; + +template +inline TableMapFill TableMapFillBegin( const TableNodeMap & nodes, TableMap & map, uint32_t n ) +{ + TableMapFill fill; + fill.map = ↦ + map.entries.value = 0; + map.count = 0; + if ( nodes.carve == NULL ) { return fill; } + if ( nodes.carve->worker != NULL ) + { + fill.worker = nodes.carve->worker; // the tool's path: the arena carves + fill.ok = true; + return fill; + } + const int64_t align = (int64_t) alignof( Entry ); + uint8_t * base = (uint8_t *) ( ( (uintptr_t) nodes.carve->at + (uintptr_t) ( align - 1 ) ) & ~( (uintptr_t) ( align - 1 ) ) ); + const int64_t bytes = (int64_t) n * (int64_t) sizeof( Entry ); + const int64_t pad = (int64_t) ( base - nodes.carve->at ); + if ( pad + bytes > nodes.carve->left ) { return fill; } // the measure and the load disagree: refused + nodes.carve->at = base + bytes; + nodes.carve->left -= pad + bytes; + fill.array = (Entry *) base; + fill.capacity = (int32_t) n; + map.entries.value = (int64_t) ( base - (const uint8_t *) &map.entries ); + fill.ok = true; + return fill; +} + +// the entry that last LANDED — NULL before the first +template inline Entry * TableMapFillLast( TableMapFill & fill ) +{ + if ( fill.map->count <= 0 ) { return NULL; } + if ( fill.array != NULL ) { return fill.array + ( fill.map->count - 1 ); } + return TableMapLive( *fill.worker->arena, *fill.map, fill.map->count - 1 ); +} + +// the next slot, at the entry type's declared defaults +template inline Entry * TableMapFillNext( TableMapFill & fill ) +{ + if ( fill.array != NULL ) + { + if ( fill.map->count >= fill.capacity ) { return NULL; } + Entry * entry = fill.array + fill.map->count; + TableReset( *entry ); + fill.map->count++; + return entry; + } + TableMapHead * head = TableMapReach( *fill.worker, *fill.map ); + if ( head == NULL ) { return NULL; } + Entry * entry = TableMapAppend( *fill.worker, head, *fill.map ); + if ( entry != NULL ) { TableReset( *entry ); } + return entry; +} + +// A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): at the first entry whose key +// kind disagrees with the reader's declaration the map resets to EMPTY, one +// kind_mismatch is counted for the map, and its remaining bytes are skipped. +template inline void TableMapFillReset( TableMapFill & fill ) +{ + if ( fill.array != NULL ) + { + fill.map->entries.value = 0; + fill.map->count = 0; + return; + } + if ( fill.map->entries.value != 0 ) + { + TableMapHead * head = (TableMapHead *) TableArenaAt( *fill.worker->arena, (uint32_t) fill.map->entries.value ); + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + } + fill.map->count = 0; +} + +// an EMPTY map's reference is null in both encodings, so a load that placed +// nothing leaves the slot exactly as a Reset does +template inline void TableMapFillEnd( TableMapFill & fill ) +{ + if ( fill.array != NULL && fill.map->count == 0 ) { fill.map->entries.value = 0; } +} + +// the k-th LIVE entry of a builder map, in insertion order — what the tool +// path's ascending check compares against +template +inline Entry * TableMapLive( const TableArena & arena, const TableMap & map, int32_t index ) +{ + if ( map.entries.value == 0 ) { return NULL; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + int32_t at = 0; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( at == index ) { return segment->entries + i; } + at++; + } + segment_ref = segment->next; + } + return NULL; +} + +// ---- LoadMeasure's term, from the FRAMING alone (§2.8, §6.5) ---- +// +// LoadMeasure's term for a map is N x sizeof( Entry ) rounded to +// alignof( Entry ), AT EVERY DEPTH. N is framing and not a value, so this +// reads no field: it walks the map's own header and, where an entry's value +// holds a map or a list of its own, the entries' headers under it. The caller +// owns the allocation precisely so it can refuse a number it did not expect. +// Every -1 carries its REASON (§6.5): the int32 cap first, because a count +// past it cannot fit any body, and then the body's own L, the one rule a +// list's term answers by. +// A MAP ENTRY'S SMALLEST WIRE FOOTPRINT that commands one storage unit is its +// own L and the body's terminator, and under this form's variable lengths that +// footprint is TWO BYTES (docs/SPEC-TABLES.md §4.2). It is what bounds the N a +// map's L can carry, and therefore what a LoadMeasure may be asked for. +static const int64_t kTableMapEntryFloor = 2; + +inline bool TableMapWireExtent( const uint8_t * body, int64_t length, int64_t & at, + int64_t entry_size, int64_t entry_align, TableWireExtentFn inner, + const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } // no array header: nothing rides + if ( r.get8() != 13 ) { return true; } // not an array of tables: §4's ordinary kind mismatch + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + if ( n > (uint64_t) INT32_MAX ) { reason = count_over_extent_cap; return false; } + const int64_t rest = length - r.offset; + if ( n > (uint64_t) ( rest / kTableMapEntryFloor ) ) { reason = count_over_length; return false; } // an N the map's L cannot carry + at = ( at + entry_align - 1 ) & ~( entry_align - 1 ); + at += (int64_t) n * entry_size; + if ( inner == NULL ) { return true; } // no map below an entry: one depth is the whole term + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } // framing damage: the load reports it + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +// ---- the TEXT form's placement (docs/SPEC-TABLES.md §2.8, §16) ---- +// +// The text is a plain JSON object keyed by the KEY, and the generic walk fills +// it through the ENTRY'S OWN descriptor — so all it needs from here is one +// entry at one key, handed back at its defaults. It is the builder's Insert +// with the ENTRY returned rather than its value, because the walk writes the +// value through a field row and not through a typed pointer. +// +// THE ONE INSERTION PRIMITIVE. Lookup, reset, allocation and the KEY COPY are +// all here, so no caller mutates an entry this did not create and no caller +// relabels one it found. A key is copied only when an entry is created, which +// is what makes a duplicate key leave the identity it matched untouched. NULL +// is one thing and one thing only: the arena refused. +template +inline Entry * TableMapPlace( TableWorker & worker, TableMap & map, Key key ) +{ + if ( worker.arena == NULL ) { return NULL; } + Entry * found = TableMapScan( *worker.arena, map, key ); + if ( found != NULL ) + { + TableResetMapValue( *found ); // a repeated key is LAST-WINS, whole + return found; + } + TableMapHead * head = TableMapReach( worker, map ); + if ( head == NULL ) { return NULL; } + Entry * entry = TableMapAppend( worker, head, map ); + if ( entry == NULL ) { return NULL; } + TableReset( *entry ); + TableEntrySetKey( *entry, key ); + return entry; +} + +// ---- the OPTIONAL RUNTIME INDEX (§2.8) ---- +// +// Open addressing with LINEAR PROBING over the sorted array, built AT LOAD for +// a map large enough that log n compares over a cold array cost more than one +// hash and a probe. IT IS NEVER STORED: the caller measures it, owns its +// storage, builds it in one pass and releases it whenever. +// +// ITS HASH AND ITS LOAD FACTOR ARE NOT A CROSS-PORT CONTRACT, and that is a +// rule. What a port is held to is the CONTRACT of the lookup: the same value +// the sorted array's Find returns for the same key, and no allocation past the +// storage the caller handed in. +struct TableMapIndex +{ + int32_t * slots = NULL; // entry indices, +1; 0 is an empty slot + int32_t capacity = 0; + bool good = false; +}; + +// this runtime's own, and no port reproduces it: fnv1a64 over the key's bytes +inline uint64_t TableMapHash( const void * bytes, int32_t length ) +{ + uint64_t hash = 0xCBF29CE484222325ull; + const uint8_t * at = (const uint8_t *) bytes; + for ( int32_t i = 0; i < length; i++ ) { hash ^= (uint64_t) at[i]; hash *= 0x100000001B3ull; } + return hash; +} +inline uint64_t TableMapHash( uint64_t key ) { return TableMapHash( (const void *) &key, (int32_t) sizeof( key ) ); } + +// this runtime's own load factor, and no port reproduces it either: the next +// power of two at or above twice the count, so a probe run stays short +inline int32_t TableMapIndexSlots( int32_t count ) +{ + int32_t slots = 8; + while ( slots < count * 2 ) { slots *= 2; } + return slots; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_MAP + +#ifndef MAPDEMO_SCHEMA_TABLE_LIST +#define MAPDEMO_SCHEMA_TABLE_LIST + +namespace mapdemo { + +// ---- an UNBOUNDED ARRAY: a counted array whose count the data decides (§2.9) ---- +// +// On the wire, in a region and in a cook a list is the kind 14 body a [..N]T +// writes, its elements by-value records inside the holder's node extent. What +// this adds is the slot, a builder that appends into segments that never +// move, and a const surface that indexes and iterates in place. There is no +// sort, no key and no lookup: the order is INSERTION order, and it is +// identity the way position is identity in a fixed array. + +// elements carved from ONE call to the allocator pair. A new segment is +// appended when the current one fills, and nothing ever moves (§6.4) +static const int32_t kTableListSegmentElements = 32; + +// THE ELEMENT STORAGE: T itself, and a TableRef slot for a []*T, whose +// elements are references exactly as a pointer field's slot is (§2.1) +template struct TableListStorage { typedef T Element; }; +template struct TableListStorage { typedef TableRef Element; }; + +// WHAT THE CONST FORM ANSWERS: the element by reference, and on a []*T the +// RESOLVED pointer, one add on the self-relative delta, NULL for a null slot, +// exactly as At answers it (§6.2, §6.3) +template struct TableListConst +{ + typedef const T & Result; + static Result At( const T * element ) { return *element; } +}; +template struct TableListConst +{ + typedef const T * Result; + static Result At( const TableRef * element ) + { + return element->value != 0 ? (const T *) ( (const uint8_t *) element + element->value ) : NULL; + } +}; + +// ---- the storage: SIXTEEN BYTES in the holder's record (§2.9, §7.2) ---- +// +// An int64 self-relative reference to the element array and an int32 count, +// then padding to eight. The reference is a TableRef like a pointer's: in the +// arena it names the builder's HEAD, in a region it is the delta from the slot +// to the first element, and 0 is the empty list in both. It is the map's slot +// exactly, because it is the same two facts. +template struct TableList +{ + typedef typename TableListStorage::Element Element; + + TableRef elements; + int32_t count = 0; // the LIVE count, in both forms + int32_t padding = 0; // named, so the record has no unwritten byte in it + + // ---- the CONST form: a locked region, a loaded one, an opened cook ---- + // + // One surface over one encoding (§6.3). A region reference resolves from + // the slot's own address, so every one of these is a member and needs no + // base and no context. + const Element * Elements() const + { + return elements.value != 0 ? (const Element *) ( (const uint8_t *) &elements + elements.value ) : NULL; + } + int32_t size() const { return count; } + + // INDEXING IS BOUNDS-CHECKED IN EVERY BUILD (§2.4, §2.9): the extent is a + // number that CAME FROM A FILE, so an index past it is not a mistake a + // release build gets to make cheaply. There is no undefined-behavior path + // here in any configuration. The assert carries the message where a + // debugger can read it and NDEBUG removes that. The fatal is what stands + // after it. Both go through the hooks: define schema_assert and + // schema_fatal and this refusal lands in your own handler. + void RefuseIndex( int32_t index ) const + { + if ( (uint32_t) index >= (uint32_t) count ) + { + schema_assert( false && "an unbounded array is indexed inside its count, which came from a file" ); + schema_fatal(); + } + } + typename TableListConst::Result operator[]( int32_t index ) const + { + RefuseIndex( index ); + return TableListConst::At( Elements() + index ); + } + + // ---- iteration: INDEX order, the element and no key ---- + // + // It carries no iterator_traits, for the reason TableKeyed's does not + // (§13.9). + struct ConstIterator + { + const Element * at; + typename TableListConst::Result operator*() const { return TableListConst::At( at ); } + ConstIterator & operator++() { at++; return *this; } + bool operator==( const ConstIterator & other ) const { return at == other.at; } + bool operator!=( const ConstIterator & other ) const { return at != other.at; } + }; + + ConstIterator begin() const { return ConstIterator{ Elements() }; } + ConstIterator end() const { return ConstIterator{ Elements() + count }; } +}; + +// ---- the BUILDER's side: a head, and segments that never move (§2.9, §6.4) ---- +// +// The head is a small node in the arena holding the segment chain, the live +// count and the dead count, allocated when the first element is added. Each +// segment is a fixed number of elements carved from one call to the allocator +// pair. An element's address is stable for the arena's life, so a T * handed +// back by Add stays valid while other elements arrive. +struct TableListHead +{ + TableRef first; // the arena offset of the first segment + TableRef last; // and of the one an Add appends into + int32_t live; + int32_t dead; +}; + +template struct TableListSegment +{ + TableRef next; + int32_t used; // elements carved from this segment + int32_t padding; + uint32_t dead[ ( kTableListSegmentElements + 31 ) / 32 ]; // Erase marks one bit, never the element + Element elements[ kTableListSegmentElements ]; +}; + +inline bool TableListSegmentDead( const uint32_t * dead, int32_t index ) +{ + return ( dead[ index / 32 ] & ( 1u << ( index % 32 ) ) ) != 0; +} + +// the head, allocated when the first element is added +template +inline TableListHead * TableListReach( TableWorker & worker, TableList & list ) +{ + if ( worker.arena == NULL || worker.arena->locked ) { return NULL; } + if ( list.elements.value != 0 ) { return (TableListHead *) TableArenaAt( *worker.arena, (uint32_t) list.elements.value ); } + uint32_t at = 0; + TableListHead * head = (TableListHead *) worker.AllocRaw( (int64_t) sizeof( TableListHead ), (int64_t) alignof( TableListHead ), at ); + if ( head == NULL ) { return NULL; } + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + list.elements.value = (int64_t) at; + return head; +} + +// one element's storage, appended: the current segment when it has room, a +// new one carved from one call to the pair when it does not. NULL means NOT +// ADDED: an arena that cannot carve another segment, or a count at the int32 +// cap (§2.2, §2.9). +template +inline typename TableList::Element * TableListAppend( TableWorker & worker, TableListHead * head, TableList & list ) +{ + typedef typename TableList::Element Element; + if ( list.count >= INT32_MAX ) { return NULL; } // the int32 storage cap + TableListSegment * segment = NULL; + if ( head->last.value != 0 ) + { + segment = (TableListSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + if ( segment->used >= kTableListSegmentElements ) { segment = NULL; } + } + if ( segment == NULL ) + { + uint32_t at = 0; + segment = (TableListSegment *) worker.AllocRaw( (int64_t) sizeof( TableListSegment ), (int64_t) alignof( TableListSegment ), at ); + if ( segment == NULL ) { return NULL; } // the arena could not carve another segment + segment->next.value = 0; + segment->used = 0; + segment->padding = 0; + for ( int32_t i = 0; i < (int32_t) ( sizeof( segment->dead ) / sizeof( segment->dead[0] ) ); i++ ) { segment->dead[i] = 0; } + if ( head->last.value != 0 ) + { + TableListSegment * previous = (TableListSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + previous->next.value = (int64_t) at; + } + else + { + head->first.value = (int64_t) at; + } + head->last.value = (int64_t) at; + } + Element * element = segment->elements + segment->used; + segment->used++; + head->live++; + list.count++; + return element; +} + +// ADD, whole: the head, the append, and the element at its declared defaults +// (§2.9). The text form's placement is this same call, because a list has no +// key to place under (§16). +template +inline typename TableList::Element * TableListPlace( TableWorker & worker, TableList & list ) +{ + typedef typename TableList::Element Element; + TableListHead * head = TableListReach( worker, list ); + if ( head == NULL ) { return NULL; } + Element * element = TableListAppend( worker, head, list ); + if ( element == NULL ) { return NULL; } + new ( element ) Element(); // value-init: the declared defaults, and null for a slot + return element; +} + +// ERASE, ADDRESSED BY THE POINTER (§2.9): the element Add handed back is the +// handle, because a list has no key and the address is the one thing the +// builder promises never moves (§6.4). It marks the element DEAD, one bit in +// the segment's slot and not in the element storage, and decrements the live +// count. False when the pointer is not this list's. Its storage is reclaimed +// at RESET and never reused mid-build, the map's rule for the map's reason. +template +inline bool TableListErase( TableArena & arena, TableList & list, const typename TableList::Element * element ) +{ + typedef typename TableList::Element Element; + if ( list.elements.value == 0 || element == NULL ) { return false; } + TableListHead * head = (TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableListSegment * segment = (TableListSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + if ( element >= segment->elements && element < segment->elements + segment->used ) + { + const int32_t i = (int32_t) ( element - segment->elements ); + if ( TableListSegmentDead( segment->dead, i ) ) { return false; } // already erased + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + list.count--; + return true; + } + segment_ref = segment->next; + } + return false; +} + +// ---- iterate on the BUILDER: INDEX order, live elements only (§2.9) ---- +template struct TableListEach +{ + typedef typename TableList::Element Element; + const TableArena * arena; + TableRef first; + + struct Iterator + { + const TableArena * arena; + TableListSegment * segment; + int32_t index; + + void Skip() + { + for ( ;; ) + { + if ( segment == NULL ) { return; } + if ( index >= segment->used ) + { + segment = segment->next.value != 0 ? (TableListSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + index = 0; + continue; + } + if ( TableListSegmentDead( segment->dead, index ) ) { index++; continue; } + return; + } + } + Element * operator*() const { return segment->elements + index; } + Iterator & operator++() { index++; Skip(); return *this; } + bool operator==( const Iterator & other ) const { return segment == other.segment && index == other.index; } + bool operator!=( const Iterator & other ) const { return !( *this == other ); } + }; + + Iterator begin() const + { + Iterator it = { arena, first.value != 0 ? (TableListSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL, 0 }; + it.Skip(); + return it; + } + Iterator end() const { Iterator it = { arena, NULL, 0 }; return it; } +}; + +template +inline TableListEach TableListEachOf( const TableArena & arena, const TableList & list ) +{ + TableListEach each = { &arena, TableRef() }; + if ( list.elements.value != 0 ) + { + const TableListHead * head = (const TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + each.first = head->first; + } + return each; +} + +// ---- the INDEX-ORDER CURSOR the four writing walks read (§2.9) ---- +// +// Measure, Save, Lock and Cook each visit a list's live elements in the order +// they were added, and they allocate nothing to do it: a region's cursor is +// the array in place, and the builder's walks the segment chain. Indexing the +// builder's form is SEQUENTIAL by construction, every walk steps i, i + 1, +// i + 2, so the cursor remembers where the last access landed and moves one +// live slot per step. An access behind the memo restarts from the first +// segment, which no walk here does. +template struct TableListCursor +{ + const Element * elements = NULL; // the region's form: the array in place + const TableArena * arena = NULL; // the builder's form: the segments + TableRef first; + int32_t count = 0; + bool ok = false; + // the memo: the segment and slot the last access landed on, and the live + // index that slot holds + mutable const TableListSegment * segment = NULL; + mutable int32_t within = -1; + mutable int32_t logical = -1; + + const Element * At( int32_t index ) const + { + if ( elements != NULL ) { return elements + index; } + if ( segment == NULL || index < logical ) + { + segment = first.value != 0 ? (const TableListSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL; + within = -1; + logical = -1; + } + while ( logical < index ) + { + for ( ;; ) + { + within++; + while ( segment != NULL && within >= segment->used ) + { + segment = segment->next.value != 0 ? (const TableListSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + within = 0; + } + if ( segment == NULL ) { return NULL; } // the slot and the head disagree + if ( !TableListSegmentDead( segment->dead, within ) ) { break; } + } + logical++; + } + return segment->elements + within; + } + const Element & operator[]( int32_t index ) const { return *At( index ); } +}; + +// the REGION form: the array is the cursor +template +inline TableListCursor::Element> TableListElements( const TableRegionCtx &, const TableList & list ) +{ + TableListCursor::Element> cursor; + cursor.elements = list.Elements(); + cursor.count = list.count; + cursor.ok = true; + return cursor; +} + +// the BUILDER's form: the live elements out of the segment chain, in the +// order they were added. A dead element costs nothing on any wire (§2.9). +template +inline TableListCursor::Element> TableListElements( const TableArena & arena, const TableList & list ) +{ + TableListCursor::Element> cursor; + cursor.arena = &arena; + cursor.count = list.count; + if ( list.elements.value == 0 || list.count <= 0 ) { cursor.ok = list.count == 0; cursor.count = 0; return cursor; } + const TableListHead * head = (const TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + if ( head->live != list.count ) { return cursor; } // the slot and the head disagree: refused, never guessed + cursor.first = head->first; + cursor.ok = true; + return cursor; +} + +template +inline TableListCursor::Element> TableListElements( const TableArenaCtx & ctx, const TableList & list ) +{ + return TableListElements( *ctx.arena, list ); +} + +// ---- the LOAD side: where a decoded element lands (§2.9) ---- +// +// The same two shapes the map's fill takes, because the decoder above them +// cannot tell which it has: a REGION carves the element array out of the +// holder node's own extent, PRE-ORDER, and the TOOL's path appends into the +// builder's arena. +template struct TableListFill +{ + typedef typename TableList::Element Element; + TableList * list = NULL; + Element * array = NULL; // the region path: the carved array + int32_t capacity = 0; + TableWorker * worker = NULL; // the TOOL's path + bool ok = false; + bool refused = false; // a count above the int32 cap on the tool's path: LoadBuilder answers NULL +}; + +template +inline TableListFill TableListFillBegin( const TableNodeMap & nodes, TableList & list, uint64_t n ) +{ + typedef typename TableList::Element Element; + TableListFill fill; + fill.list = &list; + list.elements.value = 0; + list.count = 0; + if ( nodes.carve == NULL ) { return fill; } + if ( n > (uint64_t) INT32_MAX ) + { + // A COUNT ABOVE THE int32 STORAGE CAP (§2.2, §2.9): into a region it was + // refused by LoadMeasure before this ran, and into a builder it is the + // refusal LoadBuilder answers NULL for, moving no counter + fill.refused = nodes.carve->worker != NULL; + return fill; + } + if ( nodes.carve->worker != NULL ) + { + fill.worker = nodes.carve->worker; // the tool's path: the arena carves + fill.ok = true; + return fill; + } + const int64_t align = (int64_t) alignof( Element ); + uint8_t * base = (uint8_t *) ( ( (uintptr_t) nodes.carve->at + (uintptr_t) ( align - 1 ) ) & ~( (uintptr_t) ( align - 1 ) ) ); + const int64_t bytes = (int64_t) n * (int64_t) sizeof( Element ); + const int64_t pad = (int64_t) ( base - nodes.carve->at ); + if ( pad + bytes > nodes.carve->left ) { return fill; } // the measure and the load disagree: refused + nodes.carve->at = base + bytes; + nodes.carve->left -= pad + bytes; + fill.array = (Element *) base; + fill.capacity = (int32_t) n; + list.elements.value = (int64_t) ( base - (const uint8_t *) &list.elements ); + fill.ok = true; + return fill; +} + +// the next slot, at the element's declared defaults. NULL when the arena +// could not carve, which the decoder reports as framing damage +template inline typename TableList::Element * TableListFillNext( TableListFill & fill ) +{ + typedef typename TableList::Element Element; + if ( fill.array != NULL ) + { + if ( fill.list->count >= fill.capacity ) { return NULL; } + Element * element = fill.array + fill.list->count; + new ( element ) Element(); + fill.list->count++; + return element; + } + return TableListPlace( *fill.worker, *fill.list ); +} + +// A SLOT WHOSE ELEMENT NEVER LANDED is given back (§2.9, §4): the array keeps +// what it decoded, and an element whose own framing gave out before one byte +// of it decoded was not decoded. The region's form uncounts it, and the builder's +// marks it dead, which is what the storage rule allows mid-build. +template inline void TableListFillDrop( TableListFill & fill ) +{ + typedef typename TableList::Element Element; + if ( fill.array != NULL ) + { + if ( fill.list->count > 0 ) { fill.list->count--; } + return; + } + if ( fill.list->elements.value == 0 ) { return; } + TableListHead * head = (TableListHead *) TableArenaAt( *fill.worker->arena, (uint32_t) fill.list->elements.value ); + if ( head->last.value == 0 ) { return; } + TableListSegment * segment = (TableListSegment *) TableArenaAt( *fill.worker->arena, (uint32_t) head->last.value ); + if ( segment->used <= 0 ) { return; } + const int32_t i = segment->used - 1; + if ( TableListSegmentDead( segment->dead, i ) ) { return; } + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + fill.list->count--; +} + +// an EMPTY list's reference is null in both encodings, so a load that placed +// nothing leaves the slot exactly as a Reset does +template inline void TableListFillEnd( TableListFill & fill ) +{ + if ( fill.array != NULL && fill.list->count == 0 ) { fill.list->elements.value = 0; } +} + +// ---- LoadMeasure's term, from the FRAMING alone (§2.9, §6.5) ---- +// +// N x sizeof( T ) rounded to alignof( T ), AT EVERY DEPTH. N is framing and +// not a value, so this reads no field: it walks the list's own header and, +// where a table element holds a list or a map of its own, the elements' +// headers under it. Every -1 carries its REASON (§6.5): the int32 cap first, +// because a count past it cannot fit any body, and then the body's own L. +inline bool TableListWireExtent( const uint8_t * body, int64_t length, int64_t & at, + int64_t elem_size, int64_t elem_align, uint8_t elem_kind, int64_t elem_floor, + TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } // no array header: nothing rides + if ( r.get8() != elem_kind ) { return true; } // another element kind: §4's ordinary kind mismatch, the field reads empty + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + if ( n > (uint64_t) INT32_MAX ) { reason = count_over_extent_cap; return false; } + const int64_t rest = length - r.offset; + if ( n > (uint64_t) ( rest / elem_floor ) ) { reason = count_over_length; return false; } // an N the list's L cannot carry + at = ( at + elem_align - 1 ) & ~( elem_align - 1 ); + at += (int64_t) n * elem_size; + if ( inner == NULL ) { return true; } // nothing below an element: one depth is the whole term + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } // framing damage: the load reports it + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_LIST + +#ifndef MAPDEMO_SCHEMA_BUILD_VERSION +#define MAPDEMO_SCHEMA_BUILD_VERSION + +namespace mapdemo { + +// THE BUILD VERSION (docs/SPEC-TABLES.md §20): one digest over every fact the bytes +// this build produces depend on — the type wire's protocol id, every record's +// layout as the compiler's own C ABI model computes it, and the facts that +// decide what a load PUTS in those slots. It is the number a cook's header +// carries and the number Open compares, and the number a block's prologue +// carries and BlockOpen compares: a build version answers "which build?" and +// not "which form?", and what separates the two forms is their MAGIC. +// +// There are TWO ids in the design and they are not interchangeable: the +// PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is +// what everything cooked or blocked is keyed by. A table edit moves this and +// never the protocol id; a type edit moves both. +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_BUILD_VERSION + +#ifndef MAPDEMO_SCHEMA_TABLE_COOK +#define MAPDEMO_SCHEMA_TABLE_COOK + +namespace mapdemo { + +// ---- the cooked form (docs/SPEC-TABLES.md §7) ---- +// +// A cooked file is a HEADER, a DATA part and an ATTRIBUTION part, in that +// order. Every word of the header is a u64 written in the byte order the cook +// was produced in, and the header is 64 bytes: +// +// 0 magic 0x4b4f4f434d484353, read BYTEWISE before anything else +// 8 build_version the unit's id (docs/SPEC-TABLES.md §20) +// 16 byte_order 1 little, 2 big — the order that WROTE the file +// 24 data_length the region's bytes, rounded up to alignment +// 32 attribution_length the directory's bytes, or 0 +// 40 alignment the region's alignment, never below eight +// 48 reserved zero +// 56 reserved zero +// +// The DATA part is Lock's region written verbatim (§7.2) — the root at its +// base — and it is what a runtime points at. The ATTRIBUTION part is the node +// directory (§6.3), and NOTHING THAT READS THE STRUCTURE TOUCHES IT: it is +// written beside the data for schema cook-check, so a build that ships no +// tooling need not carry it at all. +static const int64_t kTableCookHeaderBytes = 64; + +// THE MAGIC'S VALUE, and a consumer written from the page needs the constant +// rather than a description of one. It is "SCHMCOOK" read as ASCII in the byte +// order a little-endian store produces — the same shape the block form's +// SCHMABLK takes, so a hex dump of a little-endian cook is legible and the two +// accelerators sit in one vocabulary. +// +// IT IS STORED IN THE PRODUCER'S ORDER, which is what makes it the byte-order +// check as well as the form check: a consumer reads back this build's +// constant, or that constant byte-reversed — which identifies a cook of the +// OTHER order — or something that is not a cook. All three answers but the +// first refuse, and a cook and a BLOCK are separated here too, because a +// form's identity belongs in its magic rather than in a second digest. +static const uint64_t TableCookMagic = 0x4b4f4f434d484353ull; + +// THIS BUILD's byte order, as the header's own word carries it. The magic is +// what REFUSES a foreign order; this word is what RECORDS which order wrote +// the file, so a refusal names the order rather than inferring it and a tool +// dumping a cook reads the fact. A file whose magic matched and whose order +// word did not is corrupt, and there is no reading that recovers it. +// +// The BUILD VERSION cannot do either job: §20.1 digests byteorder as a +// GENERATION input, little for every target schema generates for today, so +// two builds of one schema for two orders emit the same id. +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +static const uint64_t TableCookByteOrder = 2; // big +#else +static const uint64_t TableCookByteOrder = 1; // little +#endif + +// The greatest region alignment a cooked file may name. The DATA part begins +// at align_up( 64, alignment ), which is 64 for every unit this language can +// declare — the largest alignment it has is sixteen — so a word past this cap +// describes a file no build of this schema wrote (docs/SPEC-TABLES.md §7.1). +static const uint64_t TableCookMaxAlign = 64; + +// The header read, BYTEWISE. memcpy is the portable spelling of "these eight +// bytes, in this machine's order"; every compiler this repo builds under folds +// it to one load, and it is the only read in the whole of Open that is not a +// comparison. +inline uint64_t table_cook_read64( const uint8_t * p ) +{ + uint64_t v; + memcpy( &v, p, sizeof( v ) ); + return v; +} + +// TableCookOpen: THE WHOLE CHECK, in one place, because §7 states the +// enumeration once and every generated Open is that one enumeration plus +// its own root's two layout facts. +// +// THE CHECK, in order: the magic read bytewise, the byte order it establishes, +// the build version against this build's own, both RESERVED words zero, the +// region alignment the header names, the two part lengths against the length +// the caller passed — a truncated file and a file with trailing bytes are the +// same refusal — the root's own storage inside the data part, and the +// alignment of the base. +// +// AND THAT IS ALL OF IT. On a match the bytes ARE what this build wrote, in +// this build's layout and this build's byte order, so there is nothing to +// validate and nothing to fix up: the caller gets the root. Nothing per node +// happens here, which is what makes open O(1) in the file's size; a walk of +// any shape would forfeit that, and validating an untrusted file is schema +// cook-check's job and a person's decision (§7.4). +// +// EVERY NUMBER BELOW COMES OUT OF THE FILE, so the arithmetic is unsigned and +// each term is BOUNDED BEFORE IT IS ADDED: a forged length near 2^64 must +// refuse, and an addition that wrapped would be the defect the comparison +// after it was supposed to catch. Nothing past length is read on any path, +// including every refusing one. +// A REFUSAL NAMES ITSELF, beside the null (docs/SPEC-TABLES.md §7): the reason +// is written on the refusal path only, so a match costs nothing and a caller +// that passed no out-parameter pays nothing. +inline const uint8_t * TableCookRefuse( TableRefuseReason * reason, TableRefuseReason why ) +{ + if ( reason != NULL ) { *reason = why; } + return NULL; +} + +inline uint64_t table_cook_byteswap64( uint64_t v ) +{ + return ( v >> 56 ) | ( ( v >> 40 ) & 0xff00ull ) | ( ( v >> 24 ) & 0xff0000ull ) | ( ( v >> 8 ) & 0xff000000ull ) + | ( ( v << 8 ) & 0xff00000000ull ) | ( ( v << 24 ) & 0xff0000000000ull ) | ( ( v << 40 ) & 0xff000000000000ull ) + | ( v << 56 ); +} + +inline const uint8_t * TableCookOpen( const void * bytes, uint64_t length, uint64_t root_size, uint64_t root_align, TableRefuseReason * reason ) +{ + // a null buffer is the CALLER's defect, as an unaligned base is; a buffer + // shorter than the header has no header to read and is truncated + if ( bytes == NULL ) { return TableCookRefuse( reason, unaligned_base ); } + if ( length < (uint64_t) kTableCookHeaderBytes ) { return TableCookRefuse( reason, truncated ); } + const uint8_t * raw = (const uint8_t *) bytes; + // the MAGIC, bytewise and first: it is what establishes the byte order + // every other header word is read in, so nothing else may be read before + // it. A byte-reversed constant is a cook of the other order and refuses + // here, which is why the order never reaches a fix-up pass; anything else + // is not a cook at all, a BLOCK's magic included. + const uint64_t magic = table_cook_read64( raw ); + if ( magic != TableCookMagic ) + { + return TableCookRefuse( reason, magic == table_cook_byteswap64( TableCookMagic ) ? foreign_order : not_a_cook ); + } + // a byte-order word that contradicts its own magic describes no cook in + // EITHER order, so it shares the magic's own value (§7.1) + if ( table_cook_read64( raw + 16 ) != TableCookByteOrder ) { return TableCookRefuse( reason, not_a_cook ); } + if ( table_cook_read64( raw + 8 ) != BuildVersion ) { return TableCookRefuse( reason, wrong_build_version ); } + // the RESERVED words: a non-zero one means a writer used a form this build + // does not understand, and Open refuses rather than ignoring it. + if ( table_cook_read64( raw + 48 ) != 0 ) { return TableCookRefuse( reason, reserved_not_zero ); } + if ( table_cook_read64( raw + 56 ) != 0 ) { return TableCookRefuse( reason, reserved_not_zero ); } + const uint64_t data_length = table_cook_read64( raw + 24 ); + const uint64_t attribution_length = table_cook_read64( raw + 32 ); + const uint64_t alignment = table_cook_read64( raw + 40 ); + // THE ALIGNMENT WORD IS DATA, and it is the one header field the rest of + // the check does arithmetic WITH rather than only comparison against. A + // region's alignment is a power of two, never below eight (the floor that + // puts the attribution part on an eight-byte boundary without a second + // padding rule) and never past the cap above; a word that is none of those + // rounds nothing and aligns nothing, so it is refused before it is used, + // which is why bad_alignment precedes both truncated clauses (§7). + if ( alignment < 8 || alignment > TableCookMaxAlign ) { return TableCookRefuse( reason, bad_alignment ); } + if ( ( alignment & ( alignment - 1 ) ) != 0 ) { return TableCookRefuse( reason, bad_alignment ); } + // and it must be an alignment THE ROOT CAN SIT AT, since the root is at + // the region's base: both are powers of two, so "at least the root's" + // is one division. + if ( ( alignment % root_align ) != 0 ) { return TableCookRefuse( reason, bad_alignment ); } + // The DATA part begins at align_up( 64, alignment ). It is DERIVED and not + // a header field, because a fact a reader computes is a fact two writers + // cannot disagree about. + const uint64_t data_offset = ( (uint64_t) kTableCookHeaderBytes + alignment - 1 ) & ~( alignment - 1 ); + if ( length < data_offset ) { return TableCookRefuse( reason, truncated ); } + // the two part lengths against the length the caller passed. The whole + // file is data_offset + data_length + attribution_length, and a length + // that is not EXACTLY that refuses — truncation and trailing bytes are one + // refusal, and both terms are subtracted rather than added so no sum can + // carry. + if ( data_length > length - data_offset ) { return TableCookRefuse( reason, truncated ); } + if ( attribution_length != length - data_offset - data_length ) { return TableCookRefuse( reason, truncated ); } + // the ROOT sits at the region's base, so the region has to hold it: a + // shorter data part describes a root partly outside the file, which is the + // one way a match-and-point reader could hand back storage it never + // received. It is the second clause on truncated (§7). + if ( data_length < root_size ) { return TableCookRefuse( reason, truncated ); } + const uint8_t * base = raw + data_offset; + // the alignment of the BASE, LAST, because it is the only clause that reads + // nothing out of the file. The header pads the data part to the region's + // alignment, so a base an allocator or mmap gave you is already aligned — + // mmap gives page alignment for free — and a base that is not is a caller's + // buffer this form cannot be read out of: the caller's defect, not the file's. + if ( ( (uintptr_t) base % (uintptr_t) alignment ) != 0 ) { return TableCookRefuse( reason, unaligned_base ); } + return base; +} + +// ---- the cooked form, the WRITE side (docs/SPEC-TABLES.md §7.6) ---- +// +// THE BYTE ORDER IS THE TARGET'S, NOT THE HOST'S. A cook is produced in the +// byte order of the build that will read it (§7), so the fixing happens here — +// offline, once, on the writing side — and never at Open. Passing +// TableByteOrder::Big on a little-endian machine produces a big-endian build's +// file, and nothing about the writing host reaches the bytes. +enum class TableByteOrder +{ + Little = 1, // the header's byte_order word, and the order every scalar is written in + Big = 2, +}; + +// One store, width as an argument. Every call site passes a literal width, so +// the loop folds to a store (and a byte swap on the foreign order); a name per +// width would claim four §11 names to save nothing. +inline void table_cook_put( uint8_t * at, uint64_t value, int32_t width, TableByteOrder order ) +{ + if ( order == TableByteOrder::Little ) + { + for ( int32_t i = 0; i < width; i++ ) { at[i] = (uint8_t) ( value >> ( 8 * i ) ); } + } + else + { + for ( int32_t i = 0; i < width; i++ ) { at[i] = (uint8_t) ( value >> ( 8 * ( width - 1 - i ) ) ); } + } +} + +// A 128-bit store as two lanes: sixteen bytes, the low lane first in the +// little order and the high lane first — each lane big-endian — in the big +// order, exactly as a u64 is one lane of eight (docs/SPEC-TABLES.md §7.2). +inline void table_cook_put128( uint8_t * at, uint64_t lo, uint64_t hi, TableByteOrder order ) +{ + if ( order == TableByteOrder::Little ) { table_cook_put( at, lo, 8, order ); table_cook_put( at + 8, hi, 8, order ); } + else { table_cook_put( at, hi, 8, order ); table_cook_put( at + 8, lo, 8, order ); } +} + +// A buffer piece: the USED bytes and nothing else. The tail is already zero — +// the whole extent was zeroed before any field was written — so this copies the +// used prefix and leaves the rest, which is what makes a string's unused tail a +// consequence of one memset rather than a rule per buffer. A used length past +// the buffer, or below zero, is a value no reader could have produced and it is +// clamped rather than trusted: this writes inside the caller's buffer on every +// input. +inline void table_cook_bytes( uint8_t * at, const void * source, int64_t used, int64_t capacity ) +{ + if ( used <= 0 ) { return; } + const int64_t n = used < capacity ? used : capacity; + memcpy( at, source, (size_t) n ); +} + +// A WIDE TEXT buffer piece (docs/SPEC-TABLES.md §7.2): the USED code units, +// each a TWO-BYTE SCALAR in the cook's byte order. A record is written piece +// by piece and never memcpy'd, and a swap has to know where every scalar +// begins — a char16_t is one, so the units go one store each rather than as +// bytes. The tail is already zero, as the narrow twin's is, so the terminating +// zero unit at index used costs nothing here. +inline void table_cook_units( uint8_t * at, const char16_t * source, int64_t used, int64_t capacity, TableByteOrder order ) +{ + if ( used <= 0 ) { return; } + const int64_t n = used < capacity ? used : capacity; + for ( int64_t i = 0; i < n; i++ ) { table_cook_put( at + i * 2, (uint64_t) (uint16_t) source[i], 2, order ); } +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_COOK + +#ifndef MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE +#define MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE + +namespace mapdemo { + +// ---- the cooked form's WRITE side for a POINTERED root (docs/SPEC-TABLES.md §7.6) ---- +// +// A pointered root's cook is the region of §7.2: every node the numbering +// reached (§3.1), once, at its own type's alignment, in index order, the root +// at offset zero. This is that region while it is being laid out and written — +// the tool's own Layout and Write, in one struct. +// +// The OFFSETS are one per node, the root's zero at position 0 and node index k +// at position k - 1, which is the directory's own order (§6.3); they are the +// one allocation the write makes beyond the numbering, and they go through the +// same pair. A measure needs no offsets and leaves the pointer NULL. +struct TableCookRegion +{ + const TableNumbering * numbering = NULL; // node -> index, from the walk that placed it + int64_t * offsets = NULL; // index - 1 -> the node's region offset; NULL while measuring + int64_t count = 0; // nodes, the root included + int64_t bytes = 0; // the data part's length, rounded to align + int64_t align = 0; // the region's alignment: the nodes' greatest, never below eight + uint8_t * base = NULL; // where the data part is being written; NULL while measuring +}; + +// A reference slot: the SELF-RELATIVE delta from the slot's own address to the +// node's start (§6.3), and zero for null. The node is found by the address the +// numbering keyed it under, which is the same address the walk resolved through +// the same context — so a reference the numbering does not carry is a slot the +// walk never reached (a counted array's slot past its count, an absent +// optional's value) holding a node the region will not hold, and it is refused +// rather than written as a delta to nowhere. +inline bool table_cook_ref( const TableCookRegion & region, uint8_t * at, const void * pointee, TableByteOrder order ) +{ + if ( pointee == NULL ) { table_cook_put( at, 0, 8, order ); return true; } + uint64_t index = 0; + if ( !TableNumberingIndex( *region.numbering, pointee, index ) ) { return false; } + if ( index == 0 || index > (uint64_t) region.count ) { return false; } + const int64_t delta = region.offsets[index - 1] - (int64_t) ( at - region.base ); + table_cook_put( at, (uint64_t) delta, 8, order ); + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE + +namespace mapdemo { + +// table PairsSlotsEntry — TABLE-wire storage: relocatable, bounded, defaults in the +// member initializers (docs/SPEC-TABLES.md) +struct PairsSlotsEntry { + uint32_t key = 0; + TableRef value[2]; // [2]*Item — every slot null until assigned +}; + +// table Pairs — TABLE-wire storage: relocatable, bounded, defaults in the +// member initializers (docs/SPEC-TABLES.md) +struct Pairs { + TableMap slots; // map[uint32]*Item — the sorted entry array, empty until an insert + int32_t after = 0; +}; + +// ---- prefill: the declared defaults, in place (docs/SPEC-TABLES.md) ---- + +inline void PairsSlotsEntryReset( PairsSlotsEntry & value ); +inline void PairsReset( Pairs & value ); + +inline void PairsSlotsEntryReset( PairsSlotsEntry & value ) +{ + value.key = 0; + for ( int32_t i = 0; i < 2; i++ ) { value.value[i].value = 0; } // [2]*Item — every slot null +} + +inline void PairsReset( Pairs & value ) +{ + value.slots.entries.value = 0; // map[uint32]*Item: empty + value.slots.count = 0; + value.slots.padding = 0; + value.after = 0; +} + +template inline int64_t PairsSlotsEntryMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const PairsSlotsEntry & value ); +template inline bool PairsSlotsEntrySaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const PairsSlotsEntry & value ); +inline bool PairsSlotsEntryLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, PairsSlotsEntry & value ); +template inline int64_t PairsMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Pairs & value ); +template inline bool PairsSaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Pairs & value ); +inline bool PairsLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Pairs & value ); +inline bool PairsMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ); + +// PairsSlotsEntryMessageKeyRead: the key of one entry on the message wire, before the +// slot is chosen (docs/SPEC-TABLES.md §2.8, §3.3), and the bit the entry's +// body ends at. Field order inside a body is not contractual, so this scans +// the whole body by its announced shapes rather than assuming a position. +struct PairsSlotsEntryMessageKeyRead +{ + uint32_t key; + int64_t end; // the bit after the entry's own zero reference + bool found; // the body carried the key's id + bool kind_bad; // it carried it under another kind: the MAP's event + bool widened; // under a kind the declaration WIDENS (§4): decoded exactly, the MAP counts one + bool over; // longer than this reader's bound: the ENTRY is dropped + bool malformed; // the entry's framing gave out +}; + +inline PairsSlotsEntryMessageKeyRead PairsSlotsEntryMessageReadKey( TableBitReader r, const TableVocabulary & vocabulary, int64_t index_bits ) +{ + PairsSlotsEntryMessageKeyRead out = { 0, 0, false, false, false, false, false }; + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { out.malformed = true; return out; } + if ( ref == 0 ) { out.end = r.offset; return out; } // the terminator: no key field is the key's DEFAULT + if ( ref > (uint64_t) vocabulary.count ) { out.malformed = true; return out; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { out.malformed = true; return out; } + if ( entry.id == 0x3dc94a19365b10ecull ) // `key`, the ordinary hash of an ordinary name + { + const bool kind_bad = entry.kind != 8 && !TableKindWidens( entry.kind, 8 ); // THE KEY KIND IS THE READER'S DECLARATION + out.kind_bad = kind_bad; + out.found = !kind_bad; + out.widened = entry.kind != 8; + if ( kind_bad ) + { + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { out.malformed = true; return out; } + continue; + } + { + uint64_t raw = 0; + const int64_t width = entry.value_bits; + if ( width < 0 || !r.get( raw, width ) ) { out.malformed = true; return out; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + out.key = (uint32_t) decoded_wide; + } + continue; // the LAST occurrence is the one §3 keeps + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { out.malformed = true; return out; } + } +} + +// ---- the arena's reset hook (docs/SPEC-TABLES.md §6) ---- +// +// TableWorker::Alloc is a template and cannot name a member's Reset, so +// the arena reaches it through this overload set by argument-dependent +// lookup. It is how a node born in raw arena storage comes to hold the +// declared defaults without value-initialising the whole aggregate. + +inline void TableReset( PairsSlotsEntry & value ) { PairsSlotsEntryReset( value ); } +inline void TableReset( Pairs & value ) { PairsReset( value ); } + +// ---- pointer targets: allocation and resolution (docs/SPEC-TABLES.md §2) ---- +// +// A reference resolves differently in the two forms, and the CONTEXT says +// which: in the arena it is an offset; in a region it is a self-relative +// delta, so the const deref below is one add and needs no base pointer. + +// ---- codecs: measure/save/load per closure member ---- + +template inline int64_t PairsSlotsEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const PairsSlotsEntry & value ); +template inline bool PairsSlotsEntrySaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const PairsSlotsEntry & value ); +template inline bool PairsSlotsEntrySaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const PairsSlotsEntry & value ); +inline bool PairsSlotsEntryLoadBody( TableReader & r, const TableNodeMap & nodes, PairsSlotsEntry & value ); +template inline int64_t PairsMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Pairs & value ); +template inline bool PairsSaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Pairs & value ); +template inline bool PairsSaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Pairs & value ); +inline bool PairsLoadBody( TableReader & r, const TableNodeMap & nodes, Pairs & value ); + +// ---- pointer-graph walkers: number (measure/save), pack (Lock) ---- + +template inline bool PairsSlotsEntryNumber( const Ctx & ctx, TableNumbering & numbering, const PairsSlotsEntry & value ); +template inline int64_t PairsSlotsEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const PairsSlotsEntry & value ); +template inline bool PairsSlotsEntryPack( const Ctx & ctx, TablePackMap & seen, const PairsSlotsEntry & src, PairsSlotsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); +template inline bool PairsNumber( const Ctx & ctx, TableNumbering & numbering, const Pairs & value ); +template inline int64_t PairsPackMeasure( const Ctx & ctx, TablePackMap & seen, const Pairs & value ); +template inline bool PairsPack( const Ctx & ctx, TablePackMap & seen, const Pairs & src, Pairs & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +// ---- the numbering's bridge to each member's codec (docs/SPEC-TABLES.md §3.1) ---- + +template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const PairsSlotsEntry & value ) { return PairsSlotsEntryMeasureBody( ctx, numbering, ids, value ); } +template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const PairsSlotsEntry & value ) { return PairsSlotsEntrySaveBody( ctx, numbering, w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const PairsSlotsEntry & value ) { return PairsSlotsEntryMeasureMessageBody( ctx, numbering, index_bits, at, value ); } +template inline bool TableNodeMessageSave( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const PairsSlotsEntry & value ) { return PairsSlotsEntrySaveMessageBody( ctx, numbering, index_bits, w, value ); } +template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Pairs & value ) { return PairsMeasureBody( ctx, numbering, ids, value ); } +template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Pairs & value ) { return PairsSaveBody( ctx, numbering, w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Pairs & value ) { return PairsMeasureMessageBody( ctx, numbering, index_bits, at, value ); } +template inline bool TableNodeMessageSave( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Pairs & value ) { return PairsSaveMessageBody( ctx, numbering, index_bits, w, value ); } + +// ---- PairsSlotsEntry: the order, the key and the value (docs/SPEC-TABLES.md §2.8) ---- +// +// The four overloads the map runtime's templates reach by argument-dependent +// lookup. Nothing outside this file names them. +static_assert( alignof( PairsSlotsEntry ) <= kTableAlign, "a map entry's alignment must fit the arena's" ); + +inline int TableEntryOrder( const PairsSlotsEntry & a, const PairsSlotsEntry & b ) +{ + return TableKeyOrder( (uint64_t) a.key, (uint64_t) b.key ); // integers compare by VALUE, unsigned here +} +inline int TableEntryOrder( const PairsSlotsEntry & entry, uint32_t key ) +{ + return TableKeyOrder( (uint64_t) entry.key, (uint64_t) key ); +} +inline uint32_t TableEntryKey( const PairsSlotsEntry & entry ) { return entry.key; } +inline void TableEntrySetKey( PairsSlotsEntry & entry, uint32_t key ) { entry.key = key; } +// A FIXED ARRAY VALUE IS ONE MEMBER (§2.8, §4.2): the handle points at the +// ARRAY and not at its first element, so the extent survives the handoff. +typedef TableRef PairsSlotsEntryValue[2]; +inline const PairsSlotsEntryValue * TableEntryFound( const PairsSlotsEntry * entry ) { return entry != NULL ? &entry->value : NULL; } +inline PairsSlotsEntryValue * TableEntryValue( PairsSlotsEntry * entry ) { return &entry->value; } +struct PairsSlotsEntryEach { uint32_t key; decltype( TableEntryValue( (PairsSlotsEntry *) NULL ) ) value; }; +inline PairsSlotsEntryEach TableEntryEach( PairsSlotsEntry * entry ) { return PairsSlotsEntryEach{ TableEntryKey( *entry ), TableEntryValue( entry ) }; } +inline void TableResetMapValue( PairsSlotsEntry & value ) +{ + for ( int32_t i = 0; i < 2; i++ ) { value.value[i].value = 0; } // [2]*Item — every slot null +} + +// PairsSlotsEntryReadKey: the key, before the slot is chosen (docs/SPEC-TABLES.md §2.8). +// Field order inside a body is not contractual (§3), so this scans rather +// than assuming a position — and this implementation writes the key first, +// so on any wire it wrote the scan ends at the first header. +struct PairsSlotsEntryKeyRead +{ + uint32_t key; + bool found; // the body carried the key's id + bool kind_bad; // it carried it under another kind: the MAP's event + bool widened; // under a kind the declaration WIDENS (§4): decoded exactly, the MAP counts one + bool over; // longer than this reader's bound: the ENTRY is dropped + bool malformed; // the entry's framing gave out +}; + +inline PairsSlotsEntryKeyRead PairsSlotsEntryReadKey( const uint8_t * body, int64_t length, const TableIdTable * ids ) +{ + PairsSlotsEntryKeyRead out = { 0, false, false, false, false, false }; + TableReport scratch; // the scan's own framing damage is the MAP's, raised by the caller + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { out.malformed = true; return out; } + if ( field_ref == 0 ) { return out; } // the terminator: no key field is the key's DEFAULT + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { out.malformed = true; return out; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { out.malformed = true; return out; } + uint8_t field_kind = r.get8(); + if ( field_id == 0x3dc94a19365b10ecull ) // `key`, the ordinary hash of an ordinary name + { + if ( field_kind != 8 && TableKindWidens( field_kind, 8 ) ) + { + out.widened = true; + out.found = true; + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, field_kind, widened_v ) ) { out.malformed = true; return out; } + out.key = (uint32_t) widened_v; + continue; // the LAST occurrence is the one §3 keeps + } + out.kind_bad = field_kind != 8; // THE KEY KIND IS THE READER'S DECLARATION + out.found = !out.kind_bad; + if ( !out.kind_bad ) + { + if ( !r.has( 4 ) ) { out.malformed = true; return out; } + out.key = (uint32_t) r.get32(); + continue; // the LAST occurrence is the one §3 keeps + } + } + if ( !r.skip( field_kind ) ) { out.malformed = true; return out; } + } +} + +// ---- retain-unknown: the second family (docs/SPEC-TABLES.md §6.6) ---- +// +// The same walks, with the PATH threaded and the unknown arm capturing. The +// three above are untouched and cost nothing for these being here: a caller +// that does not ask instantiates none of them. + +template inline int64_t PairsSlotsEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool PairsSlotsEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool PairsSlotsEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool PairsSlotsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline int64_t PairsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool PairsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool PairsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ); +inline bool PairsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Pairs & value, TableRetain * retain, const TableRetainPath & path ); + +template +inline int64_t PairsSlotsEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const PairsSlotsEntry & value ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + { + bool any_value = false; + for ( int32_t i = 0; i < 2; i++ ) { if ( ItemAt( ctx, value.value[i] ) != NULL ) { any_value = true; break; } } + if ( any_value ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( 2 ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < 2; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return -1; } + body_value += TableLebBytes( slot_index ); + } + } + bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); // value: [2]*Item + } + } + return bytes; +} + +template +inline bool PairsSlotsEntrySaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const PairsSlotsEntry & value ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 8 ); // key + w.put32( uint32_t( value.key ) ); + } + { + bool any_value = false; + for ( int32_t i = 0; i < 2; i++ ) { if ( ItemAt( ctx, value.value[i] ) != NULL ) { any_value = true; break; } } + if ( any_value ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( 2 ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < 2; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + body_value += TableLebBytes( slot_index ); + } + } + w.putleb( ref_value ); w.put8( 14 ); w.putleb( (uint64_t) body_value ); // value + w.put8( 17 ); w.putleb( (uint64_t) ( 2 ) ); + for ( int32_t elem_i = 0; elem_i < 2; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + w.putleb( slot_index ); + } + } + } + } + return !w.overflow; +} + +template +inline bool PairsSlotsEntrySaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const PairsSlotsEntry & value ) +{ + if ( !PairsSlotsEntrySaveBodyFields( ctx, numbering, w, ids, value ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool PairsSlotsEntryLoadBody( TableReader & r, const TableNodeMap & nodes, PairsSlotsEntry & value ) +{ + PairsSlotsEntryReset( value ); // prefill declared defaults in place, then overlay + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x3dc94a19365b10ecull: // key + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.key = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.key = decoded_v; + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER — the element kind byte and the + // count, so fewer than two bytes — is INERT (§4): the field keeps the + // value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + // A DAMAGED COUNT stops the elements and nothing else: the field + // RODE, so an optional is still PRESENT (§2.3) — only a foreign + // ELEMENT KIND says the payload is not this array's at all. + if ( !counted_ok ) { r.report->malformed = true; } + else if ( elem_kind != 17 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + uint64_t keep = count; + if ( keep > 2 ) { keep = 2; r.report->clamped++; } + // elements are BOUNDED by the field body: a count the length + // cannot cover keeps the decoded prefix, flags malformed, and + // the parent continues at the next field — following fields' + // bytes are never fabricated into elements + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + for ( uint64_t i = 0; i < keep; i++ ) + { + { + uint64_t node_index = 0; + if ( !sub.getleb( node_index ) ) { r.report->malformed = true; break; } + TableNodeResolve( nodes, value.value[(int32_t) i], node_index, 0x52cfa1d198476806ull, r.report ); // *Item + } + } + } + } + r.offset = body_end; // excess elements and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's cost, in BITS (docs/SPEC-TABLES.md §3.3). `at` is the +// body's own bit position in the batch, because a `string(N)` ALIGNS before +// its bytes and an align costs what the position says it costs. +template +inline int64_t PairsSlotsEntryMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const PairsSlotsEntry & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + int64_t bits = 0; + if ( value.key != 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + } + { + bool rides_value = false; + for ( int32_t i = 0; i < 2; i++ ) { if ( ItemAt( ctx, value.value[i] ) != NULL ) { rides_value = true; break; } } + if ( rides_value ) + { + bits += kTableMessageRefBitsHere; + bits += (int64_t) ( 2 ) * index_bits; + } + } + bits += kTableMessageRefBitsHere; // the ZERO REFERENCE that ends the body + (void) at; + return bits; +} + +// The BITPACKED body: the fields, then the ZERO REFERENCE that ends it. No +// kind byte rides at all, and no length frames a nested body, because a +// body is self-delimiting: it is written where the file form put an L. +template +inline bool PairsSlotsEntrySaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const PairsSlotsEntry & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + if ( value.key != 0 ) + { + w.put( 9, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.key ), 32 ); + } + { + bool rides_value = false; + for ( int32_t i = 0; i < 2; i++ ) { if ( ItemAt( ctx, value.value[i] ) != NULL ) { rides_value = true; break; } } + if ( rides_value ) + { + w.put( 10, kTableMessageRefBitsHere ); + for ( int32_t i = 0; i < 2; i++ ) + { + const Item * pointee_value = ItemAt( ctx, value.value[i] ); // *Item + uint64_t index_value = 0; + if ( pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) pointee_value, index_value ) ) { return false; } + w.put( index_value, index_bits ); + } + } + } + w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +inline bool PairsSlotsEntryLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, PairsSlotsEntry & value ) +{ + (void) nodes; (void) index_bits; + PairsSlotsEntryReset( value ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = (uint64_t) entry.min; + const int64_t count_bits = TableBitsRequired( entry.min, entry.max ); + if ( count_bits > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, count_bits ) ) { report->malformed = true; return false; } + n = raw + (uint64_t) entry.min; + } + if ( entry.elem_kind == 6 && !r.align() ) { report->malformed = true; return false; } + int32_t kept = 0; + if ( n > (uint64_t) 2 ) { kept = 2; report->clamped++; } else { kept = (int32_t) n; } + const uint64_t walk = n; + for ( uint64_t i = 0; i < walk; i++ ) + { + const bool in_bounds = (int32_t) i < kept; + TableRef scratch; + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( in_bounds ? value.value[i] : scratch ), node_index_2, 0x52cfa1d198476806ull, report ); // *Item + } + } + } + break; + } + default: + report->unknown++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + } +} + +template +inline int64_t PairsMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Pairs & value ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // slots: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); + if ( !order_slots.ok ) { return -1; } // the sort could not run + if ( order_slots.count > 0 ) + { + const uint64_t ref_slots = ids.ref( 0xe68c2e6bb1ee5646ull ); + int64_t body_slots = 1 + TableLebBytes( (uint64_t) order_slots.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_slots = PairsSlotsEntryMeasureBody( ctx, numbering, ids, *order_slots[i] ); + if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return -1; } + body_slots += TableLebBytes( (uint64_t) ( elem_slots ) ) + ( elem_slots ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_slots ) + 1 + TableLebBytes( (uint64_t) ( body_slots ) ) + ( body_slots ); + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + return bytes; +} + +template +inline bool PairsSaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Pairs & value ) +{ + { + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); // slots + if ( !order_slots.ok ) { return false; } + if ( order_slots.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_slots = ids.ref( 0xe68c2e6bb1ee5646ull ); + int64_t body_slots = 1 + TableLebBytes( (uint64_t) order_slots.count ); + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_slots = PairsSlotsEntryMeasureBody( ctx, numbering, ids, *order_slots[i] ); + if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return false; } + body_slots += TableLebBytes( (uint64_t) ( elem_slots ) ) + ( elem_slots ); + } + w.putleb( ref_slots ); w.put8( 14 ); w.putleb( (uint64_t) body_slots ); + w.put8( 13 ); w.putleb( (uint64_t) order_slots.count ); + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_len_slots = PairsSlotsEntryMeasureBody( ctx, numbering, ids, *order_slots[i] ); + if ( elem_len_slots < 0 ) { TableMapRelease( order_slots ); return false; } + w.putleb( (uint64_t) elem_len_slots ); + if ( !PairsSlotsEntrySaveBody( ctx, numbering, w, ids, *order_slots[i] ) ) { TableMapRelease( order_slots ); return false; } + } + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + return !w.overflow; +} + +template +inline bool PairsSaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Pairs & value ) +{ + if ( !PairsSaveBodyFields( ctx, numbering, w, ids, value ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool PairsLoadBody( TableReader & r, const TableNodeMap & nodes, Pairs & value ) +{ + PairsReset( value ); // prefill declared defaults in place, then overlay + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0xe68c2e6bb1ee5646ull: // slots + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + TableMapFill fill = TableMapFillBegin( nodes, value.slots, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + PairsSlotsEntryKeyRead read = PairsSlotsEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + PairsSlotsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + PairsSlotsEntryLoadBody( elem, nodes, *slot ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's cost, in BITS (docs/SPEC-TABLES.md §3.3). `at` is the +// body's own bit position in the batch, because a `string(N)` ALIGNS before +// its bytes and an align costs what the position says it costs. +template +inline int64_t PairsMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Pairs & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + int64_t bits = 0; + { + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); // slots + if ( !order_slots.ok ) { return -1; } // the sort could not run + if ( order_slots.count > 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; // the count the data decides + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_slots = PairsSlotsEntryMeasureMessageBody( ctx, numbering, index_bits, at + bits, *order_slots[i] ); + if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return -1; } + bits += elem_slots; // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + } + bits += kTableMessageRefBitsHere; // the ZERO REFERENCE that ends the body + (void) at; + return bits; +} + +// The BITPACKED body: the fields, then the ZERO REFERENCE that ends it. No +// kind byte rides at all, and no length frames a nested body, because a +// body is self-delimiting: it is written where the file form put an L. +template +inline bool PairsSaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Pairs & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + { + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); // slots + if ( !order_slots.ok ) { return false; } // the sort could not run + if ( order_slots.count > 0 ) + { + w.put( 37, kTableMessageRefBitsHere ); + w.put( (uint64_t) order_slots.count, 32 ); // the count the data decides + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + if ( !PairsSlotsEntrySaveMessageBody( ctx, numbering, index_bits, w, *order_slots[i] ) ) { TableMapRelease( order_slots ); return false; } + } + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) + { + w.put( 18, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.after ), 32 ); + } + w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +// PairsMessageExtent: the extent Pairs's maps command on the message wire, from +// the FRAMING alone (docs/SPEC-TABLES.md §2.8, §3.3, §6.5). +inline bool PairsMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { return false; } + if ( entry.id == 0xe68c2e6bb1ee5646ull && entry.kind == 14 && entry.elem_kind == 13 ) // slots + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { return false; } + n += (uint64_t) entry.min; + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( PairsSlotsEntry ) + at += (int64_t) n * (int64_t) sizeof( PairsSlotsEntry ); // the whole array FIRST + for ( uint64_t i = 0; i < n; i++ ) // then, entry by entry in key order + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { return false; } + } + continue; + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { return false; } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +inline bool PairsLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Pairs & value ) +{ + (void) nodes; (void) index_bits; + PairsReset( value ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0xe68c2e6bb1ee5646ull: // slots + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + TableMapFill fill = TableMapFillBegin( nodes, value.slots, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const PairsSlotsEntryMessageKeyRead read = PairsSlotsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + PairsSlotsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !PairsSlotsEntryLoadMessageBody( r, vocabulary, report, nodes, index_bits, *slot ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + } +} + +// PairsSlotsEntryWireExtent: the extent PairsSlotsEntry's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool PairsSlotsEntryWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + (void) body; (void) length; (void) at; (void) ids; (void) reason; // no list or map below this record + return true; +} + +// PairsSlotsEntryExtentAt: the node extent PairsSlotsEntry's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as PairsSlotsEntryExtentPack advances it (§2.8, §2.9). +template +inline bool PairsSlotsEntryExtentAt( const Ctx & ctx, const PairsSlotsEntry & value, int64_t & at ) +{ + (void) ctx; (void) value; (void) at; // no list or map below this record + return true; +} + +template +inline int64_t PairsSlotsEntryExtent( const Ctx & ctx, const PairsSlotsEntry & value ) +{ + (void) ctx; (void) value; // no list or map below this record + return 0; +} + +// PairsSlotsEntryExtentPack: carve PairsSlotsEntry's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset PairsSlotsEntryExtentAt advances (§2.8, §2.9). +template +inline bool PairsSlotsEntryExtentPack( const Ctx & ctx, const PairsSlotsEntry & src, PairsSlotsEntry & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + (void) ctx; (void) src; (void) dst; (void) extent; (void) at; (void) capacity; // no list or map below this record + return true; +} + +// PairsWireExtent: the extent Pairs's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool PairsWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; // the scan's framing damage is the LOAD's to report + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { return true; } + if ( field_ref == 0 ) { return true; } + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { return true; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { return true; } + uint8_t field_kind = r.get8(); + if ( field_id == 0xe68c2e6bb1ee5646ull && field_kind == 14 ) // slots + { + uint64_t map_len = 0; + if ( !r.getleb( map_len ) || !r.room( map_len ) ) { return true; } + const uint8_t * map_body = r.buffer + r.offset; + r.offset += (int64_t) map_len; + if ( !TableMapWireExtent( map_body, (int64_t) map_len, at, (int64_t) sizeof( PairsSlotsEntry ), (int64_t) alignof( PairsSlotsEntry ), NULL, ids, reason ) ) { return false; } + continue; + } + if ( !r.skip( field_kind ) ) { return true; } + } +} + +// PairsExtentAt: the node extent Pairs's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as PairsExtentPack advances it (§2.8, §2.9). +template +inline bool PairsExtentAt( const Ctx & ctx, const Pairs & value, int64_t & at ) +{ + { + TableMapCursor cursor = TableMapOrder( ctx, value.slots ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( PairsSlotsEntry ) + at += (int64_t) cursor.count * (int64_t) sizeof( PairsSlotsEntry ); // the whole array FIRST + for ( int32_t i = 0; i < cursor.count; i++ ) // then, entry by entry in key order + { + if ( !PairsSlotsEntryExtentAt( ctx, *cursor[i], at ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// the whole extent of one node, from a fresh offset: what a pack reserves +// for it beside the record's own storage. +template +inline int64_t PairsExtent( const Ctx & ctx, const Pairs & value ) +{ + int64_t at = 0; + if ( !PairsExtentAt( ctx, value, at ) ) { return -1; } + return at; +} + +// PairsExtentPack: carve Pairs's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset PairsExtentAt advances (§2.8, §2.9). +template +inline bool PairsExtentPack( const Ctx & ctx, const Pairs & src, Pairs & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + { + TableMapCursor cursor = TableMapOrder( ctx, src.slots ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; + const int64_t bytes = (int64_t) cursor.count * (int64_t) sizeof( PairsSlotsEntry ); + if ( at + bytes > capacity ) { TableMapRelease( cursor ); return false; } + PairsSlotsEntry * placed = (PairsSlotsEntry *) ( extent + at ); + at += bytes; + dst.slots.count = cursor.count; + dst.slots.padding = 0; + dst.slots.entries.value = cursor.count > 0 ? (int64_t) ( (uint8_t *) placed - (const uint8_t *) &dst.slots.entries ) : 0; + for ( int32_t i = 0; i < cursor.count; i++ ) + { + memcpy( (void *) ( placed + i ), (const void *) cursor[i], sizeof( PairsSlotsEntry ) ); // trivially copyable, by construction + } + for ( int32_t i = 0; i < cursor.count; i++ ) + { + if ( !PairsSlotsEntryExtentPack( ctx, *cursor[i], placed[i], extent, at, capacity ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// ---- Pairs.slots: the builder's five and the side index (§2.8) ---- + +// INSERT: the key is copied, the value is handed back at its defaults to +// fill. A DUPLICATE key REPLACES — the value is reset and the same entry +// handed back, key and address unchanged — so a caller that wants to know +// writes Find first. NULL is NOT INSERTED: a key longer than the bound, +// because a truncated key would be a merged entry, and an arena that +// cannot carve another segment, alike. +// +// It is a WRAPPER: TableMapPlace owns the lookup, the reset, the +// allocation and the key copy, and this half is the bound and the +// const char * key's length. Nothing here mutates an entry (§2.8). +inline PairsSlotsEntryValue * PairsSlotsInsert( TableWorker & worker, TableMap & map, uint32_t key ) +{ + PairsSlotsEntry * entry = TableMapPlace( worker, map, key ); + return entry != NULL ? TableEntryValue( entry ) : NULL; +} + +// FIND on the builder: the same linear scan, O( n ) key compares over the +// segments in insertion order. NULL when absent. The builder builds NO +// INDEX, and that is a rule — the sort happens once, at Lock, Save or +// Cook, and every lookup that matters runs over the sorted region. +inline PairsSlotsEntryValue * PairsSlotsFind( TableArena & arena, TableMap & map, uint32_t key ) +{ + PairsSlotsEntry * found = TableMapScan( arena, map, key ); + return found != NULL ? TableEntryValue( found ) : NULL; +} + +// ERASE: marks the entry DEAD, one bit in the segment's slot and not in the +// entry table. False when absent. Its storage is held until the builder +// resets and never reused mid-build, because reusing a slot would make "an +// entry's address is stable" false for exactly one case. +inline bool PairsSlotsErase( TableArena & arena, TableMap & map, uint32_t key ) +{ + return TableMapErase( arena, map, key ); +} + +// EACH on the builder: INSERTION order, live entries only. +inline TableMapEach PairsSlotsEach( const TableArena & arena, const TableMap & map ) +{ + return TableMapEachOf( arena, map ); +} + +// ---- the OPTIONAL INDEX: caller-owned, built at load, never stored ---- +// +// Open addressing with linear probing over the sorted array, for a map large +// enough that log n compares over a cold array cost more than one hash and a +// probe. ITS HASH AND ITS LOAD FACTOR ARE NOT A CROSS-PORT CONTRACT: the +// index is never stored, so no golden, no cook-check rule and no +// build-version line ever names either. What a port is held to is the +// CONTRACT of the lookup — the same value the sorted array's Find returns +// for the same key, and no allocation past the storage the caller handed in. +inline int64_t PairsSlotsIndexMeasure( const TableMap & map ) +{ + return (int64_t) TableMapIndexSlots( map.count ) * (int64_t) sizeof( int32_t ); +} + +inline TableMapIndex PairsSlotsIndex( const TableMap & map, void * storage, int64_t bytes ) +{ + TableMapIndex index; + const int32_t slots = TableMapIndexSlots( map.count ); + if ( storage == NULL || bytes < (int64_t) slots * (int64_t) sizeof( int32_t ) ) { return index; } + index.slots = (int32_t *) storage; + index.capacity = slots; + for ( int32_t i = 0; i < slots; i++ ) { index.slots[i] = 0; } + const PairsSlotsEntry * entries = map.Entries(); + for ( int32_t i = 0; i < map.count; i++ ) // ONE PASS over the sorted array + { + int32_t at = (int32_t) ( TableMapHash( (uint64_t) entries[i].key ) & (uint64_t) ( slots - 1 ) ); + while ( index.slots[at] != 0 ) { at = ( at + 1 ) & ( slots - 1 ); } + index.slots[at] = i + 1; // slots are ENTRY INDICES; 0 is an empty slot + } + index.good = true; + return index; +} + +inline const PairsSlotsEntryValue * PairsSlotsIndexFind( const TableMapIndex & index, const TableMap & map, uint32_t key ) +{ + if ( !index.good ) { return map.Find( key ); } // an index that did not build is not a wrong answer + const PairsSlotsEntry * entries = map.Entries(); + int32_t at = (int32_t) ( TableMapHash( (uint64_t) key ) & (uint64_t) ( index.capacity - 1 ) ); + for ( int32_t probe = 0; probe < index.capacity; probe++ ) + { + const int32_t slot = index.slots[at]; + if ( slot == 0 ) { return NULL; } + if ( TableEntryOrder( entries[slot - 1], key ) == 0 ) { return TableEntryFound( entries + slot - 1 ); } + at = ( at + 1 ) & ( index.capacity - 1 ); + } + return NULL; +} + +// PairsSlotsEntryNumber: number everything PairsSlotsEntry POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool PairsSlotsEntryNumber( const Ctx & ctx, TableNumbering & numbering, const PairsSlotsEntry & value ) +{ + for ( int32_t k = 0; k < 2; k++ ) // value: [2]*Item + { + { + const Item * pointee = ItemAt( ctx, value.value[k] ); // value + if ( pointee != NULL ) + { + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( numbering.seen, (const void *) pointee, + (int64_t) ( numbering.count + 2 ), taken, slot ); // its index, if this is its first visit + if ( entry == NULL ) { return false; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return false; } // a data cycle + } + else + { + TableNodeEntry node; + node.node = (const void *) pointee; + node.type_id = 0x52cfa1d198476806ull; // fnv1a64( "Item" ) + node.type_slot = 65; // its slot in the unit's vocabulary (§3.3) + node.measure = &TableNodeMeasureThunk; + node.save = &TableNodeSaveThunk; + node.message_measure = &TableNodeMessageMeasureThunk; + node.message_save = &TableNodeMessageSaveThunk; + if ( !TableNumberingAppend( numbering, node ) ) { return false; } + if ( !ItemNumber( ctx, numbering, *pointee ) ) { return false; } + TablePackMapClose( numbering.seen, (const void *) pointee, slot ); + } + } + } + } + return true; +} + +// PairsSlotsEntryPackMeasure: the packed region bytes of everything PairsSlotsEntry POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t PairsSlotsEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const PairsSlotsEntry & value ) +{ + int64_t bytes = 0; + for ( int32_t k = 0; k < 2; k++ ) // value: [2]*Item + { + { + const Item * pointee = ItemAt( ctx, value.value[k] ); // value + if ( pointee != NULL ) + { + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( seen, (const void *) pointee, 0, taken, slot ); + if ( entry == NULL ) { return -1; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return -1; } // a data cycle + } + else + { + int64_t inner = ItemPackMeasure( ctx, seen, *pointee ); + if ( inner < 0 ) { return -1; } + TablePackMapClose( seen, (const void *) pointee, slot ); + bytes += TableAlignUp64( (int64_t) sizeof( Item ) ) + inner; + } + } + } + } + return bytes; +} + +// PairsSlotsEntryPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool PairsSlotsEntryPackEdges( const Ctx & ctx, TablePackMap & seen, const PairsSlotsEntry & src, PairsSlotsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool PairsSlotsEntryPack( const Ctx & ctx, TablePackMap & seen, const PairsSlotsEntry & src, PairsSlotsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( PairsSlotsEntry ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( PairsSlotsEntry ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !PairsSlotsEntryExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return PairsSlotsEntryPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool PairsSlotsEntryPackEdges( const Ctx & ctx, TablePackMap & seen, const PairsSlotsEntry & src, PairsSlotsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + for ( int32_t k = 0; k < 2; k++ ) // value: [2]*Item + { + { + dst.value[k].value = 0; // value + const Item * pointee = ItemAt( ctx, src.value[k] ); + if ( pointee != NULL ) + { + int64_t at = TableAlignUp64( used ); // where it WOULD land, if this is its first visit + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( seen, (const void *) pointee, at, taken, slot ); + if ( entry == NULL ) { return false; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return false; } // a data cycle + dst.value[k].value = (int64_t) ( ( base + entry->offset ) - (const uint8_t *) &dst.value[k] ); // the one body it already has + } + else + { + if ( at + (int64_t) sizeof( Item ) > capacity ) { return false; } + used = at + TableAlignUp64( (int64_t) sizeof( Item ) ); + Item * child = new ( base + at ) Item; // lifetime only: the Pack below memcpy's the whole node over it + dst.value[k].value = (int64_t) ( ( base + at ) - (const uint8_t *) &dst.value[k] ); + if ( !ItemPack( ctx, seen, *pointee, *child, base, capacity, used ) ) { return false; } + TablePackMapClose( seen, (const void *) pointee, slot ); + } + } + } + } + return true; +} + +// PairsNumber: number everything Pairs POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool PairsNumber( const Ctx & ctx, TableNumbering & numbering, const Pairs & value ) +{ + { // slots: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_slots = TableMapOrder( ctx, value.slots ); + if ( !cursor_slots.ok ) { return false; } + for ( int32_t i = 0; i < cursor_slots.count; i++ ) + { + if ( !PairsSlotsEntryNumber( ctx, numbering, *cursor_slots[i] ) ) { TableMapRelease( cursor_slots ); return false; } + } + TableMapRelease( cursor_slots ); + } + return true; +} + +// PairsPackMeasure: the packed region bytes of everything Pairs POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t PairsPackMeasure( const Ctx & ctx, TablePackMap & seen, const Pairs & value ) +{ + int64_t bytes = 0; + { // slots: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_slots = TableMapOrder( ctx, value.slots ); + if ( !cursor_slots.ok ) { return -1; } + for ( int32_t i = 0; i < cursor_slots.count; i++ ) + { + int64_t inner = PairsSlotsEntryPackMeasure( ctx, seen, *cursor_slots[i] ); + if ( inner < 0 ) { TableMapRelease( cursor_slots ); return -1; } + bytes += inner; + } + TableMapRelease( cursor_slots ); + } + return bytes; +} + +// PairsPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool PairsPackEdges( const Ctx & ctx, TablePackMap & seen, const Pairs & src, Pairs & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool PairsPack( const Ctx & ctx, TablePackMap & seen, const Pairs & src, Pairs & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( Pairs ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( Pairs ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !PairsExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return PairsPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool PairsPackEdges( const Ctx & ctx, TablePackMap & seen, const Pairs & src, Pairs & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + { // slots: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_slots = TableMapOrder( ctx, src.slots ); + if ( !cursor_slots.ok ) { return false; } + PairsSlotsEntry * placed_slots = (PairsSlotsEntry *) ( dst.slots.entries.value != 0 ? ( (uint8_t *) &dst.slots.entries + dst.slots.entries.value ) : NULL ); + for ( int32_t i = 0; i < cursor_slots.count; i++ ) + { + if ( !PairsSlotsEntryPackEdges( ctx, seen, *cursor_slots[i], placed_slots[i], base, capacity, used ) ) { TableMapRelease( cursor_slots ); return false; } + } + TableMapRelease( cursor_slots ); + } + return true; +} + +// ---- Pairs: the variable-length life (docs/SPEC-TABLES.md §2, §6, §9) ---- +// +// MUTABLE: PairsBuilder — allocate nodes, wire them together, then Lock. +// CONST: one packed region, root at its base. Lock produces it and Load +// produces it, so a locked structure and a loaded one are the +// SAME representation with one view API. There is no unlock: +// re-editing means loading the const form into a fresh builder. +// Pairs is never held by value — a file-format-scale structure is a region +// and a root pointer, not a struct you copy. + +struct PairsBuilder +{ + TableArena arena; + TableWorker main; // the calling thread's allocation front + TableRef root_ref; + uint8_t * region = NULL; // the packed const form, produced by Lock() + int64_t region_bytes = 0; + + // THE ALLOCATOR IS THE BUILDER'S, and everything this structure ever + // allocates goes through it: the arena's segments, Lock's identity map, + // the packed region, the wire walks' numbering, and the tool path's node + // directory. Name your own and a profiler sees every byte under it. + PairsBuilder( TableAllocator allocator = TableDefaultAllocator() ) + { + TableArenaInit( arena, allocator ); + main.arena = &arena; + TableSlot slot = main.Alloc(); + root_ref = slot.ref; + } + ~PairsBuilder() { TableArenaShutdown( arena ); arena.allocator.free( arena.allocator.context, region ); } + PairsBuilder( const PairsBuilder & ) = delete; + PairsBuilder & operator=( const PairsBuilder & ) = delete; + + // Alloc a node in THIS thread's slab: no lock, no atomic per node. + // The result is usable both as the node pointer and as the reference + // to store in a pointer field. + template TableSlot Alloc() { return main.Alloc(); } + // a BYTE BUFFER's node of exactly `length` bytes (docs/SPEC-TABLES.md §2.5): + // the bytes to write through, and the reference to store in a *bytes + // or *string slot; a blob past a slab takes a span of its own + TableBytesSlot AllocBytes( int64_t length ) { return main.AllocBytes( length ); } + TableStringSlot AllocString( int64_t length ) { return main.AllocString( length ); } + // one worker per thread; allocate on your own, and synchronize your own + // writes to nodes another worker allocated + TableWorker Worker() { TableWorker worker; worker.arena = &arena; return worker; } + + // GetRoot/AsConst, not Root/Const: a member function hides the type + // name it shares, and `table Root` is this spec's own canonical + // example. The checker refuses a table named after any member here, + // so the remaining spellings cannot collide either. + Pairs * GetRoot() { return arena.locked ? NULL : (Pairs *) TableArenaAt( arena, (uint32_t) root_ref.value ); } + bool Locked() const { return arena.locked; } + const Pairs * AsConst() const { return (const Pairs *) region; } + const uint8_t * Region() const { return region; } + int64_t RegionBytes() const { return region_bytes; } + + // Lock is ONE WAY and it is the compaction: the segmented arena becomes + // one exact-packed region with zero slack, references rewritten + // self-relative, and the mutable life released. Single-threaded: call + // it after the workers have joined. + bool Lock(); +}; + +inline bool PairsBuilder::Lock() +{ + if ( arena.locked ) { return region != NULL; } + if ( root_ref.null() ) { return false; } + TableArenaCtx ctx = { &arena }; + const Pairs & root = *(const Pairs *) TableArenaAt( arena, (uint32_t) root_ref.value ); + // The ROOT takes the map's first entry: it is packed at offset 0, and its + // descent is open for the whole walk (docs/SPEC-TABLES.md §3.1). + TablePackMap seen; + TablePackMapInit( seen, arena.allocator ); + bool root_taken = false; + int64_t root_slot = 0; + int64_t below = -1; + if ( TablePackMapReach( seen, (const void *) &root, 0, root_taken, root_slot ) != NULL ) + { + below = PairsPackMeasure( ctx, seen, root ); + } + if ( below < 0 ) { TablePackMapShutdown( seen ); return false; } // a data cycle, named at the reference that closes it + int64_t root_extent = PairsExtent( ctx, root ); + if ( root_extent < 0 ) { TablePackMapShutdown( seen ); return false; } // the sort could not run + int64_t total = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ) + below; + // the AUTHORING path may allocate (§6.5), and it does so through the + // builder's own pair. The region comes back ZEROED, which is the + // allocator's contract: a packed region carries node padding. + uint8_t * packed = (uint8_t *) arena.allocator.alloc( arena.allocator.context, total ); + if ( packed == NULL ) { TablePackMapShutdown( seen ); return false; } + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + Pairs * destination = new ( packed ) Pairs; // lifetime only: the Pack below memcpy's the whole node over it + // The pack walk RE-DERIVES the same numbering rather than carrying the + // measure's — nothing passes between them, which is what makes + // `used == total` below a real check and not a tautology (§3.1). The + // map keeps the capacity the measure paid for, so the second walk + // rehashes nothing. + TablePackMapReset( seen ); + if ( TablePackMapReach( seen, (const void *) &root, 0, root_taken, root_slot ) == NULL || + !PairsPack( ctx, seen, root, *destination, packed, total, used ) || used != total ) + { + TablePackMapShutdown( seen ); + arena.allocator.free( arena.allocator.context, packed ); + return false; + } + TablePackMapShutdown( seen ); + region = packed; + region_bytes = total; + arena.locked = true; // MONOTONIC: there is no unlock + TableArenaShutdown( arena ); + return true; +} + +// ---- Pairs on the wire: the FLAT NODE TABLE (docs/SPEC-TABLES.md §3.1) ---- +// +// A pointered save writes every reachable node ONCE, into a node table under +// the reserved id 0xFFFF, and a pointer field rides as a u32 INDEX into it +// under kind 17. No pointer edge is a nesting level, so a chain's length is +// not a depth and two references to one node are one node. + +// PairsNodeStorage: the region bytes one record commands, or -1 for a type id +// this build cannot name — which keeps its index and reads null. A BYTE +// BUFFER's record commands its header and its bytes (docs/SPEC-TABLES.md §2.5), +// which is the one answer the record's LENGTH decides, and a blob past the +// size cap answers kTableNodeRefused with its reason (§3.1, §6.5). +// A MAP'S ENTRIES RIDE IN THEIR HOLDER'S EXTENT (docs/SPEC-TABLES.md §2.8), +// so a record's storage is its type's PLUS N x sizeof( Entry ) at every +// depth, summed from the FRAMING: N is framing and not a value, and this +// reads no field. kTableNodeRefused is a wire whose N its L cannot carry. +inline int64_t PairsNodeStorage( uint64_t type_id, int64_t length, TableRefuseReason & reason ) +{ + (void) length; // no byte buffer below this root: every node's storage is its type's + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( (int64_t) sizeof( Item ) ); // Item + default: break; + } + (void) reason; // no blob and no extent below this root: nothing here refuses + return -1; +} + +// PairsNodePlace: start one record's node's lifetime in the storage pass one +// reserved for it, holding exactly the declared defaults — a byte buffer's +// header holds its length, and its bytes come in pass two. +inline void PairsNodePlace( uint64_t type_id, uint8_t * at, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: { Item * node = new ( at ) Item; ItemReset( *node ); break; } // Item + default: break; + } +} + +// PairsNodeRecordBytes: one record's OWN storage, before the extent its maps +// take (docs/SPEC-TABLES.md §2.8) — where a node's extent begins. +inline int64_t PairsNodeRecordBytes( uint64_t type_id ) +{ + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( (int64_t) sizeof( Item ) ); // Item + default: break; + } + return 0; +} + +// PairsNodeAlloc: the TOOL's path — one record's node in the builder's arena. +// Zero is the arena's null, and it is also what a type id this build cannot +// name answers. +inline uint32_t PairsNodeAlloc( uint64_t type_id, TableWorker & worker, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return (uint32_t) worker.Alloc().ref.value; // Item + default: break; + } + return 0; +} + +// PairsNodeBody: PASS TWO's half — decode one record's body into the storage it +// already owns. +inline void PairsNodeBody( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = PairsNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? PairsNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ItemLoadBody( r, *(Item *) at ); break; // Item + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// PairsNodeMessageStorage: the region bytes one record commands on the message +// wire, or -1 for a type id this build cannot name. A table's is its own +// storage plus the extent its maps take; a byte buffer's is its header and +// its bytes, which is the one answer the record's LENGTH decides. +inline int64_t PairsNodeMessageStorage( uint64_t type_id, int64_t extent, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Item ) ) + extent ); // Item + default: break; + } + return -1; +} + +// PairsNodeMessageExtent: step over one TABLE record's body, tallying the extent +// its maps take where its type has any (§2.8). A type this build cannot +// name is stepped over by its announced shapes and takes no extent. +inline bool PairsNodeMessageExtent( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & extent ) +{ + extent = 0; + (void) type_id; // no map below any node this root can name + return TableMessageSkipBody( r, vocabulary, index_bits ); +} + +// PairsNodeMessageBody: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +inline bool PairsNodeMessageBody( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at ) +{ + TableExtentCarve carve; + carve.at = at + PairsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !PairsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemLoadMessageBody( r, vocabulary, report, index_bits, *(Item *) at ); break; // Item + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// The numbering both wire walks derive, and NEITHER CARRIES THE OTHER'S: the +// root takes index 1 and its entry stays open for the whole walk, so a +// reference back at it is the cycle it is (§3.1). +template +inline bool PairsNumberFrom( const Ctx & ctx, TableNumbering & numbering, const Pairs & root ) +{ + bool taken = false; + int64_t slot = 0; + if ( TablePackMapReach( numbering.seen, (const void *) &root, (int64_t) kTableNodeIndexRoot, taken, slot ) == NULL ) { return false; } + return PairsNumber( ctx, numbering, root ); +} + +template +inline int64_t PairsMeasureWire( const Ctx & ctx, const Pairs & root, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bytes = -1; + if ( PairsNumberFrom( ctx, numbering, root ) ) + { + TableIds ids; + bytes = PairsMeasureBody( ctx, numbering, ids, root ); + if ( bytes >= 0 ) + { + const int64_t table = TableNodeTableMeasure( ctx, ids, numbering ); + // the FORM BYTE, the ROOT BODY — its own fields, the node table + // and the terminator — and the ID TABLE (docs/SPEC-TABLES.md §3) + bytes = table < 0 || ids.overflow ? -1 : 1 + bytes + table + TableIdsBytes( ids ); + } + } + TableNumberingShutdown( numbering ); + return bytes; +} + +template +inline int64_t PairsSaveWire( const Ctx & ctx, const Pairs & root, uint8_t * buffer, int64_t capacity, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + if ( !PairsNumberFrom( ctx, numbering, root ) ) { TableNumberingShutdown( numbering ); return -1; } + TableWriter w( buffer, capacity ); + TableIds ids; + w.put8( kTableWireForm ); // the FORM BYTE is the whole header (§3) + // the root's own fields, then the node table's field, then the + // terminator: a reader that gives up inside the table has already + // decoded the ROOT'S OWN FIELDS (§3.1) + bool ok = PairsSaveBodyFields( ctx, numbering, w, ids, root ) && TableNodeTableSave( ctx, w, ids, numbering ); + TableNumberingShutdown( numbering ); + if ( !ok || ids.overflow ) { return -1; } + w.put8( 0 ); // the ZERO REFERENCE that ends the root body + TableIdsWrite( w, ids ); + if ( w.overflow ) { return -1; } // the caller's buffer was too small + return w.offset; // == PairsMeasure( root ) +} + +inline int64_t PairsMeasure( const Pairs * root, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return PairsMeasureWire( ctx, *root, allocator ); +} + +inline int64_t PairsSave( const Pairs * root, uint8_t * buffer, int64_t capacity, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return PairsSaveWire( ctx, *root, buffer, capacity, allocator ); +} + +inline int64_t PairsMeasure( const PairsBuilder & builder ) +{ + if ( builder.region != NULL ) { return PairsMeasure( builder.AsConst(), builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return PairsMeasureWire( ctx, *(const Pairs *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), builder.arena.allocator ); +} + +inline int64_t PairsSave( const PairsBuilder & builder, uint8_t * buffer, int64_t capacity ) +{ + if ( builder.region != NULL ) { return PairsSave( builder.AsConst(), buffer, capacity, builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return PairsSaveWire( ctx, *(const Pairs *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), buffer, capacity, builder.arena.allocator ); +} + +// ---- Pairs on the MESSAGE wire: the batch over a region (docs/SPEC-TABLES.md §3.3) ---- + +// PairsMessageBodyBits: one root body's bits at bit position `at` of the batch, +// with the numbering derived from the graph, the node table FIRST, then the +// fields, then the zero reference. Measure derives the numbering and save +// derives the same one, and nothing passes between them (§3.1). +template +inline int64_t PairsMessageBodyBits( const Ctx & ctx, const Pairs & root, int64_t at, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bits = -1; + if ( PairsNumberFrom( ctx, numbering, root ) ) + { + const int64_t index_bits = TableBitsRequired( 0, numbering.count + 1 ); + const int64_t table = TableMessageNodeTableMeasure( ctx, numbering, index_bits, at ); + if ( table >= 0 ) + { + const int64_t body = PairsMeasureMessageBody( ctx, numbering, index_bits, at + table, root ); + bits = body < 0 ? -1 : table + body; + } + } + TableNumberingShutdown( numbering ); + return bits; +} + +template +inline bool PairsMessageBodySave( const Ctx & ctx, const Pairs & root, TableBitWriter & w, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + bool ok = false; + if ( PairsNumberFrom( ctx, numbering, root ) ) + { + const int64_t index_bits = TableBitsRequired( 0, numbering.count + 1 ); + ok = TableMessageNodeTableSave( ctx, numbering, index_bits, w ) && PairsSaveMessageBody( ctx, numbering, index_bits, w, root ); + } + TableNumberingShutdown( numbering ); + return ok && !w.overflow; +} + +// THE PRIMITIVE IS A BATCH (§3.3): a number of ROOTS in one buffer, one count +// and one continuous bit stream, each body carrying its own numbering. A +// root is a locked region's, `builder.AsConst()`, or a loaded one's. M above +// 256 is a refusal by name, batch_too_large, with nothing written. +inline int64_t PairsMeasureMessages( const Pairs * const * roots, int64_t count, TableReport * report, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( roots == NULL || count < 1 ) { return -1; } + if ( count > kTableMessageBatchMax ) { TableMessageRefuseBatch( report ); return -1; } + TableRegionCtx ctx; + int64_t bits = 8; // the body count + for ( int64_t i = 0; i < count; i++ ) + { + if ( roots[i] == NULL ) { return -1; } + const int64_t body = PairsMessageBodyBits( ctx, *roots[i], bits, allocator ); + if ( body < 0 ) { return -1; } + bits += body; + } + return 1 + ( bits + 7 ) / 8; +} + +inline int64_t PairsSaveMessages( const Pairs * const * roots, int64_t count, uint8_t * buffer, int64_t capacity, TableReport * report, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( roots == NULL || count < 1 ) { return -1; } + if ( count > kTableMessageBatchMax ) { TableMessageRefuseBatch( report ); return -1; } + TableMessageBatch batch; + if ( !TableMessageBatchBegin( batch, buffer, capacity, count ) ) { return -1; } + TableRegionCtx ctx; + for ( int64_t i = 0; i < count; i++ ) + { + if ( roots[i] == NULL || !PairsMessageBodySave( ctx, *roots[i], batch.w, allocator ) ) { return -1; } + batch.written++; + } + return TableMessageBatchEnd( batch ); // == PairsMeasureMessages( roots, count, report, allocator ) +} + +// PairsMessageRecordScan: one node record's type id and the extent its maps +// take, or a blob's length, the reader left after the record. A type id +// reference of 0, one past E, or one naming anything but a kind-0 entry is +// damage, as §3.1 and §3.3 say. +inline bool PairsMessageRecordScan( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, uint64_t & type_id, int64_t & extent, int64_t & length ) +{ + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { return false; } + TableMessageEntry type_entry; + if ( !TableMessageNameEntry( vocabulary, type_ref, type_entry ) ) { return false; } + type_id = type_entry.id; + extent = 0; + length = 0; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + // A BLOB RECORD CARRIES A LENGTH AT THIRTY-TWO RAW BITS, then ALIGNS, + // then the bytes verbatim (§3.3) + uint64_t n = 0; + if ( !r.get( n, 32 ) || !r.align() || !r.skip( (int64_t) n * 8 ) ) { return false; } + length = (int64_t) n; + return true; + } + return PairsNodeMessageExtent( type_id, r, vocabulary, index_bits, extent ); +} + +// PairsMessageBodyStorage: one body's node count and data bytes from the FRAMING +// alone, the reader left at the next body. The node table is walked record +// by record, a table record's body stepped over by its announced shapes and +// a blob's by its length, then the root's own fields. False is a numbering +// that could not be sized; `complete` false is a ROOT body whose own framing +// gave out, which the load meets as damage inside this body after the +// bodies before it were delivered, so the batch is sized through this body +// and no further (§3.3). +inline bool PairsMessageBodyStorage( TableBitReader & r, const TableVocabulary & vocabulary, int64_t & records, int64_t & data, bool & complete ) +{ + complete = true; + records = 0; + data = 0; + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { return false; } + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !PairsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { return false; } + const int64_t storage = PairsNodeMessageStorage( type_id, extent, length ); + if ( storage > 0 ) { data += storage; } // a type id this build cannot name commands none + records++; + } + int64_t root_extent = 0; + if ( !PairsMessageExtent( r, vocabulary, index_bits, root_extent ) ) { complete = false; } + data += TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + return true; +} + +// PairsLoadMeasure's MESSAGE overload: the exact region bytes ONE BATCH needs, +// which is one measurement, one allocation and one bounds check for however +// many bodies ride (§3.3, §6.5). It is a scan by the announced shapes and +// reads no field value. The answer is the data bytes plus the attribution, +// one node directory a body, and -1 for a wire it cannot size: no vocabulary, +// another form, or framing that gives out. +inline int64_t PairsLoadMeasure( const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, int64_t * attribution_bytes = NULL ) +{ + TableReport ignored; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, &ignored ); + if ( bodies < 0 ) { return -1; } + int64_t data = 0, attribution = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + int64_t records = 0, body_data = 0; + bool complete = true; + if ( !PairsMessageBodyStorage( br.r, vocabulary, records, body_data, complete ) ) { return -1; } + data += body_data; + attribution += ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( !complete ) { break; } // damage inside this body: the load delivers the ones before it + } + if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; } + return data + attribution; +} + +// PairsLoadMessageBodyInto: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +inline bool PairsLoadMessageBodyInto( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Pairs * & root_out ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !PairsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = PairsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + PairsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !PairsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0x1404200dab337086ull; + Pairs * root = new ( region + used ) Pairs; // lifetime only: LoadMessageBody's first act is PairsReset + PairsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Pairs ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + out->unknown += unknown_records; + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !PairsNodeMessageBody( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return PairsLoadMessageBody( r, vocabulary, out, nodes, index_bits, *root ); +} + +// PairsLoadMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +inline bool PairsLoadMessages( const Pairs ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !PairsLoadMessageBodyInto( br.r, vocabulary, out, region, region_bytes, used, roots[b] ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + +// PairsLoadMeasure: the exact region bytes a wire buffer will need, and it is +// ONE SCAN — a record's type id gives its storage size, its length gives the +// next record — reading no field value at all, so the caller owns the +// allocation and can refuse a number it did not expect (§6.5). +// +// It reports the DATA bytes and the ATTRIBUTION bytes separately, because the +// attribution is the wire's numbering made resident (§6.3) and a caller may +// release it once Load returns. The answer is their sum. +inline int64_t PairsLoadMeasure( const uint8_t * wire_file, int64_t wire_file_bytes, int64_t * attribution_bytes = NULL, TableRefuseReason * reason_out = NULL ) +{ + TableReport ignored; + TableIdTable ids_table; + int64_t body_bytes = 0; + // a FORM BYTE this build does not carry is refused by name (§3, §6.5); + // a trailer that cannot be read whole is damage and names no reason + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict == TableOpenRefused ) { if ( reason_out != NULL ) { *reason_out = unknown_form; } return -1; } + if ( verdict != TableOpenOk ) { return -1; } + // ANY BYTE BETWEEN THE ROOT'S TERMINATOR AND THE TABLE'S FIRST ENTRY + // IS MALFORMED (docs/SPEC-TABLES.md §3): the two ends of the file have + // met, nothing is decoded, and no region is sized from it. + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) { return -1; } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &ignored, &ids_table ); + TableRefuseReason reason = count_over_length; + int64_t root_extent = 0; + if ( !PairsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { if ( reason_out != NULL ) { *reason_out = reason; } return -1; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + int64_t records = 0; + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = PairsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { if ( reason_out != NULL ) { *reason_out = reason; } return -1; } // an N the record's framing cannot carry, or a blob past the cap (§2.8, §2.9, §3.1) + if ( storage > 0 ) { data += storage; } // a type id this build cannot name commands none + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; } + return data + attribution; +} + +// PairsLoad: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +inline const Pairs * PairsLoad( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Pairs ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !PairsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = PairsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0x1404200dab337086ull; + Pairs * root = new ( region ) Pairs; // lifetime only: LoadBody's first act is PairsReset + PairsReset( *root ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = PairsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + PairsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + if ( nodes.good ) { out->unknown += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + PairsNodeBody( type_id, sub, nodes, region + directory[k + 1].offset ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Pairs ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + PairsLoadBody( r, nodes, *root ); + return root; +} + +// PairsLoadBuilder: the TOOL's path — the same tolerant decode into a fresh +// builder, so loaded data can be edited and locked again. The numbering is +// the same one; what differs is where a node lives and therefore what a +// resolved slot holds — an arena offset here, a self-relative delta there. +inline bool PairsLoadBuilder( PairsBuilder & builder, const uint8_t * wire_file, int64_t wire_file_bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; } + return false; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return false; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + Pairs * root = builder.GetRoot(); + if ( root == NULL ) { out->malformed = true; return false; } + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) { records++; } + } + // the AUTHORING side may allocate (§6.5), and this is the tool's path. + // It goes through the builder's own pair, like everything else the + // builder reaches, and the entries come back zeroed. + const TableAllocator allocator = builder.arena.allocator; + TableNodeDirEntry * directory = (TableNodeDirEntry *) allocator.alloc( allocator.context, ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ) ); + if ( directory == NULL ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) builder.root_ref.value; + directory[0].type_id = 0x1404200dab337086ull; + TableNodeMap nodes; + nodes.base = NULL; + nodes.entries = directory; + nodes.count = records + 1; + nodes.arena = true; // a resolved slot holds the node's ARENA OFFSET here + nodes.worker = &builder.main; // and a map's entries and a list's elements are the arena's, not a node extent's (§2.8, §2.9) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + uint32_t at = PairsNodeAlloc( type_id, builder.main, length ); + if ( at == 0 ) + { + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + } + else + { + directory[k + 1].offset = (uint64_t) at; + } + directory[k + 1].type_id = type_id; + k++; + } + nodes.good = TableNodeScanWhole( scan ); + if ( nodes.good ) { out->unknown += unknown_records; } else { out->malformed = true; } + } + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + PairsNodeBody( type_id, sub, nodes, TableArenaAt( builder.arena, (uint32_t) directory[k + 1].offset ) ); + } + k++; + } + } + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.worker = &builder.main; + nodes.carve = &root_carve; + bool ok = PairsLoadBody( r, nodes, *root ); + // A COUNT ABOVE THE int32 CAP is this path's refusal (docs/SPEC-TABLES.md + // §2.9): the partial builder is the caller's to discard, and the report + // holds what it held when the count was met + ok = ok && !nodes.refused; + allocator.free( allocator.context, directory ); + return ok; +} + +template +inline int64_t PairsSlotsEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + { + bool any_value = false; + for ( int32_t i = 0; i < 2; i++ ) { if ( ItemAt( ctx, value.value[i] ) != NULL ) { any_value = true; break; } } + if ( any_value ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( 2 ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < 2; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return -1; } + body_value += TableLebBytes( slot_index ); + } + } + bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); // value: [2]*Item + } + } + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool PairsSlotsEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 8 ); // key + w.put32( uint32_t( value.key ) ); + } + { + bool any_value = false; + for ( int32_t i = 0; i < 2; i++ ) { if ( ItemAt( ctx, value.value[i] ) != NULL ) { any_value = true; break; } } + if ( any_value ) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( 2 ) ); // the element kind byte and the count + for ( int32_t elem_i = 0; elem_i < 2; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + body_value += TableLebBytes( slot_index ); + } + } + w.putleb( ref_value ); w.put8( 14 ); w.putleb( (uint64_t) body_value ); // value + w.put8( 17 ); w.putleb( (uint64_t) ( 2 ) ); + for ( int32_t elem_i = 0; elem_i < 2; elem_i++ ) + { + { + const Item * slot_pointee = ItemAt( ctx, value.value[elem_i] ); + uint64_t slot_index = 0; + if ( slot_pointee != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee, slot_index ) ) { return false; } + w.putleb( slot_index ); + } + } + } + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool PairsSlotsEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !PairsSlotsEntrySaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool PairsSlotsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, PairsSlotsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + PairsSlotsEntryReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x3dc94a19365b10ecull: // key + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.key = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.key = decoded_v; + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER — the element kind byte and the + // count, so fewer than two bytes — is INERT (§4): the field keeps the + // value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + // A DAMAGED COUNT stops the elements and nothing else: the field + // RODE, so an optional is still PRESENT (§2.3) — only a foreign + // ELEMENT KIND says the payload is not this array's at all. + if ( !counted_ok ) { r.report->malformed = true; } + else if ( elem_kind != 17 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + uint64_t keep = count; + if ( keep > 2 ) { keep = 2; r.report->clamped++; } + // elements are BOUNDED by the field body: a count the length + // cannot cover keeps the decoded prefix, flags malformed, and + // the parent continues at the next field — following fields' + // bytes are never fabricated into elements + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + for ( uint64_t i = 0; i < keep; i++ ) + { + { + uint64_t node_index = 0; + if ( !sub.getleb( node_index ) ) { r.report->malformed = true; break; } + TableNodeResolve( nodes, value.value[(int32_t) i], node_index, 0x52cfa1d198476806ull, r.report ); // *Item + } + } + } + } + r.offset = body_end; // excess elements and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +template +inline int64_t PairsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // slots: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); + if ( !order_slots.ok ) { return -1; } // the sort could not run + if ( order_slots.count > 0 ) + { + const uint64_t ref_slots = ids.ref( 0xe68c2e6bb1ee5646ull ); + int64_t body_slots = 1 + TableLebBytes( (uint64_t) order_slots.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_slots = PairsSlotsEntryMeasureBodyRetain( ctx, numbering, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return -1; } + body_slots += TableLebBytes( (uint64_t) ( elem_slots ) ) + ( elem_slots ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_slots ) + 1 + TableLebBytes( (uint64_t) ( body_slots ) ) + ( body_slots ); + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool PairsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + { + TableMapCursor order_slots = TableMapOrder( ctx, value.slots ); // slots + if ( !order_slots.ok ) { return false; } + if ( order_slots.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_slots = ids.ref( 0xe68c2e6bb1ee5646ull ); + int64_t body_slots = 1 + TableLebBytes( (uint64_t) order_slots.count ); + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_slots = PairsSlotsEntryMeasureBodyRetain( ctx, numbering, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_slots < 0 ) { TableMapRelease( order_slots ); return false; } + body_slots += TableLebBytes( (uint64_t) ( elem_slots ) ) + ( elem_slots ); + } + w.putleb( ref_slots ); w.put8( 14 ); w.putleb( (uint64_t) body_slots ); + w.put8( 13 ); w.putleb( (uint64_t) order_slots.count ); + for ( int32_t i = 0; i < order_slots.count; i++ ) + { + const int64_t elem_len_slots = PairsSlotsEntryMeasureBodyRetain( ctx, numbering, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_len_slots < 0 ) { TableMapRelease( order_slots ); return false; } + w.putleb( (uint64_t) elem_len_slots ); + if ( !PairsSlotsEntrySaveBodyRetain( ctx, numbering, w, ids, *order_slots[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_slots ); return false; } + } + } + TableMapRelease( order_slots ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool PairsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !PairsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool PairsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Pairs & value, TableRetain * retain, const TableRetainPath & path ) +{ + PairsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0xe68c2e6bb1ee5646ull: // slots + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.slots, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + PairsSlotsEntryKeyRead read = PairsSlotsEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + PairsSlotsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + PairsSlotsEntryLoadBodyRetain( elem, nodes, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// PairsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void PairsNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = PairsNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? PairsNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ItemLoadBodyRetain( r, *(Item *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Item + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// PairsLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +// UNDER RETENTION it also fills the caller's two stores with the fields +// this build cannot name, and the report carries what it could not keep +// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the +// reader's data is exactly what it would have been with retention off. +inline const Pairs * PairsLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Pairs ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !PairsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = PairsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0x1404200dab337086ull; + Pairs * root = new ( region ) Pairs; // lifetime only: LoadBody's first act is PairsReset + PairsReset( *root ); + + // LoadRetain RESETS BOTH STORES and writes into neither id list: a + // retained record carries its field's identity in the record itself, + // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer + // belongs to this region from here on. + TableRetainReset( retain, nodes, region ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Pairs ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = PairsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + PairsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + PairsNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Pairs ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + PairsLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// PairsMeasureRetain and PairsSaveRetain: the pair, with the retained tail in +// every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same +// records under the same walk, so Measure's answer is the size the save +// writes even where a record could not be placed. +template +inline int64_t PairsMeasureWireRetain( const Ctx & ctx, const Pairs & root, TableRetain * retain, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bytes = -1; + auto retain_measure = []( const Ctx & c, const TableNumbering & nn, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> int64_t + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemMeasureBodyRetain( ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return -1; + }; + if ( PairsNumberFrom( ctx, numbering, root ) ) + { + TableRetainIds ids( retain ); + if ( retain != NULL ) { retain->id_used = 0; } // one walk fills the list, and the save's own walk refills it + bytes = PairsMeasureBodyRetain( ctx, numbering, ids, root, retain, TableRetainPathRoot( (const void *) &root, 1 ) ); + if ( bytes >= 0 ) + { + const int64_t table = TableNodeTableMeasureRetain( ctx, ids, numbering, retain, retain_measure ); + bytes = table < 0 || ids.overflow ? -1 : 1 + bytes + table + TableRetainIdsBytes( ids ); + } + } + TableNumberingShutdown( numbering ); + return bytes; +} + +template +inline int64_t PairsSaveWireRetain( const Ctx & ctx, const Pairs & root, TableRetain * retain, uint8_t * buffer, int64_t capacity, TableReport * report ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, TableDefaultAllocator() ); + auto retain_measure = []( const Ctx & c, const TableNumbering & nn, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> int64_t + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemMeasureBodyRetain( ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return -1; + }; + auto retain_save = []( const Ctx & c, const TableNumbering & nn, TableWriter & ww, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> bool + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ww; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemSaveBodyRetain( ww, ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return false; + }; + if ( !PairsNumberFrom( ctx, numbering, root ) ) { TableNumberingShutdown( numbering ); return -1; } + TableWriter w( buffer, capacity ); + TableRetainIds ids( retain ); + if ( retain != NULL ) { retain->id_used = 0; } + TableRetainClearPlaced( retain ); + w.put8( kTableWireForm ); // the FORM BYTE is the whole header (§3) + // the root's own fields, then the RETAINED TAIL, then the node table's + // field: a retained field is one of the root's own values, and the tail + // is pinned before the large and damage-prone part (§6.6, §3.1) + bool ok = PairsSaveBodyFieldsRetain( ctx, numbering, w, ids, root, retain, TableRetainPathRoot( (const void *) &root, 1 ) ) && + TableNodeTableSaveRetain( ctx, w, ids, numbering, retain, retain_measure, retain_save ); + TableNumberingShutdown( numbering ); + if ( !ok || ids.overflow ) { return -1; } + w.put8( 0 ); // the ZERO REFERENCE that ends the root body + TableRetainIdsWrite( w, ids ); + if ( w.overflow ) { return -1; } // the caller's buffer was too small + // THE SAVE'S OWN SHARE OF retain_lost, read after the save (§6.6): every + // record the walk did not place, counted once. + TableRetainCountLost( retain, report ); + return w.offset; +} + +inline int64_t PairsMeasureRetain( const Pairs * root, TableRetain * retain, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return PairsMeasureWireRetain( ctx, *root, retain, allocator ); +} + +// SaveRetain REFUSES A NULL REPORT and returns -1 (docs/SPEC-TABLES.md +// §6.6): the save is the only place a caller learns that a record was +// dropped, so the report is required here where it is optional everywhere +// else. A surface that let a caller retain, save and never find out would +// be a promise it could not check. +inline int64_t PairsSaveRetain( const Pairs * root, TableRetain * retain, uint8_t * buffer, int64_t capacity, TableReport * report ) +{ + if ( root == NULL || report == NULL ) { return -1; } + TableRegionCtx ctx; + return PairsSaveWireRetain( ctx, *root, retain, buffer, capacity, report ); +} + +// PairsSaveRetainMessages: RETENTION WRITING FORM 2 IS REFUSED BY NAME +// (docs/SPEC-TABLES.md §3.3). It is a MISUSE refusal on §6.6's own +// precedent and never a silent drop, and the two answers are named: a +// caller that must carry unknowns across a rewrite writes the FILE form, +// which carries its own table and takes §6.6 unchanged, and a RELAY +// forwards the sending peer's announcement and its batch bytes verbatim. +template +inline int64_t PairsSaveRetainMessages( Args &&... ) +{ + static_assert( sizeof...( Args ) == (size_t) -1, + "Pairs: a form 2 writer names entries through slots of a vocabulary the compiler settled, and a retained id is one this build's closure does not contain, so it has neither a slot nor an announced shape. Retention writing the MESSAGE form is refused by name (docs/SPEC-TABLES.md §3.3). Write the FILE form, which carries its own table and takes §6.6 unchanged, or relay the sender's announcement and batch bytes verbatim." ); + return -1; +} + +// ---- the cooked form: point at a cook (docs/SPEC-TABLES.md §7) ---- + +// PairsOpen: match the header and POINT. On a match the bytes ARE what this +// build wrote, in this build's layout and this build's byte order, so there +// is nothing to validate and nothing to fix up and the root comes back as it +// lies. On ANY refusal it returns NULL and NAMES the refusal in the caller's +// TableRefuseReason, the first failing clause in §7's order (a wrong build +// version is a re-cook, a foreign order a cross-endian cook, a truncated +// file a bad download, an unaligned base the caller's own buffer), and the +// caller falls back to a wire load, which is the path that carries every +// version. The reason is written on the refusal path only; a caller that +// passes nothing gets the null alone. +// +// It is O(1) IN THE FILE'S SIZE — the header and nothing per node — so a one +// megabyte cook and a one gigabyte cook open in the same time, and a mapped +// file's pages are touched only as they are used. That is a property of +// touching nothing at open rather than a separate mechanism. +// +// A REFERENCE INSIDE THE REGION IS DEREFERENCED THROUGH PairsAt: the slot holds +// the signed self-relative byte delta of §6.3, so a deref is one add and +// needs no base pointer, a whole region relocates by plain memcpy, and a +// delta of zero is null. +// +// There is ONE entry point and no tolerant twin: a build either wrote this +// file or it did not, and the build version is what says which. Validating a +// file whose provenance a person doubts is schema cook-check, offline, +// over the ATTRIBUTION part beside the data — a person's decision, never a +// parameter on a load. +inline const Pairs * PairsOpen( const void * bytes, uint64_t length, TableRefuseReason * reason = NULL ) +{ + return (const Pairs *) TableCookOpen( bytes, length, (uint64_t) sizeof( Pairs ), (uint64_t) alignof( Pairs ), reason ); +} + +// ---- the cooked form: WRITE a cook (docs/SPEC-TABLES.md §7.6) ---- +// +// The bytes are `schema cook`'s, and the tool stays the reference: the two +// writers are held to one file, byte for byte, in both byte orders. A cook is +// content-addressed by (asset hash, build version), so two writers of one +// instance produce ONE artifact or the pair means nothing. + +template inline bool PairsSlotsEntryCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const PairsSlotsEntry & value, TableByteOrder order ); +template inline bool PairsCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Pairs & value, TableByteOrder order ); + +template inline bool PairsSlotsEntryCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const PairsSlotsEntry & value, TableByteOrder order ) +{ + table_cook_put( at + 0, (uint64_t) value.key, 4, order ); + for ( int32_t i = 0; i < 2; i++ ) // value: an array of pointers, every slot + { + if ( !table_cook_ref( region, at + 8 + i * 8, (const void *) ItemAt( ctx, value.value[ i ] ), order ) ) { return false; } + } + return true; +} + +template inline bool PairsCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Pairs & value, TableByteOrder order ) +{ + (void) ctx; (void) region; // no reference resolves in this body: a list's and a map's slots are the extent writer's, and the class was decided elsewhere in the closure + table_cook_put( at + 0, 0, 8, order ); // slots: the array's delta, filled by the extent writer + table_cook_put( at + 8, 0, 4, order ); // and its count + table_cook_put( at + 16, (uint64_t) value.after, 4, order ); + return true; +} + +template inline bool PairsSlotsEntryCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const PairsSlotsEntry & value, TableByteOrder order ); +template inline bool PairsCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const Pairs & value, TableByteOrder order ); + +// PairsSlotsEntryCookExtent: PairsSlotsEntry's arrays into the node's extent, PRE-ORDER, a map's entries +// in ASCENDING key order and a list's elements in INDEX order, each through its +// own cook writer (§2.8, §2.9, §7.6). +template inline bool PairsSlotsEntryCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const PairsSlotsEntry & value, TableByteOrder order ) +{ + (void) ctx; (void) region; (void) extent; (void) at; (void) record; (void) value; (void) order; + return true; // no list or map below this record +} + +// PairsCookExtent: Pairs's arrays into the node's extent, PRE-ORDER, a map's entries +// in ASCENDING key order and a list's elements in INDEX order, each through its +// own cook writer (§2.8, §2.9, §7.6). +template inline bool PairsCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const Pairs & value, TableByteOrder order ) +{ + (void) region; // a table element's and an entry's references resolve through their own bodies + { // slots + TableMapCursor cursor = TableMapOrder( ctx, value.slots ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( PairsSlotsEntry ) + uint8_t * array = extent + at; + at += (int64_t) cursor.count * 24; // the whole array FIRST + // the SIXTEEN BYTES of the slot: the self-relative delta, then the count + table_cook_put( record + 0, cursor.count > 0 ? (uint64_t) (int64_t) ( array - ( record + 0 ) ) : 0, 8, order ); + table_cook_put( record + 8, (uint64_t) (uint32_t) cursor.count, 4, order ); + for ( int32_t i = 0; i < cursor.count; i++ ) + { + if ( !PairsSlotsEntryCookBody( ctx, region, array + i * 24, *cursor[i], order ) ) { return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// PairsSlotsEntryCookNode: one node, the record, then the extent its lists and maps take (§2.8, §2.9). +template inline bool PairsSlotsEntryCookNode( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const PairsSlotsEntry & value, TableByteOrder order ) +{ + if ( !PairsSlotsEntryCookBody( ctx, region, at, value, order ) ) { return false; } + int64_t extent_at = 0; + return PairsSlotsEntryCookExtent( ctx, region, at + 24, extent_at, at, value, order ); +} + +// PairsCookNode: one node, the record, then the extent its lists and maps take (§2.8, §2.9). +template inline bool PairsCookNode( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Pairs & value, TableByteOrder order ) +{ + if ( !PairsCookBody( ctx, region, at, value, order ) ) { return false; } + int64_t extent_at = 0; + if ( !PairsCookExtent( ctx, region, at + 24, extent_at, at, value, order ) ) { return false; } + return extent_at == PairsExtent( ctx, value ); // the extent written is the extent measured, or no header is written +} + +// PairsCookLayout: the tool's own Layout (docs/SPEC-TABLES.md §7.2) over one +// numbering — the root at zero, then every node in index order at +// align_up( offset, alignof ) for its OWN type, no slack between them, the +// data length rounded to the greatest alignment among them and never below +// eight. The offsets go into the region's table when it has one, and are only +// summed when it does not (a measure). A type id the numbering carries that +// this root cannot name is the two walks disagreeing, and it is refused. +// A NODE'S SIZE DEPENDS ON ITS VALUE where a list or a map rides in its extent +// (docs/SPEC-TABLES.md §2.8), so the layout takes the resolution context +// the numbering walked and reads the same arrays that walk read. +template +inline bool PairsCookLayout( const Ctx & ctx, const Pairs & root, const TableNumbering & numbering, TableCookRegion & region ) +{ + region.numbering = &numbering; + region.count = numbering.count + 1; + const int64_t root_extent = PairsExtent( ctx, root ); + if ( root_extent < 0 ) { return false; } + int64_t offset = 24 + root_extent; // the root at zero, its extent behind it + int64_t align = 8; + if ( region.offsets != NULL ) { region.offsets[0] = 0; } + for ( int64_t k = 0; k < numbering.count; k++ ) + { + int64_t size = 0; + int64_t node_align = 0; + switch ( numbering.entries[k].type_id ) + { + case 0x52cfa1d198476806ull: size = 4; node_align = 4; break; // Item + default: return false; + } + offset = ( offset + node_align - 1 ) & ~( node_align - 1 ); + if ( region.offsets != NULL ) { region.offsets[k + 1] = offset; } + offset += size; + if ( node_align > align ) { align = node_align; } + } + region.bytes = ( offset + align - 1 ) & ~( align - 1 ); + region.align = align; + return true; +} + +// PairsCookMeasureFrom: the whole cooked file's bytes for one graph — the header, +// the data part and the attribution part (§7.1). IT DEPENDS ON THE VALUE, +// because the answer is the numbering: the depth-first walk of §3.1 is run +// here and run again by the write, and neither carries the other's (§7.6). A +// data cycle is refused by the walk and answers -1. +template +inline int64_t PairsCookMeasureFrom( const Ctx & ctx, const Pairs & root, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + TableCookRegion region; + int64_t bytes = -1; + if ( PairsNumberFrom( ctx, numbering, root ) && PairsCookLayout( ctx, root, numbering, region ) ) + { + const int64_t data_offset = ( kTableCookHeaderBytes + region.align - 1 ) & ~( region.align - 1 ); + bytes = data_offset + region.bytes + region.count * (int64_t) sizeof( TableNodeDirEntry ); + } + TableNumberingShutdown( numbering ); + return bytes; +} + +// PairsCookFrom: write one cooked file of a pointered graph, in the byte order +// the caller names. The bytes are `schema cook`'s, byte for byte (§7.6). +// +// THE CALLER OWNS THE OUTPUT and nothing is allocated toward it. What is +// allocated is the numbering — the identity map, the entry array and one +// offset per node — through the pair handed in, and released before this +// returns (§6.5, §13.9). A capacity short of the measure writes nothing. +// +// THE HEADER IS WRITTEN LAST. A reference the numbering did not carry is +// found while a body is being written, and a write that refuses there has +// already put bytes in the buffer; with no magic ahead of them, no Open can +// mistake them for a cook. +template +inline bool PairsCookFrom( const Ctx & ctx, const Pairs & root, void * out, uint64_t capacity, TableByteOrder order, TableAllocator allocator ) +{ + if ( out == NULL ) { return false; } + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + TableCookRegion region; + bool ok = PairsNumberFrom( ctx, numbering, root ); + if ( ok ) + { + region.offsets = (int64_t *) allocator.alloc( allocator.context, ( numbering.count + 1 ) * (int64_t) sizeof( int64_t ) ); + ok = region.offsets != NULL && PairsCookLayout( ctx, root, numbering, region ); + } + if ( ok ) + { + const int64_t data_offset = ( kTableCookHeaderBytes + region.align - 1 ) & ~( region.align - 1 ); + const int64_t attribution = region.count * (int64_t) sizeof( TableNodeDirEntry ); + const int64_t need = data_offset + region.bytes + attribution; + ok = (uint64_t) need <= capacity; + if ( ok ) + { + uint8_t * raw = (uint8_t *) out; + memset( raw, 0, (size_t) need ); // EVERY BYTE NO FIELD COVERS IS ZERO (§7.2) + region.base = raw + data_offset; + // the DATA part: the root at the region's base, then every numbered + // node at the offset the layout gave it, each through its own writer + ok = PairsCookNode( ctx, region, region.base, root, order ); + for ( int64_t k = 0; ok && k < numbering.count; k++ ) + { + uint8_t * at = region.base + region.offsets[k + 1]; + const void * node = numbering.entries[k].node; + switch ( numbering.entries[k].type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemCookNode( ctx, region, at, *(const Item *) node, order ); break; // Item + default: ok = false; break; + } + } + // the ATTRIBUTION part: the node directory (§6.3), one entry per node + // in index order, for `schema cook-check` + uint8_t * entry = raw + data_offset + region.bytes; + table_cook_put( entry, 0, 8, order ); + table_cook_put( entry + 8, 0x1404200dab337086ull, 8, order ); // the root: fnv1a64( "Pairs" ) + for ( int64_t k = 0; k < numbering.count; k++ ) + { + entry += sizeof( TableNodeDirEntry ); + table_cook_put( entry, (uint64_t) region.offsets[k + 1], 8, order ); + table_cook_put( entry + 8, numbering.entries[k].type_id, 8, order ); + } + // and the HEADER (§7.1), every word a u64 in the order the file is + // produced in; the two RESERVED words are the memset's zeros + if ( ok ) + { + table_cook_put( raw + 0, TableCookMagic, 8, order ); + table_cook_put( raw + 8, BuildVersion, 8, order ); + table_cook_put( raw + 16, (uint64_t) ( order == TableByteOrder::Big ? 2 : 1 ), 8, order ); + table_cook_put( raw + 24, (uint64_t) region.bytes, 8, order ); + table_cook_put( raw + 32, (uint64_t) attribution, 8, order ); + table_cook_put( raw + 40, (uint64_t) region.align, 8, order ); + } + } + } + allocator.free( allocator.context, region.offsets ); + TableNumberingShutdown( numbering ); + return ok; +} + +// PairsCookMeasure / PairsCook over a REGION root — a locked builder's AsConst, a +// region PairsLoad produced, or an opened cook — with the pair the numbering +// allocates through as an optional last argument, as the wire's own entries +// take it (§13.9). +inline int64_t PairsCookMeasure( const Pairs * root, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return PairsCookMeasureFrom( ctx, *root, allocator ); +} + +inline bool PairsCook( const Pairs * root, void * out, uint64_t capacity, TableByteOrder order, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return false; } + TableRegionCtx ctx; + return PairsCookFrom( ctx, *root, out, capacity, order, allocator ); +} + +// and over a BUILDER, locked or not: the builder's own pair, and the arena +// encoding while it is still mutable (§6.3). +inline int64_t PairsCookMeasure( const PairsBuilder & builder ) +{ + if ( builder.region != NULL ) { return PairsCookMeasure( builder.AsConst(), builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return PairsCookMeasureFrom( ctx, *(const Pairs *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), builder.arena.allocator ); +} + +inline bool PairsCook( const PairsBuilder & builder, void * out, uint64_t capacity, TableByteOrder order ) +{ + if ( builder.region != NULL ) { return PairsCook( builder.AsConst(), out, capacity, order, builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return false; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return PairsCookFrom( ctx, *(const Pairs *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), out, capacity, order, builder.arena.allocator ); +} + +// ---- relocatability, enforced: the wire is a pure length-prefixed +// stream AND the decoded storage is pointer-free — every closure type +// must stay trivially copyable and standard-layout, so instances can be +// memcpy'd, mmap'd, shared across processes, and walked through +// descriptor offsets. A failure here means a pointer, virtual or +// non-trivial member crept into generated storage. +// +// They ask the COMPILER ITSELF, which is what every C++ standard library +// answers the same two questions with — and it costs this header no +// include at all. +// A pointer FIELD is a TableRef — eight bytes and no address — so the +// property holds in BOTH forms: a fixed-size table is one relocatable +// struct, and a packed region is one relocatable block whose references +// are self-relative and therefore survive a plain memcpy. +static_assert( __is_trivially_copyable( PairsSlotsEntry ), "PairsSlotsEntry must stay relocatable" ); +static_assert( __is_standard_layout( PairsSlotsEntry ), "PairsSlotsEntry must stay standard-layout for offsetof" ); +static_assert( __is_trivially_copyable( Pairs ), "Pairs must stay relocatable" ); +static_assert( __is_standard_layout( Pairs ), "Pairs must stay standard-layout for offsetof" ); + +// ---- the cook's layout contract (docs/SPEC-TABLES.md §20.3) ---- +// +// The compiler derived every number below from the declaration and folded it +// into the BUILD VERSION; these asserts are this compiler saying whether it +// agrees. The model is not self-evidently right — on 32-bit System V +// alignof(uint64_t) is 4, not 8 — which is precisely why it is asserted +// rather than assumed. +static_assert( sizeof( PairsSlotsEntry ) == 24, "PairsSlotsEntry's sizeof moved: the build version was taken over 24, so a cook of it would not be this build's file (docs/SPEC-TABLES.md §20.3)" ); +static_assert( alignof( PairsSlotsEntry ) == 8, "PairsSlotsEntry's alignof moved: the build version was taken over 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( PairsSlotsEntry, key ) == 0, "PairsSlotsEntry's field key moved: the build version was taken over offset 0 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( PairsSlotsEntry, value ) == 8, "PairsSlotsEntry's field value moved: the build version was taken over offset 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( sizeof( Pairs ) == 24, "Pairs's sizeof moved: the build version was taken over 24, so a cook of it would not be this build's file (docs/SPEC-TABLES.md §20.3)" ); +static_assert( alignof( Pairs ) == 8, "Pairs's alignof moved: the build version was taken over 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( Pairs, slots ) == 0, "Pairs's field slots moved: the build version was taken over offset 0 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( Pairs, after ) == 16, "Pairs's field after moved: the build version was taken over offset 16 (docs/SPEC-TABLES.md §20.3)" ); + + +// ---- reflection descriptors (tables only, docs/SPEC-TABLES.md) ---- + +inline const TableTypeInfo * PairsSlotsEntryTableType(); +inline const TableTypeInfo * PairsTableType(); +// The descriptors are CONSTANT-INITIALISED data, and a field's target is +// the ADDRESS of another descriptor. These declarations are what let a +// self- or mutually-referential graph — Node naming itself through *Node — +// be expressed as constant data instead of a lazy link, which could not +// have been written race-free OR recursion-safe. The whole reflection +// surface is therefore immutable: read it from any thread, any time. +extern const TableTypeInfo PairsSlotsEntryTableInfo; +extern const TableTypeInfo PairsTableInfo; + +inline const TableFieldInfo PairsSlotsEntryTableFields[] = { + { "key", "key", "uint32", 0x3dc94a19365b10ecull, 8, false, false, NULL, NULL, false, false, 0, (uint32_t) offsetof( PairsSlotsEntry, key ), (uint32_t) sizeof( PairsSlotsEntry::key ), 0xffffffffu, 0xffffffffu, NULL, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, + { "value", "value", "Item", 0x7ce4fd9430e80ceaull, 17, true, true, []( const void * slot ) -> const void * { return (const void *) ItemAt( *(const TableRef *) slot ); }, []( TableWorker & worker, void * slot ) -> void * { return (void *) ItemEmplace( worker, *(TableRef *) slot ); }, false, false, 2, (uint32_t) offsetof( PairsSlotsEntry, value ), (uint32_t) sizeof( PairsSlotsEntry::value[0] ), 0xffffffffu, 0xffffffffu, &ItemTableInfo, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, +}; +inline const TableTypeInfo PairsSlotsEntryTableInfo = { "PairsSlotsEntry", (uint32_t) sizeof( PairsSlotsEntry ), 2, PairsSlotsEntryTableFields, +[]( void * p ) { PairsSlotsEntryReset( *(PairsSlotsEntry *) p ); }, true, TableDocNone, 0, NULL }; +inline const TableTypeInfo * PairsSlotsEntryTableType() { return &PairsSlotsEntryTableInfo; } + +inline const TableFieldInfo PairsTableFields[] = { + { "slots", "slots", "map[uint32]*Item", 0xe68c2e6bb1ee5646ull, 13, true, false, NULL, NULL, true, false, 0, (uint32_t) offsetof( Pairs, slots ), (uint32_t) sizeof( PairsSlotsEntry ), (uint32_t) offsetof( Pairs, slots.count ), 0xffffffffu, &PairsSlotsEntryTableInfo, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, []( TableWorker & worker, void * slot, const char *, int32_t, int64_t key_value ) -> void * { return (void *) TableMapPlace( worker, *(TableMap *) slot, (uint32_t) key_value ); }, "", TableDocNone, 0, NULL }, + { "after", "after", "int32", 0xbf82010f6f71eae9ull, 4, false, false, NULL, NULL, false, false, 0, (uint32_t) offsetof( Pairs, after ), (uint32_t) sizeof( Pairs::after ), 0xffffffffu, 0xffffffffu, NULL, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, +}; +inline const TableTypeInfo PairsTableInfo = { "Pairs", (uint32_t) sizeof( Pairs ), 2, PairsTableFields, +[]( void * p ) { PairsReset( *(Pairs *) p ); }, true, TableDocNone, 0, NULL }; +inline const TableTypeInfo * PairsTableType() { return &PairsTableInfo; } + +// ---- the text form (docs/SPEC-TABLES.md §16) ---- + +// Pairs in and out of a JSON text (docs/SPEC-TABLES.md §16.7): read into a +// builder, written from a region's const root. A node named more than once +// carries `&node` in the text. Defined in PairsTable.cpp; link it to use them. +bool PairsFromJson( PairsBuilder & builder, const char * text, int64_t bytes, TableReport * report ); +int64_t PairsToJsonMeasure( const Pairs * root, TableAllocator allocator = TableDefaultAllocator() ); +int64_t PairsToJson( const Pairs * root, char * buffer, int64_t capacity, TableAllocator allocator = TableDefaultAllocator() ); + +} // namespace mapdemo diff --git a/testdata/golden/tables/maps/RowsTable.h b/testdata/golden/tables/maps/RowsTable.h index e3c970741..6c76612df 100644 --- a/testdata/golden/tables/maps/RowsTable.h +++ b/testdata/golden/tables/maps/RowsTable.h @@ -377,7 +377,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -879,13 +879,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1407,7 +1407,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1425,10 +1425,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1437,6 +1437,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1445,54 +1446,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3430,25 +3440,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -5692,7 +5704,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -7291,7 +7303,7 @@ inline bool RowSaveMessageBody( const Ctx & ctx, const TableNumbering & numberin if ( !order_entries.ok ) { return false; } // the sort could not run if ( order_entries.count > 0 ) { - w.put( 34, kTableMessageRefBitsHere ); + w.put( 38, kTableMessageRefBitsHere ); w.put( (uint64_t) order_entries.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_entries.count; i++ ) { @@ -7302,7 +7314,7 @@ inline bool RowSaveMessageBody( const Ctx & ctx, const TableNumbering & numberin } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body @@ -8004,7 +8016,7 @@ inline bool WideRowSaveMessageBody( const Ctx & ctx, const TableNumbering & numb if ( !order_entries.ok ) { return false; } // the sort could not run if ( order_entries.count > 0 ) { - w.put( 34, kTableMessageRefBitsHere ); + w.put( 38, kTableMessageRefBitsHere ); w.put( (uint64_t) order_entries.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_entries.count; i++ ) { @@ -8015,7 +8027,7 @@ inline bool WideRowSaveMessageBody( const Ctx & ctx, const TableNumbering & numb } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body @@ -8352,7 +8364,7 @@ inline bool EdgeRowNamesEntrySaveMessageBody( TableBitWriter & w, const EdgeRowN if ( value.key_length < 0 || value.key_length > 300 ) { return false; } // storage invariant if ( value.key_length > 0 ) { - w.put( 25, kTableMessageRefBitsHere ); + w.put( 28, kTableMessageRefBitsHere ); w.put( (uint64_t) value.key_length, 9 ); w.align(); // a string or a bytes ALIGNS before its bytes w.putbytes( (const uint8_t *) value.key, value.key_length ); @@ -8593,7 +8605,7 @@ inline bool EdgeRowIdsEntrySaveMessageBody( TableBitWriter & w, const EdgeRowIds { if ( value.key != 0 ) { - w.put( 27, kTableMessageRefBitsHere ); + w.put( 30, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.key ), 64 ); } { @@ -9106,7 +9118,7 @@ inline bool EdgeRowSaveMessageBody( const Ctx & ctx, const TableNumbering & numb if ( !order_names.ok ) { return false; } // the sort could not run if ( order_names.count > 0 ) { - w.put( 24, kTableMessageRefBitsHere ); + w.put( 27, kTableMessageRefBitsHere ); w.put( (uint64_t) order_names.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_names.count; i++ ) { @@ -9120,7 +9132,7 @@ inline bool EdgeRowSaveMessageBody( const Ctx & ctx, const TableNumbering & numb if ( !order_ids.ok ) { return false; } // the sort could not run if ( order_ids.count > 0 ) { - w.put( 26, kTableMessageRefBitsHere ); + w.put( 29, kTableMessageRefBitsHere ); w.put( (uint64_t) order_ids.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_ids.count; i++ ) { @@ -9131,7 +9143,7 @@ inline bool EdgeRowSaveMessageBody( const Ctx & ctx, const TableNumbering & numb } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/RunsTable.h b/testdata/golden/tables/maps/RunsTable.h index 2eb12b0c3..1a17d617e 100644 --- a/testdata/golden/tables/maps/RunsTable.h +++ b/testdata/golden/tables/maps/RunsTable.h @@ -377,7 +377,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -879,13 +879,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1407,7 +1407,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1425,10 +1425,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1437,6 +1437,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1445,54 +1446,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3430,25 +3440,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -3753,6 +3765,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4129,6 +4151,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -5677,7 +5704,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6416,13 +6443,13 @@ inline bool RunsSpansEntrySaveMessageBody( TableBitWriter & w, const RunsSpansEn { if ( value.key != 0 ) { - w.put( 13, kTableMessageRefBitsHere ); + w.put( 14, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.key ), 16 ); } if ( value.value_count < 0 || value.value_count > 4 ) { return false; } // storage invariant if ( value.value_count > 0 ) { - w.put( 36, kTableMessageRefBitsHere ); + w.put( 40, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.value_count ) - 0, 3 ); for ( int32_t i = 0; i < value.value_count; i++ ) { @@ -6807,7 +6834,7 @@ inline bool RunsSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi if ( !order_spans.ok ) { return false; } // the sort could not run if ( order_spans.count > 0 ) { - w.put( 35, kTableMessageRefBitsHere ); + w.put( 39, kTableMessageRefBitsHere ); w.put( (uint64_t) order_spans.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_spans.count; i++ ) { @@ -6818,7 +6845,7 @@ inline bool RunsSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/SlotsTable.h b/testdata/golden/tables/maps/SlotsTable.h index c2c0bed59..c205575b6 100644 --- a/testdata/golden/tables/maps/SlotsTable.h +++ b/testdata/golden/tables/maps/SlotsTable.h @@ -377,7 +377,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -879,13 +879,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1407,7 +1407,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1425,10 +1425,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1437,6 +1437,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1445,54 +1446,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3430,25 +3440,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -3753,6 +3765,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4129,6 +4151,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -5677,7 +5704,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6037,8 +6064,8 @@ inline bool TableEnumSlot( Slot value, uint64_t & slot ) switch ( value ) { case Slot::None: slot = 0; return true; - case Slot::Alpha: slot = 44; return true; - case Slot::Beta: slot = 45; return true; + case Slot::Alpha: slot = 50; return true; + case Slot::Beta: slot = 51; return true; default: return false; // no variant names this value: no wire identity } } @@ -6549,7 +6576,7 @@ inline bool SlotsSeatsEntrySaveMessageBody( TableBitWriter & w, const SlotsSeats { if ( value.key != 0 ) { - w.put( 11, kTableMessageRefBitsHere ); + w.put( 12, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.key ), 32 ); } { @@ -6562,7 +6589,7 @@ inline bool SlotsSeatsEntrySaveMessageBody( TableBitWriter & w, const SlotsSeats } if ( pairs_value > 0 ) { - w.put( 40, kTableMessageRefBitsHere ); + w.put( 44, kTableMessageRefBitsHere ); w.put( (uint64_t) pairs_value, 2 ); // ASCENDING BY VARIANT ORDINAL, which is slot order. It is // this writer's choice and a reader must not rely on it: every @@ -6980,7 +7007,7 @@ inline bool SlotsSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_seats.ok ) { return false; } // the sort could not run if ( order_seats.count > 0 ) { - w.put( 39, kTableMessageRefBitsHere ); + w.put( 43, kTableMessageRefBitsHere ); w.put( (uint64_t) order_seats.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_seats.count; i++ ) { @@ -6991,7 +7018,7 @@ inline bool SlotsSaveMessageBody( const Ctx & ctx, const TableNumbering & number } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/SpansTable.h b/testdata/golden/tables/maps/SpansTable.h index 580da8759..1014e660d 100644 --- a/testdata/golden/tables/maps/SpansTable.h +++ b/testdata/golden/tables/maps/SpansTable.h @@ -377,7 +377,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -879,13 +879,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1407,7 +1407,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1425,10 +1425,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1437,6 +1437,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1445,54 +1446,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3430,25 +3440,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -3753,6 +3765,16 @@ inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t le } break; } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) default: // every other content is bytes: a string, wide text, an escape, a @@ -4129,6 +4151,11 @@ inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t len } break; } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; default: TableRetainOutRaw( s, s.in + s.at, length ); s.at += length; @@ -5677,7 +5704,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6841,7 +6868,7 @@ inline bool SpansSaveMessageBody( const Ctx & ctx, const TableNumbering & number if ( !order_tracks.ok ) { return false; } // the sort could not run if ( order_tracks.count > 0 ) { - w.put( 41, kTableMessageRefBitsHere ); + w.put( 45, kTableMessageRefBitsHere ); w.put( (uint64_t) order_tracks.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_tracks.count; i++ ) { @@ -6852,7 +6879,7 @@ inline bool SpansSaveMessageBody( const Ctx & ctx, const TableNumbering & number } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/TextTable.h b/testdata/golden/tables/maps/TextTable.h index e81498914..99a0cf947 100644 --- a/testdata/golden/tables/maps/TextTable.h +++ b/testdata/golden/tables/maps/TextTable.h @@ -376,7 +376,7 @@ inline int64_t TableLebBytes( uint64_t v ) // nothing rides. struct TableIds { - static const int32_t kCapacity = 67; + static const int32_t kCapacity = 76; static const int32_t kBuckets = 256; uint64_t ids[ kCapacity ]; @@ -878,13 +878,13 @@ static const int64_t kTableMessageRefBitsHere = 7; // build announces exactly this many entries; a receiver that means to meet // OTHER builds declares more, and an announcement above whatever it declared // is refused as vocabulary_too_large. -static const int64_t kTableMessageEntriesHere = 66; +static const int64_t kTableMessageEntriesHere = 75; // The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A // pointered body names the node table through it, and the node table is the // ROOT body's FIRST field because a pointer index's width is settled by the // node count it carries. -static const uint64_t kTableNodeTableFieldSlot = 48; +static const uint64_t kTableNodeTableFieldSlot = 54; // THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own // layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, @@ -1406,7 +1406,7 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val return -1; } -// THE UNIT'S ANNOUNCEMENT, byte for byte: 66 entries and 786 bytes. It is an +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an // ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under // the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 // over element kind 6, and a trailer of those two reserved ids. @@ -1424,10 +1424,10 @@ inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t val // the projection's sorted record order. The tail is UNCONDITIONAL, so an // ordinary edit only ever grows it at its end and never moves a slot a // generated field header carries as a literal. -static const int64_t kTableAnnounceBytes = 786; +static const int64_t kTableAnnounceBytes = 901; static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { - 0x01, 0x01, 0x09, 0x19, 0x3e, 0x9c, 0x0e, 0x70, 0x2b, 0xae, 0xe4, 0x02, - 0x0e, 0xea, 0x05, 0x06, 0xe7, 0x05, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, @@ -1436,6 +1436,7 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, @@ -1444,54 +1445,63 @@ static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, - 0xff, 0xff, 0x0f, 0x0d, 0xaf, 0x5c, 0xca, 0x21, 0x19, 0xaa, 0x08, 0x1a, - 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, 0x1f, 0x0e, 0x00, 0x03, - 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, 0x70, 0x10, 0x02, 0x0d, - 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, 0x0f, 0x9f, 0x76, 0x48, - 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, 0xaf, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, - 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, 0x19, 0xea, 0x7d, 0x2b, - 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, - 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, 0xe1, 0x13, 0x49, 0x5c, - 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x04, 0x34, - 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, 0xdf, 0x63, 0x11, 0x70, - 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, - 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, 0xdc, 0xd8, 0x6d, 0x0e, - 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, 0xa2, 0x79, 0x44, 0x8e, - 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, - 0xc5, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, - 0xb2, 0x8a, 0xfc, 0x7d, 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, - 0x0d, 0x86, 0x1b, 0x63, 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, - 0xa9, 0x8b, 0x28, 0xb5, 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, - 0x30, 0x30, 0x48, 0x65, 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, - 0x00, 0xbd, 0x0f, 0x47, 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, - 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, - 0x1c, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, - 0x55, 0xf6, 0xf1, 0x33, 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, - 0x0d, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, 0x4f, 0x00, 0x87, 0x94, - 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, 0x63, 0x3e, 0xd6, 0x95, - 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, 0x4e, 0x19, 0x4d, 0xfd, - 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x00, 0xe4, - 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, 0x58, 0xfc, 0xaf, 0xfa, - 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, 0x26, 0xb0, 0x9d, 0x29, - 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, 0x78, 0x1f, 0x00, 0x83, - 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, 0x4a, 0x0d, 0xe3, 0x6f, - 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, 0x6f, - 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, 0xaf, - 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, 0x98, - 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, - 0xa0, 0x00, 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, - 0x4f, 0xb1, 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, - 0xcd, 0x15, 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, - 0x03, 0x00, 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, - 0xb4, 0x05, 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x16, 0xa3, 0x71, 0x35, - 0x4e, 0x96, 0x13, 0xb4, 0x00, 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, }; // TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries @@ -3429,25 +3439,27 @@ inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * r // derived from. An id inside a retained record takes its trailer entry from // the GENERATED table when it is here and from the CALLER's list otherwise, so // no retained id ever enters the generated table and no id is written twice. -static const int32_t kTableRetainKnownIds = 67; +static const int32_t kTableRetainKnownIds = 76; static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { - 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x0a53e00afba279afull, 0x0c2643993e3ece2eull, - 0x11e7ec757c03c70aull, 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, - 0x1c84390d304f4f42ull, 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, - 0x294a5c4913e1ad44ull, 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, - 0x2f2ec0474f1c4fe4ull, 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, - 0x437dfc8ab2566816ull, 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, - 0x610dcbb318a2e4faull, 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, - 0x70551ff29550f15dull, 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x7b024c46e98d3404ull, + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, - 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xbc08b7f228c93506ull, 0xbf82010f6f71eae9ull, - 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, - 0xdcdbddf89c9310a1ull, 0xe1185043515c812bull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, - 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf96b15cd3921d4a6ull, 0xfa903574575fc678ull, - 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, }; inline bool TableRetainNameable( uint64_t id ) @@ -5691,7 +5703,7 @@ namespace mapdemo { // PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is // what everything cooked or blocked is keyed by. A table edit moves this and // never the protocol id; a type edit moves both. -static const uint64_t BuildVersion = 0xe4ae2b700e9c3e19ull; +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; } // namespace mapdemo @@ -6718,7 +6730,7 @@ inline bool TextNamesEntrySaveMessageBody( TableBitWriter & w, const TextNamesEn if ( value.value_length < 0 || value.value_length > 16 ) { return false; } // storage invariant if ( value.value_length > 0 ) { - w.put( 15, kTableMessageRefBitsHere ); + w.put( 16, kTableMessageRefBitsHere ); w.put( (uint64_t) value.value_length, 5 ); w.align(); // a string or a bytes ALIGNS before its bytes w.putbytes( (const uint8_t *) value.value, value.value_length ); @@ -6951,13 +6963,13 @@ inline bool TextWideEntrySaveMessageBody( TableBitWriter & w, const TextWideEntr { if ( value.key != 0 ) { - w.put( 13, kTableMessageRefBitsHere ); + w.put( 14, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.key ), 16 ); } if ( value.value_length < 0 || value.value_length > 6 ) { return false; } // storage invariant if ( value.value_length > 0 ) { - w.put( 14, kTableMessageRefBitsHere ); + w.put( 15, kTableMessageRefBitsHere ); w.put( (uint64_t) value.value_length, 3 ); for ( int32_t i = 0; i < value.value_length; i++ ) { w.put( (uint64_t) (uint16_t) value.value[i], 16 ); } // sixteen bits a unit, no align } @@ -7246,13 +7258,13 @@ inline bool TextBlobsEntrySaveMessageBody( TableBitWriter & w, const TextBlobsEn { if ( value.key != 0 ) { - w.put( 11, kTableMessageRefBitsHere ); + w.put( 12, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.key ), 32 ); } if ( value.value_length < 0 || value.value_length > 10 ) { return false; } // storage invariant if ( value.value_length > 0 ) { - w.put( 12, kTableMessageRefBitsHere ); + w.put( 13, kTableMessageRefBitsHere ); w.put( (uint64_t) value.value_length, 4 ); w.align(); // a string or a bytes ALIGNS before its bytes w.putbytes( (const uint8_t *) value.value, value.value_length ); @@ -7930,7 +7942,7 @@ inline bool TextSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi if ( !order_names.ok ) { return false; } // the sort could not run if ( order_names.count > 0 ) { - w.put( 24, kTableMessageRefBitsHere ); + w.put( 27, kTableMessageRefBitsHere ); w.put( (uint64_t) order_names.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_names.count; i++ ) { @@ -7944,7 +7956,7 @@ inline bool TextSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi if ( !order_wide.ok ) { return false; } // the sort could not run if ( order_wide.count > 0 ) { - w.put( 43, kTableMessageRefBitsHere ); + w.put( 47, kTableMessageRefBitsHere ); w.put( (uint64_t) order_wide.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_wide.count; i++ ) { @@ -7958,7 +7970,7 @@ inline bool TextSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi if ( !order_blobs.ok ) { return false; } // the sort could not run if ( order_blobs.count > 0 ) { - w.put( 18, kTableMessageRefBitsHere ); + w.put( 19, kTableMessageRefBitsHere ); w.put( (uint64_t) order_blobs.count, 32 ); // the count the data decides for ( int32_t i = 0; i < order_blobs.count; i++ ) { @@ -7969,7 +7981,7 @@ inline bool TextSaveMessageBody( const Ctx & ctx, const TableNumbering & numberi } if ( value.after != 0 ) { - w.put( 17, kTableMessageRefBitsHere ); + w.put( 18, kTableMessageRefBitsHere ); w.put( (uint64_t) ( value.after ), 32 ); } w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body diff --git a/testdata/golden/tables/maps/TrailsTable.cpp b/testdata/golden/tables/maps/TrailsTable.cpp new file mode 100644 index 000000000..37ea897b5 --- /dev/null +++ b/testdata/golden/tables/maps/TrailsTable.cpp @@ -0,0 +1,3448 @@ +// Code generated by the schema compiler from Trails.schema. DO NOT EDIT. +// SPDX-License-Identifier: NONE — this generated output is yours, under terms of +// your choice. See the LICENSE exception in the schema compiler; the compiler is +// AGPL-3.0, its output is not. +// package mapdemo — the TABLE wire's text form (docs/SPEC-TABLES.md §16). +// Compile this file to use FromJson / ToJson; a project that +// never reads or writes a text does not compile it and pays nothing. + +#include "TrailsTable.h" + +#include // the text form: number formatting +#include // the text form: exact number conversion +#include // the text form: the runtime's decimal point + +// The guard is not vestigial. Several mapdemo Table.cpp files may be +// concatenated into ONE translation unit — a unity build — and without it +// each would redefine the walk. It is also why the walk's functions may be +// weak (vague linkage) across separate objects: ODR requires their +// definitions to be token-identical, and the generic-walk gate is what +// proves that, byte for byte, across every generated .cpp. +#ifndef MAPDEMO_SCHEMA_TABLE_JSON +#define MAPDEMO_SCHEMA_TABLE_JSON + +namespace mapdemo { + +// ---- the pointer adapters (docs/SPEC-TABLES.md §16.7) ---- +// +// The walk below is ONE walk, byte-identical in every generated .cpp, and a +// pointer is the one kind it cannot walk alone: reading one needs the +// builder's arena and writing one needs a region's deref, and neither exists +// in a unit that declares no pointer. So the walk calls these three and does +// not define them. A unit with no pointer defines them as stubs no field ever +// reaches; a pointered unit defines them in the graph half that follows the +// walk. + +struct TableJsonIn; +struct TableJsonOut; + +// a pointer field's object, or the `&node` reference standing in for it, into +// the slot; the cursor is on the opening brace +inline bool TableJsonReadPointer( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); +// the node a pointer slot names, in place — or as `&node` when it is shared +inline bool TableJsonWritePointer( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// the FIRST key of an object the walk is skipping begins with `&`: the cursor is +// on its value. A dropped definition still takes its label (§16.7); a fixed reader +// skips the value whole, as it skips everything else it does not place. +inline bool TableJsonSkippedAmpersand( TableJsonIn & in, const char * key, int32_t depth ); + +// ---- the map and list adapters (docs/SPEC-TABLES.md §2.8, §2.9, §16) ---- +// +// A MAP and an UNBOUNDED ARRAY are the other constructs the walk cannot walk +// alone: their arrays live behind a TableMap or a TableList this +// walk has no name for, reading one needs the builder's arena, and neither +// exists in a unit that declares neither construct. Same shape as the +// pointer's three: declared here, defined after the walk by whichever half +// the unit carries. Both are OUT-OF-LINE ARRAYS to the descriptors (§8.1): +// array_bound = 0 is the tell, and the type name says which of the two. + +// a map field: an out-of-line array whose type name spells the map +inline bool TableJsonIsMap( const TableFieldInfo * f ); +// the map as a plain JSON object keyed by the KEY, in ASCENDING key order +inline bool TableJsonWriteMap( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// that object back into the slot, in whatever order the text gives it +inline bool TableJsonReadMap( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); +// an unbounded array: the other out-of-line array +inline bool TableJsonIsList( const TableFieldInfo * f ); +// the list as a JSON array, in INDEX order +inline bool TableJsonWriteList( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ); +// that array back into the slot, every element the text carries +inline bool TableJsonReadList( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ); + +// ---- json walk: begin ---- +// +// The TEXT form (docs/SPEC-TABLES.md §16): one table, one text, one walk over the +// reflection descriptors (§8). Reading fills ONE caller-owned instance and +// allocates nothing beyond it; writing targets a caller buffer with the +// wire's measure/write symmetry. Everything AROUND this — which file goes +// with which instance, what key an instance is filed under, how instances +// link into a root table's collections — is a packer's opinion and stays +// with the tool that holds it. +// +// The dialect: trailing commas are accepted on read (the authoring files +// this exists for carry them) and never written; comments are not JSON and +// are refused; unknown keys are skipped and counted; a duplicate key is +// last-wins and counted; a key present with the wrong JSON type is skipped +// and counted, never coerced. + +static const int32_t kTableJsonMaxDepth = 128; + +// A key longer than this cannot name a field, so it is skipped as unknown. +static const int32_t kTableJsonMaxKey = 256; + +// The longest numeric token the walk will convert. Anything longer is a +// value no field can hold and counts as a kind mismatch. +static const int32_t kTableJsonMaxNumber = 512; + +// The decimal point the C runtime is CURRENTLY using. Number conversion is +// the one locale-sensitive corner of the grammar — JSON's point is always +// '.', the runtime's is whatever the program set — so every number crosses +// this one character on the way out and on the way back in. Nothing else in +// the walk consults the locale. +inline char TableJsonDecimalPoint() +{ + const struct lconv * conv = localeconv(); + if ( conv != NULL && conv->decimal_point != NULL && conv->decimal_point[0] != 0 ) + { + return conv->decimal_point[0]; + } + return '.'; +} + +// ---- storage access: the descriptors give an offset and a width, and the +// ---- storage is the HOST's, so every load and store goes through a width +// ---- switch rather than a memcpy into the low bytes of a wider word + +// finite: not a NaN, not an infinity. Written without — the walk's +// runtime surface stays the handful of functions it already names. +// A vocabulary entry the descriptor could not spell. The generated name +// functions answer "???" for a value outside the declared set, and that is +// not a name — writing it would put a spelling in the text that the reader +// then counts as unknown, turning a refusal into a silent loss. +inline bool TableJsonNamed( const char * name ) +{ + return name != NULL && strcmp( name, "???" ) != 0; +} + +inline bool TableJsonFinite( double v ) +{ + return v == v && v <= 1.7976931348623157e308 && v >= -1.7976931348623157e308; +} + +inline uint64_t TableJsonGetRaw( const void * storage, uint32_t width ) +{ + switch ( width ) + { + case 1: { uint8_t v = 0; memcpy( &v, storage, 1 ); return v; } + case 2: { uint16_t v = 0; memcpy( &v, storage, 2 ); return v; } + case 4: { uint32_t v = 0; memcpy( &v, storage, 4 ); return v; } + case 8: { uint64_t v = 0; memcpy( &v, storage, 8 ); return v; } + } + return 0; +} + +inline void TableJsonSetRaw( void * storage, uint32_t width, uint64_t value ) +{ + switch ( width ) + { + case 1: { uint8_t v = (uint8_t) value; memcpy( storage, &v, 1 ); break; } + case 2: { uint16_t v = (uint16_t) value; memcpy( storage, &v, 2 ); break; } + case 4: { uint32_t v = (uint32_t) value; memcpy( storage, &v, 4 ); break; } + case 8: { uint64_t v = value; memcpy( storage, &v, 8 ); break; } + } +} + +inline int64_t TableJsonGetSigned( const void * storage, uint32_t width ) +{ + uint64_t raw = TableJsonGetRaw( storage, width ); + if ( width < 8 ) + { + uint64_t sign = uint64_t( 1 ) << ( width * 8 - 1 ); + if ( ( raw & sign ) != 0 ) + { + raw |= ~( ( sign << 1 ) - 1 ); + } + } + return (int64_t) raw; +} + +// ---- the WIDE kinds (docs/SPEC-TABLES.md §3, §16.2) ---- +// +// The 128-bit integers and the fixed-point family convert EXACTLY, over two +// 64-bit lanes: a 128-bit integer is a decimal integer, a fixed value a +// decimal in WHOLE UNITS (1.0, -0.25, 3.0000152587890625) and nothing +// on either path passes through a double. Nothing here needs a 128-bit type +// either, which is what keeps this walk one text for every unit. +struct TableJsonWide +{ + uint64_t lo; + uint64_t hi; +}; + +inline bool TableJsonKindWide( uint8_t kind ) { return kind >= 18 && kind <= 29; } +inline bool TableJsonKindWideSigned( uint8_t kind ) { return kind == 18 || ( kind >= 20 && kind <= 24 ); } +inline bool TableJsonKindFixed( uint8_t kind ) { return kind >= 20 && kind <= 29; } + +inline bool TableJsonWideZero( TableJsonWide v ) { return v.lo == 0 && v.hi == 0; } +inline bool TableJsonWideNegative( TableJsonWide v ) { return ( v.hi >> 63 ) != 0; } + +inline int TableJsonWideCompare( TableJsonWide a, TableJsonWide b, bool is_signed ) +{ + if ( is_signed && TableJsonWideNegative( a ) != TableJsonWideNegative( b ) ) { return TableJsonWideNegative( a ) ? -1 : 1; } + if ( a.hi != b.hi ) { return a.hi < b.hi ? -1 : 1; } + if ( a.lo != b.lo ) { return a.lo < b.lo ? -1 : 1; } + return 0; +} + +inline TableJsonWide TableJsonWideShl( TableJsonWide v, int n ) +{ + TableJsonWide r = { 0, 0 }; + if ( n <= 0 ) { return v; } + if ( n >= 128 ) { return r; } + if ( n >= 64 ) { r.hi = v.lo << ( n - 64 ); return r; } + r.hi = ( v.hi << n ) | ( v.lo >> ( 64 - n ) ); + r.lo = v.lo << n; + return r; +} + +inline TableJsonWide TableJsonWideShr( TableJsonWide v, int n ) +{ + TableJsonWide r = { 0, 0 }; + if ( n <= 0 ) { return v; } + if ( n >= 128 ) { return r; } + if ( n >= 64 ) { r.lo = v.hi >> ( n - 64 ); return r; } + r.lo = ( v.lo >> n ) | ( v.hi << ( 64 - n ) ); + r.hi = v.hi >> n; + return r; +} + +inline TableJsonWide TableJsonWideNeg( TableJsonWide v ) +{ + TableJsonWide r; + r.lo = ~v.lo + 1; + r.hi = ~v.hi + ( r.lo == 0 ? 1 : 0 ); + return r; +} + +// v = v * m + a; the return is the carry out of 128 bits +inline uint32_t TableJsonWideMulAdd( TableJsonWide * v, uint32_t m, uint32_t a ) +{ + uint64_t limb[4] = { v->lo & 0xffffffffull, v->lo >> 32, v->hi & 0xffffffffull, v->hi >> 32 }; + uint64_t carry = a; + for ( int i = 0; i < 4; i++ ) + { + uint64_t p = limb[i] * m + carry; + limb[i] = p & 0xffffffffull; + carry = p >> 32; + } + v->lo = limb[0] | ( limb[1] << 32 ); + v->hi = limb[2] | ( limb[3] << 32 ); + return (uint32_t) carry; +} + +// v = v / d; the return is the remainder +inline uint32_t TableJsonWideDiv( TableJsonWide * v, uint32_t d ) +{ + uint64_t limb[4] = { v->lo & 0xffffffffull, v->lo >> 32, v->hi & 0xffffffffull, v->hi >> 32 }; + uint64_t rem = 0; + for ( int i = 3; i >= 0; i-- ) + { + uint64_t cur = ( rem << 32 ) | limb[i]; + limb[i] = cur / d; + rem = cur % d; + } + v->lo = limb[0] | ( limb[1] << 32 ); + v->hi = limb[2] | ( limb[3] << 32 ); + return (uint32_t) rem; +} + +// The storage of a wide kind, as lanes. A sixteen-byte storage is serialize's +// pair — native __int128 in the host's byte order, or the emulated struct with +// its low lane first — so the lanes are read in the host's order; a narrower +// storage is one lane, sign-extended for a signed kind. +inline TableJsonWide TableJsonWideLoad( const void * storage, uint32_t width, bool is_signed ) +{ + TableJsonWide v = { 0, 0 }; + if ( width == 16 ) + { + uint64_t half[2]; + memcpy( half, storage, 16 ); + uint16_t probe = 1; + bool little = *(const uint8_t *) &probe == 1; + v.lo = little ? half[0] : half[1]; + v.hi = little ? half[1] : half[0]; + return v; + } + v.lo = is_signed ? (uint64_t) TableJsonGetSigned( storage, width ) : TableJsonGetRaw( storage, width ); + v.hi = ( is_signed && ( v.lo >> 63 ) != 0 ) ? ~uint64_t( 0 ) : 0; + return v; +} + +inline void TableJsonWideStore( void * storage, uint32_t width, TableJsonWide v ) +{ + if ( width == 16 ) + { + uint16_t probe = 1; + bool little = *(const uint8_t *) &probe == 1; + uint64_t half[2]; + half[0] = little ? v.lo : v.hi; + half[1] = little ? v.hi : v.lo; + memcpy( storage, half, 16 ); + return; + } + TableJsonSetRaw( storage, width, v.lo ); +} + +// a counted field's companion: a string's length, a bytes' length, a counted +// array's count. Bounded by the declared extent on the way out, so a storage +// invariant a caller broke cannot walk off the end of the array. +inline int32_t TableJsonCount( const void * base, const TableFieldInfo * f ) +{ + if ( !f->counted ) + { + return f->array_bound; + } + int32_t count = 0; + memcpy( &count, (const uint8_t *) base + f->count_offset, sizeof( count ) ); + if ( count < 0 ) { count = 0; } + if ( count > f->array_bound ) { count = f->array_bound; } + return count; +} + +inline void TableJsonSetCount( void * base, const TableFieldInfo * f, int32_t count ) +{ + if ( f->counted ) + { + memcpy( (uint8_t *) base + f->count_offset, &count, sizeof( count ) ); + } +} + +// ---- what a field's kind expects to see in the text ---- +// +// One classifier, consulted by both directions, so a reader and a writer can +// never disagree about a kind's JSON form. 'o' object, 'a' array, 's' +// string, 'n' number, 'b' boolean. +// +// A vocabulary field is spelled by NAME: an enum is one name, a flags mask +// is the array of the names of its set bits. The two are told apart by the +// id column — an enum variant rides under a wire id, a flags BIT never does +// (docs/SPEC-TABLES.md §4), so a name function with no id function is flags. +// +// bytes(N) is the one kind whose element kind does not decide its form: it +// shares u8 with a plain array of u8, and rides as base64. The schema type +// name settles it, and "bytes" is a keyword no declaration can claim. +inline bool TableJsonIsBytes( const TableFieldInfo * f ) +{ + return f->is_array && f->kind == 6 && strcmp( f->type_name, "bytes" ) == 0; +} + +// An ENUM-KEYED array (docs/SPEC-TABLES.md §2.4): its JSON form is an OBJECT +// keyed by variant name, not a positional array, because that is what the +// storage is — one slot per variant, addressed by the variant. +inline bool TableJsonIsKeyed( const TableFieldInfo * f ) +{ + return f->key_name != NULL; +} + +// THE KEY A STORAGE SLOT HOLDS (§2.4, §8): the storage shifts left, so slot i +// holds the key i + 1 and nothing is stored for None. This is the ONE place +// the walker spells the shift. +inline uint64_t TableJsonKeyedSlotKey( int64_t slot ) +{ + return (uint64_t) ( slot + 1 ); +} + +// A slot whose key names a variant of the keying enum. Every slot in +// [0, array_bound) does, unless the enum carries max-headroom variants outside +// a table closure, where a reserved value names nothing and its key id is 0 — +// the reserved id no declared name can fold to (§5). +inline bool TableJsonKeyedSlotValid( const TableFieldInfo * f, int64_t slot ) +{ + return f->key_id( TableJsonKeyedSlotKey( slot ) ) != 0; +} + +inline bool TableJsonIsFlags( const TableFieldInfo * f ) +{ + return f->enum_name != NULL && f->variant_id == NULL; +} + +inline bool TableJsonIsEnum( const TableFieldInfo * f ) +{ + return f->variant_id != NULL && f->arms == NULL; +} + +inline char TableJsonShape( const TableFieldInfo * f ) +{ + if ( TableJsonIsMap( f ) ) return 'o'; // a MAP: an object keyed by the KEY (§2.8) + if ( f->kind == 12 ) return 's'; // string + if ( f->kind == 33 ) return 's'; // wstring: the same text, transcoded (§16.2) + if ( TableJsonIsBytes( f ) ) return 's'; // bytes: base64 + if ( TableJsonIsKeyed( f ) ) return 'o'; // an object keyed by variant NAME + if ( f->is_array ) return 'a'; + if ( f->arms != NULL ) return 'o'; // union: an object with ONE key + if ( f->kind == 13 ) return 'o'; // nested table or type + if ( f->kind == 17 ) return f->table != NULL ? 'o' : 's'; // a pointer: the pointee's object in place, or null (§16.7); a byte buffer's string (§2.5) + if ( TableJsonIsEnum( f ) ) return 's'; + if ( TableJsonIsFlags( f ) ) return 'a'; + if ( f->kind == 1 ) return 'b'; + return 'n'; +} + +// the ELEMENT shape of an array field — the same classifier one level down +inline char TableJsonElementShape( const TableFieldInfo * f ) +{ + if ( f->arms != NULL ) return 'o'; // an element of an array of unions: one key, the arm (§2.6) + if ( f->kind == 13 ) return 'o'; + if ( TableJsonIsEnum( f ) ) return 's'; + if ( TableJsonIsFlags( f ) ) return 'a'; + if ( f->kind == 1 ) return 'b'; + return 'n'; +} + +// A guarded group rides only when its guard reads true — the wire's own +// elision (§4), carried into the text so a text and a wire written from one +// instance say the same thing. The guard is spelled as its branch condition +// over bool fields of the SAME type ("at_rest", "!at_rest", +// "active && has_target"), so evaluating it is a walk of the same +// descriptor. Nothing is inferred in the other direction: reading places +// every key it can name, and the guard is a plain bool key (§16.2). +inline bool TableJsonGuardHolds( const void * base, const TableTypeInfo * info, const char * guard ) +{ + const char * p = guard; + for ( ;; ) + { + while ( *p == ' ' || *p == '&' ) { p++; } + if ( *p == 0 ) { return true; } + bool want = true; + if ( *p == '!' ) { want = false; p++; } + const char * start = p; + while ( *p != 0 && *p != ' ' && *p != '&' ) { p++; } + size_t length = (size_t) ( p - start ); + bool value = false; + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + const TableFieldInfo * f = &info->fields[i]; + if ( strlen( f->name ) == length && strncmp( f->name, start, length ) == 0 ) + { + value = TableJsonGetRaw( (const uint8_t *) base + f->offset, f->elem_size ) != 0; + break; + } + } + if ( value != want ) { return false; } + } +} + +// ---- writing ---- + +// The writer sink MEASURES when the buffer is NULL and WRITES when it is +// not, over one code path — so measure and write agree byte for byte, the +// wire's invariant (§9) carried across. +struct TableJsonOut +{ + char * buffer; + int64_t capacity; + int64_t offset; + bool overflow; + void * graph; // the pointered write's identity map (§16.7); NULL for a fixed table + + void raw( const char * data, int64_t count ) + { + if ( buffer != NULL ) + { + if ( offset + count > capacity ) { overflow = true; return; } + memcpy( buffer + offset, data, (size_t) count ); + } + offset += count; + } + void put( char c ) { raw( &c, 1 ); } + void text( const char * s ) { raw( s, (int64_t) strlen( s ) ); } + void line( int32_t depth ) + { + put( '\n' ); + for ( int32_t i = 0; i < depth; i++ ) { raw( " ", 2 ); } + } +}; + +inline const char * TableJsonBase64Alphabet() +{ + return "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; +} + +inline void TableJsonWriteBase64( TableJsonOut & out, const uint8_t * data, int32_t length ) +{ + const char * alphabet = TableJsonBase64Alphabet(); + out.put( '"' ); + int32_t i = 0; + for ( ; i + 3 <= length; i += 3 ) + { + uint32_t triple = ( uint32_t( data[i] ) << 16 ) | ( uint32_t( data[i+1] ) << 8 ) | uint32_t( data[i+2] ); + char quad[4] = { alphabet[ ( triple >> 18 ) & 0x3f ], alphabet[ ( triple >> 12 ) & 0x3f ], + alphabet[ ( triple >> 6 ) & 0x3f ], alphabet[ triple & 0x3f ] }; + out.raw( quad, 4 ); + } + if ( i < length ) + { + int32_t left = length - i; + uint32_t triple = uint32_t( data[i] ) << 16; + if ( left == 2 ) { triple |= uint32_t( data[i+1] ) << 8; } + char quad[4] = { alphabet[ ( triple >> 18 ) & 0x3f ], alphabet[ ( triple >> 12 ) & 0x3f ], '=', '=' }; + if ( left == 2 ) { quad[2] = alphabet[ ( triple >> 6 ) & 0x3f ]; } + out.raw( quad, 4 ); + } + out.put( '"' ); +} + +// One UTF-8 sequence at s, or -1 when the bytes there are not one. Rejects +// the lot: a stray continuation, an overlong form, a surrogate half, and +// anything past U+10FFFF. +inline int32_t TableJsonUtf8( const char * s, int32_t remaining, int32_t * width ) +{ + unsigned char lead = (unsigned char) s[0]; + int32_t want = 0; + int32_t code = 0; + if ( lead < 0x80 ) { *width = 1; return lead; } + else if ( lead >= 0xc2 && lead <= 0xdf ) { want = 2; code = lead & 0x1f; } + else if ( lead >= 0xe0 && lead <= 0xef ) { want = 3; code = lead & 0x0f; } + else if ( lead >= 0xf0 && lead <= 0xf4 ) { want = 4; code = lead & 0x07; } + else { return -1; } + if ( remaining < want ) { return -1; } + for ( int32_t i = 1; i < want; i++ ) + { + unsigned char next = (unsigned char) s[i]; + if ( ( next & 0xc0 ) != 0x80 ) { return -1; } + code = ( code << 6 ) | ( next & 0x3f ); + } + if ( want == 3 && code < 0x800 ) { return -1; } // overlong + if ( want == 4 && code < 0x10000 ) { return -1; } // overlong + if ( code >= 0xd800 && code <= 0xdfff ) { return -1; } // a surrogate half + if ( code > 0x10ffff ) { return -1; } + *width = want; + return code; +} + +// The inverse: one code point encoded as UTF-8 into unit, its length +// answered. Both text kinds' writers reach it, and so does the escape +// grammar's U+FFFD replacement. +inline int32_t TableJsonEncodeUtf8( uint32_t code, char * unit ) +{ + if ( code < 0x80 ) { unit[0] = (char) code; return 1; } + if ( code < 0x800 ) + { + unit[0] = (char) ( 0xc0 | ( code >> 6 ) ); + unit[1] = (char) ( 0x80 | ( code & 0x3f ) ); + return 2; + } + if ( code < 0x10000 ) + { + unit[0] = (char) ( 0xe0 | ( code >> 12 ) ); + unit[1] = (char) ( 0x80 | ( ( code >> 6 ) & 0x3f ) ); + unit[2] = (char) ( 0x80 | ( code & 0x3f ) ); + return 3; + } + unit[0] = (char) ( 0xf0 | ( code >> 18 ) ); + unit[1] = (char) ( 0x80 | ( ( code >> 12 ) & 0x3f ) ); + unit[2] = (char) ( 0x80 | ( ( code >> 6 ) & 0x3f ) ); + unit[3] = (char) ( 0x80 | ( code & 0x3f ) ); + return 4; +} + +// A JSON text MUST be valid UTF-8 (RFC 8259 §8.1). The read path is +// byte-transparent — the wire imposes no encoding (§3) and a string may hold +// anything — so the WRITER is where that obligation is met: a byte that is +// not part of a well-formed sequence is written as U+FFFD, one per bad byte, +// and never raw. A text this walk writes is therefore readable by any +// conforming parser, which a raw byte would not be. The cost is stated +// plainly: for a string holding invalid UTF-8, the round trip is NOT +// byte-identical, because the alternative is emitting a text that is not +// JSON. +inline void TableJsonWriteString( TableJsonOut & out, const char * s, int32_t length ) +{ + static const char hex[] = "0123456789abcdef"; + out.put( '"' ); + for ( int32_t i = 0; i < length; i++ ) + { + unsigned char c = (unsigned char) s[i]; + switch ( c ) + { + case '"': out.raw( "\\\"", 2 ); break; + case '\\': out.raw( "\\\\", 2 ); break; + case '\b': out.raw( "\\b", 2 ); break; + case '\f': out.raw( "\\f", 2 ); break; + case '\n': out.raw( "\\n", 2 ); break; + case '\r': out.raw( "\\r", 2 ); break; + case '\t': out.raw( "\\t", 2 ); break; + default: + if ( c < 0x20 ) + { + char escape[6] = { '\\', 'u', '0', '0', hex[ c >> 4 ], hex[ c & 0xf ] }; + out.raw( escape, 6 ); + } + else if ( c < 0x80 ) + { + out.put( (char) c ); + } + else + { + int32_t width = 0; + if ( TableJsonUtf8( s + i, length - i, &width ) < 0 ) + { + out.raw( "\xef\xbf\xbd", 3 ); // U+FFFD, one per bad byte + } + else + { + out.raw( s + i, width ); + i += width - 1; + } + } + break; + } + } + out.put( '"' ); +} + +// A WIDE field's text: the code units transcoded back to UTF-8 (§16.2). A +// SURROGATE PAIR is one code point; an UNPAIRED SURROGATE is not a code point +// at all, encodes to nothing, and writes one U+FFFD per ill-formed unit; a +// ZERO UNIT is U+0000, which JSON has an escape for, and writes \u0000 +// (§16.3). No wire can put either into storage (§3), so both answer for +// storage a PROGRAM built. +inline void TableJsonWriteWString( TableJsonOut & out, const char16_t * s, int32_t length ) +{ + static const char hex[] = "0123456789abcdef"; + out.put( '"' ); + for ( int32_t i = 0; i < length; i++ ) + { + uint32_t code = (uint32_t) (uint16_t) s[i]; + if ( code >= 0xd800 && code <= 0xdbff && i + 1 < length ) + { + const uint32_t low = (uint32_t) (uint16_t) s[i + 1]; + if ( low >= 0xdc00 && low <= 0xdfff ) + { + code = 0x10000 + ( ( code - 0xd800 ) << 10 ) + ( low - 0xdc00 ); + i++; + } + } + if ( code >= 0xd800 && code <= 0xdfff ) { code = 0xfffd; } // an unpaired surrogate + switch ( code ) + { + case '"': out.raw( "\\\"", 2 ); continue; + case '\\': out.raw( "\\\\", 2 ); continue; + case '\b': out.raw( "\\b", 2 ); continue; + case '\f': out.raw( "\\f", 2 ); continue; + case '\n': out.raw( "\\n", 2 ); continue; + case '\r': out.raw( "\\r", 2 ); continue; + case '\t': out.raw( "\\t", 2 ); continue; + default: break; + } + if ( code < 0x20 ) + { + char escape[6] = { '\\', 'u', '0', '0', hex[ code >> 4 ], hex[ code & 0xf ] }; + out.raw( escape, 6 ); + continue; + } + char encoded[4]; + const int32_t encoded_length = TableJsonEncodeUtf8( code, encoded ); + out.raw( encoded, encoded_length ); + } + out.put( '"' ); +} + +inline void TableJsonWriteUnsigned( TableJsonOut & out, uint64_t value ) +{ + char digits[24]; + int32_t n = 0; + do + { + digits[n++] = (char) ( '0' + (int) ( value % 10 ) ); + value /= 10; + } while ( value != 0 ); + char text[24]; + for ( int32_t i = 0; i < n; i++ ) { text[i] = digits[n - 1 - i]; } + out.raw( text, n ); +} + +inline void TableJsonWriteSigned( TableJsonOut & out, int64_t value ) +{ + if ( value < 0 ) + { + out.put( '-' ); + TableJsonWriteUnsigned( out, uint64_t( 0 ) - (uint64_t) value ); + return; + } + TableJsonWriteUnsigned( out, (uint64_t) value ); +} + +// A wide kind writes its raw storage as §16.2's text: a 128-bit integer as a +// decimal integer; a fixed value in WHOLE UNITS as the shortest exact decimal +// with at least one fractional digit (1.0, -0.25), the spelling the schema text +// gives a fixed default. The fraction terminates because a dyadic fraction has +// a finite decimal expansion — at most F digits. +inline void TableJsonWriteWide( TableJsonOut & out, const void * storage, const TableFieldInfo * f ) +{ + bool is_signed = TableJsonKindWideSigned( f->kind ); + TableJsonWide v = TableJsonWideLoad( storage, f->elem_size, is_signed ); + if ( is_signed && TableJsonWideNegative( v ) ) + { + out.put( '-' ); + v = TableJsonWideNeg( v ); + } + int frac = f->frac_bits; + TableJsonWide whole = TableJsonWideShr( v, frac ); + char digits[40]; + int32_t n = 0; + do + { + digits[n++] = (char) ( '0' + (int) TableJsonWideDiv( &whole, 10 ) ); + } while ( !TableJsonWideZero( whole ) ); + char text[40]; + for ( int32_t i = 0; i < n; i++ ) { text[i] = digits[n - 1 - i]; } + out.raw( text, n ); + if ( !TableJsonKindFixed( f->kind ) ) { return; } + out.put( '.' ); + // the fraction bits alone: v with everything at and above bit F cleared + TableJsonWide fraction = v; + if ( frac < 64 ) { fraction.hi = 0; fraction.lo &= ( uint64_t( 1 ) << frac ) - 1; } + else { fraction.hi &= ( uint64_t( 1 ) << ( frac - 64 ) ) - 1; } + if ( frac == 0 ) { fraction.lo = 0; } + if ( TableJsonWideZero( fraction ) ) + { + out.put( '0' ); + return; + } + while ( !TableJsonWideZero( fraction ) ) + { + // ×10: the digit is what lands at and above bit F, including the + // carry out of 128 bits when F leaves no room for it below + uint32_t carry = TableJsonWideMulAdd( &fraction, 10, 0 ); + uint64_t digit = TableJsonWideShr( fraction, frac ).lo; + if ( frac > 64 ) { digit |= uint64_t( carry ) << ( 128 - frac ); } + out.put( (char) ( '0' + (int) digit ) ); + if ( frac < 64 ) { fraction.hi = 0; fraction.lo &= ( uint64_t( 1 ) << frac ) - 1; } + else { fraction.hi &= ( uint64_t( 1 ) << ( frac - 64 ) ) - 1; } + } +} + +// A float writes at the SHORTEST precision that reads back as the same value +// at the field's own width, so a round trip is exact and a text stays +// readable. Non-finite values have no JSON spelling at all, and the writer +// REFUSES rather than losing one silently — the same rule measure and save +// already apply to an enum value no variant names (§5). +inline bool TableJsonWriteFloat( TableJsonOut & out, double value, bool single ) +{ + if ( !TableJsonFinite( value ) ) { return false; } + char text[64]; + int low = single ? 6 : 15; + int high = single ? 9 : 17; + int length = 0; + for ( int digits = low; ; digits++ ) + { + length = snprintf( text, sizeof( text ), "%.*g", digits, value ); + if ( length <= 0 || length >= (int) sizeof( text ) ) { return false; } + if ( digits >= high ) { break; } + // the round-trip check runs BEFORE the decimal point is normalised: + // the token still carries whatever point snprintf just produced + if ( single ) + { + if ( (double) strtof( text, NULL ) == value ) { break; } + } + else + { + if ( strtod( text, NULL ) == value ) { break; } + } + } + char point = TableJsonDecimalPoint(); + if ( point != '.' ) + { + for ( int i = 0; i < length; i++ ) + { + if ( text[i] == point ) { text[i] = '.'; } + } + } + out.raw( text, length ); + return true; +} + +inline bool TableJsonWriteValue( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth ); +// a UNION ARM that names no declaration writes through the field walk one key +// down (docs/SPEC-TABLES.md §2.6, §16.2), which is defined below +inline bool TableJsonWriteField( TableJsonOut & out, const void * base, const TableFieldInfo * f, int32_t depth ); + +// one scalar, at one storage address: a nested object, a union, a +// vocabulary, or a number +inline bool TableJsonWriteScalar( TableJsonOut & out, const void * storage, const TableFieldInfo * f, int32_t depth ) +{ + if ( f->arms != NULL ) + { + // a union is an object with ONE key, the arm's name; None is {} + const TableUnionInfo * arms = f->arms(); + uint64_t tag = TableJsonGetRaw( (const uint8_t *) storage + arms->tag_offset, arms->tag_size ); + if ( tag == 0 ) + { + out.raw( "{}", 2 ); + return true; + } + if ( (int64_t) tag > f->enum_max ) + { + return false; // a tag no arm names, exactly as measure refuses it + } + const char * arm = f->enum_name( tag ); + // and refuse on the NAME, not merely on the bound: §16.2 says a value + // no variant NAMES is refused, so the check is the name. Writing + // whatever came back would emit "???", a spelling the reader counts + // as unknown — a silent round-trip loss in place of a refusal. + if ( !TableJsonNamed( arm ) ) { return false; } + out.put( '{' ); + out.line( depth + 1 ); + TableJsonWriteString( out, arm, (int32_t) strlen( arm ) ); + out.raw( ": ", 2 ); + // THE ARM'S VALUE TAKES THE ARM'S OWN ROW (§16.2): an arm that names + // no declaration carries the FIELD descriptor a field of its type + // would carry, offsets taken inside the union storage (§2.6), so the + // value walks through the field writer one key down. + if ( arms->arms[tag].field != NULL ) + { + if ( !TableJsonWriteField( out, storage, arms->arms[tag].field, depth + 1 ) ) + { + return false; + } + } + else if ( arms->arms[tag].table == NULL ) + { + out.raw( "null", 4 ); // a payload-free arm: the name selects it (§2.6) + } + else if ( !TableJsonWriteValue( out, (const uint8_t *) storage + arms->arms[tag].offset, arms->arms[tag].table, depth + 1 ) ) + { + return false; + } + out.line( depth ); + out.put( '}' ); + return true; + } + if ( f->kind == 13 ) + { + return TableJsonWriteValue( out, storage, f->table, depth ); + } + if ( TableJsonIsEnum( f ) ) + { + uint64_t value = TableJsonGetRaw( storage, f->elem_size ); + // a value no variant names has no text spelling, exactly as it has no + // wire identity: the writer REFUSES rather than writing None over it, + // the rule measure and save already apply (docs/SPEC-TABLES.md §5) + if ( (int64_t) value > f->enum_max ) { return false; } + if ( value != 0 && f->variant_id( value ) == 0 ) { return false; } + const char * name = f->enum_name( value ); + if ( !TableJsonNamed( name ) ) { return false; } + TableJsonWriteString( out, name, (int32_t) strlen( name ) ); + return true; + } + if ( TableJsonIsFlags( f ) ) + { + uint64_t bits = TableJsonGetRaw( storage, f->elem_size ); + if ( bits == 0 ) + { + out.raw( "[]", 2 ); + return true; + } + out.put( '[' ); + bool first = true; + for ( int64_t bit = 0; bit < 64; bit++ ) + { + if ( ( bits & ( uint64_t( 1 ) << bit ) ) == 0 ) { continue; } + if ( bit > f->enum_max ) + { + return false; // a bit no variant names has no text spelling + } + const char * name = f->enum_name( (uint64_t) bit ); + if ( !TableJsonNamed( name ) ) { return false; } + if ( !first ) { out.put( ',' ); } + first = false; + out.line( depth + 1 ); + TableJsonWriteString( out, name, (int32_t) strlen( name ) ); + } + out.line( depth ); + out.put( ']' ); + return true; + } + switch ( f->kind ) + { + case 1: + out.text( TableJsonGetRaw( storage, f->elem_size ) != 0 ? "true" : "false" ); + return true; + case 10: + { + float v = 0.0f; + memcpy( &v, storage, sizeof( v ) ); + return TableJsonWriteFloat( out, (double) v, true ); + } + case 11: + { + double v = 0.0; + memcpy( &v, storage, sizeof( v ) ); + return TableJsonWriteFloat( out, v, false ); + } + case 2: case 3: case 4: case 5: + TableJsonWriteSigned( out, TableJsonGetSigned( storage, f->elem_size ) ); + return true; + default: + if ( TableJsonKindWide( f->kind ) ) + { + TableJsonWriteWide( out, storage, f ); + return true; + } + TableJsonWriteUnsigned( out, TableJsonGetRaw( storage, f->elem_size ) ); + return true; + } +} + +inline bool TableJsonWriteField( TableJsonOut & out, const void * base, const TableFieldInfo * f, int32_t depth ) +{ + const uint8_t * storage = (const uint8_t *) base + f->offset; + if ( TableJsonIsMap( f ) ) + { + return TableJsonWriteMap( out, (const void *) storage, f, depth ); + } + if ( TableJsonIsList( f ) ) + { + return TableJsonWriteList( out, (const void *) storage, f, depth ); + } + if ( f->kind == 17 && !f->is_array ) + { + return TableJsonWritePointer( out, storage, f, depth ); + } + if ( f->kind == 17 ) + { + // an ARRAY OF POINTERS (§2.1): the pointer row per element — the + // pointee's object in place, null, or `&node` for a shared one (§16.7) + int32_t count = TableJsonCount( base, f ); + if ( count == 0 ) { out.raw( "[]", 2 ); return true; } + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + if ( !TableJsonWritePointer( out, storage + (int64_t) i * f->elem_size, f, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( ']' ); + return true; + } + if ( f->kind == 12 ) + { + TableJsonWriteString( out, (const char *) storage, TableJsonCount( base, f ) ); + return true; + } + if ( f->kind == 33 ) + { + TableJsonWriteWString( out, (const char16_t *) (const void *) storage, TableJsonCount( base, f ) ); + return true; + } + if ( TableJsonIsBytes( f ) ) + { + TableJsonWriteBase64( out, storage, TableJsonCount( base, f ) ); + return true; + } + if ( TableJsonIsKeyed( f ) ) + { + // one entry per SLOT, keyed by the variant that owns it, so inserting + // a variant next season moves nothing in the text either. Slot i holds + // the key i + 1: nothing is stored for None, so nothing is written for it. + out.put( '{' ); + bool first = true; + for ( int64_t slot = 0; slot < f->array_bound; slot++ ) + { + if ( !TableJsonKeyedSlotValid( f, slot ) ) { continue; } + if ( !first ) { out.put( ',' ); } + first = false; + out.line( depth + 1 ); + const char * key = f->key_name( TableJsonKeyedSlotKey( slot ) ); + TableJsonWriteString( out, key, (int32_t) strlen( key ) ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteScalar( out, storage + slot * f->elem_size, f, depth + 1 ) ) + { + return false; + } + } + if ( first ) { out.raw( "}", 1 ); return true; } + out.line( depth ); + out.put( '}' ); + return true; + } + if ( f->is_array ) + { + int32_t count = TableJsonCount( base, f ); + if ( count == 0 ) + { + out.raw( "[]", 2 ); + return true; + } + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + if ( !TableJsonWriteScalar( out, storage + (int64_t) i * f->elem_size, f, depth + 1 ) ) + { + return false; + } + } + out.line( depth ); + out.put( ']' ); + return true; + } + return TableJsonWriteScalar( out, storage, f, depth ); +} + +// One instance's fields, in DECLARATION ORDER, defaults included — a text is +// for people and tools, and a text that elides is a text a reader has to know +// the schema to complete. `any` says whether the object is already open on +// entry — a shared node's `&node` opens it before the fields (§16.7) — and +// whether it is open on return. +inline bool TableJsonWriteFields( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth, bool & any ) +{ + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + const TableFieldInfo * f = &info->fields[i]; + if ( f->guard[0] != 0 && !TableJsonGuardHolds( base, info, f->guard ) ) { continue; } + // an ABSENT optional writes no key: presence of the key IS the + // presence (§16.2), so an absent field is an absent key and nothing + // else would read back as absent + if ( f->optional && + TableJsonGetRaw( (const uint8_t *) base + f->present_offset, 1 ) == 0 ) + { + continue; + } + if ( !any ) { out.put( '{' ); } + else { out.put( ',' ); } + any = true; + out.line( depth + 1 ); + TableJsonWriteString( out, f->json, (int32_t) strlen( f->json ) ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteField( out, base, f, depth + 1 ) ) { return false; } + } + return true; +} + +// One instance as one object. The writer carries the reader's depth cap +// (§16.2): a pointer chain nests as deep as it is long (§16.7), and a text the +// writer produced past the cap would be a text the reader refuses. +inline bool TableJsonWriteValue( TableJsonOut & out, const void * base, const TableTypeInfo * info, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { return false; } + bool any = false; + if ( !TableJsonWriteFields( out, base, info, depth, any ) ) { return false; } + if ( !any ) + { + out.raw( "{}", 2 ); + return true; + } + out.line( depth ); + out.put( '}' ); + return true; +} + +// ---- reading ---- + +struct TableJsonIn +{ + const char * text; + int64_t size; + int64_t pos; + TableReport * report; + bool bad; // the text is not JSON: the walk stops and keeps what it placed + void * graph; // the pointered read's builder and label map (§16.7); NULL for a fixed table +}; + +inline void TableJsonSpace( TableJsonIn & in ) +{ + while ( in.pos < in.size ) + { + char c = in.text[in.pos]; + if ( c == ' ' || c == '\t' || c == '\n' || c == '\r' ) { in.pos++; continue; } + // COMMENTS ARE ACCEPTED ON READ AND NEVER WRITTEN (docs/SPEC-TABLES.md + // §16.2): a line comment runs to the end of the line or of the input, + // a block comment to its closing delimiter, which does not nest, and + // an UNCLOSED block comment is malformed on the terms an unclosed + // string is. Both are legal wherever whitespace is; a lone slash is not JSON. + if ( c == '/' && in.pos + 1 < in.size && in.text[in.pos + 1] == '/' ) + { + in.pos += 2; + while ( in.pos < in.size && in.text[in.pos] != '\n' ) { in.pos++; } + continue; + } + if ( c == '/' && in.pos + 1 < in.size && in.text[in.pos + 1] == '*' ) + { + int64_t at = in.pos + 2; + while ( at + 1 < in.size && !( in.text[at] == '*' && in.text[at + 1] == '/' ) ) { at++; } + if ( at + 1 >= in.size ) { in.bad = true; in.pos = in.size; return; } + in.pos = at + 2; + continue; + } + if ( c == '/' ) { in.bad = true; } + return; + } +} + +inline char TableJsonPeek( TableJsonIn & in ) +{ + TableJsonSpace( in ); + return in.pos < in.size ? in.text[in.pos] : 0; +} + +// the shape of the value sitting at the cursor, without consuming it +inline char TableJsonValueShape( TableJsonIn & in ) +{ + char c = TableJsonPeek( in ); + switch ( c ) + { + case '{': return 'o'; + case '[': return 'a'; + case '"': return 's'; + case 't': case 'f': return 'b'; + case 'n': return 'z'; + case 0: return 0; + default: return 'n'; + } +} + +inline bool TableJsonLiteral( TableJsonIn & in, const char * word ) +{ + int64_t length = (int64_t) strlen( word ); + if ( in.pos + length > in.size || memcmp( in.text + in.pos, word, (size_t) length ) != 0 ) + { + in.bad = true; + return false; + } + in.pos += length; + return true; +} + +// one \uXXXX escape body; -1 when the four hex digits are not there +inline int TableJsonHex4( TableJsonIn & in ) +{ + if ( in.pos + 4 > in.size ) { return -1; } + int value = 0; + for ( int i = 0; i < 4; i++ ) + { + char c = in.text[in.pos + i]; + int digit; + if ( c >= '0' && c <= '9' ) { digit = c - '0'; } + else if ( c >= 'a' && c <= 'f' ) { digit = c - 'a' + 10; } + else if ( c >= 'A' && c <= 'F' ) { digit = c - 'A' + 10; } + else { return -1; } + value = ( value << 4 ) | digit; + } + in.pos += 4; + return value; +} + + +// One STRING BODY CHARACTER at the cursor, encoded into unit as UTF-8 and +// its length answered: an escape's code point, or a UTF-8 sequence read +// whole. It is ONE grammar serving both text kinds — the narrow scan places +// these bytes and the wide scan converts them back to code units — so the +// escape table, the lone-surrogate rule and the U+FFFD replacement are stated +// once. false means the text is not JSON and in.bad says so; a returned +// length of 0 means the closing quote was consumed and the string is done. +inline bool TableJsonScanUnit( TableJsonIn & in, char * unit, int32_t * unit_length_out ) +{ + int32_t unit_length = 0; + *unit_length_out = 0; + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos]; + if ( c == '"' ) { in.pos++; return true; } + if ( c == '\\' ) + { + in.pos++; + if ( in.pos >= in.size ) { in.bad = true; return false; } + char escape = in.text[in.pos++]; + switch ( escape ) + { + case '"': unit[0] = '"'; unit_length = 1; break; + case '\\': unit[0] = '\\'; unit_length = 1; break; + case '/': unit[0] = '/'; unit_length = 1; break; + case 'b': unit[0] = '\b'; unit_length = 1; break; + case 'f': unit[0] = '\f'; unit_length = 1; break; + case 'n': unit[0] = '\n'; unit_length = 1; break; + case 'r': unit[0] = '\r'; unit_length = 1; break; + case 't': unit[0] = '\t'; unit_length = 1; break; + case 'u': + { + int high = TableJsonHex4( in ); + if ( high < 0 ) { in.bad = true; return false; } + uint32_t code = (uint32_t) high; + if ( high >= 0xd800 && high <= 0xdbff && in.pos + 2 <= in.size && + in.text[in.pos] == '\\' && in.text[in.pos + 1] == 'u' ) + { + int64_t mark = in.pos; + in.pos += 2; + int low = TableJsonHex4( in ); + if ( low >= 0xdc00 && low <= 0xdfff ) + { + code = 0x10000 + ( ( (uint32_t) high - 0xd800 ) << 10 ) + ( (uint32_t) low - 0xdc00 ); + } + else + { + in.pos = mark; // a lone lead surrogate rides as itself + } + } + // a surrogate half that never found its partner has no + // UTF-8 encoding: encoding it anyway would manufacture + // CESU-8 — invalid UTF-8 — out of input that was valid + // JSON, so it reads as the replacement character + if ( code >= 0xd800 && code <= 0xdfff ) { code = 0xfffd; } + unit_length = TableJsonEncodeUtf8( code, unit ); + break; + } + default: in.bad = true; return false; + } + } + else if ( (unsigned char) c < 0x20 ) + { + in.bad = true; // a raw control character is not a JSON string body + return false; + } + else + { + // a UTF-8 sequence read WHOLE, so the clamp below can only land + // between code points. Only bytes that ACTUALLY look like + // continuations are taken: the wire imposes no encoding (§3), so + // a string may legitimately hold a stray lead byte, and one at + // the end of a text must not swallow the closing quote. + unsigned char lead = (unsigned char) c; + int32_t want = 1; + if ( ( lead & 0xe0 ) == 0xc0 ) { want = 2; } + else if ( ( lead & 0xf0 ) == 0xe0 ) { want = 3; } + else if ( ( lead & 0xf8 ) == 0xf0 ) { want = 4; } + unit[0] = c; + in.pos++; + unit_length = 1; + while ( unit_length < want && in.pos < in.size && + ( (unsigned char) in.text[in.pos] & 0xc0 ) == 0x80 ) + { + unit[unit_length++] = in.text[in.pos++]; + } + // A SEQUENCE THAT IS NOT A CODE POINT READS AS U+FFFD, which is + // §16.3's rule at the point the defect ENTERS rather than at the + // point it leaves: a lone surrogate escape already reads that way + // above, RFC 8259 requires a JSON text to be valid UTF-8, and a + // kind 12 payload is well-formed UTF-8 (§3), so storage the text + // form built has to be storage the wire can carry (§5). + if ( !TableUtf8Valid( (const uint8_t *) unit, unit_length ) ) + { + unit_length = TableJsonEncodeUtf8( 0xfffd, unit ); + } + } + } + *unit_length_out = unit_length; + return true; +} + +// Scan one JSON string into a caller buffer. Bytes are appended ONE CODE +// POINT AT A TIME — an escape's encoding, or a UTF-8 sequence read whole — +// so a string longer than the field is clamped AT A CODE POINT BOUNDARY and +// never cut through a multi-byte character. Clamping is counted, never +// fatal, exactly as it is on the wire (§4). A NULL destination scans past a +// string without keeping it. +// +// A CALLER THAT TAKES clamped_out OWNS THE COUNTER. The value paths leave it +// NULL, and the clamp is a value's clamp, counted here. A MAP KEY takes it, +// because a key never clamps: a key this buffer could not hold whole is not a +// shorter key, and its entry drops instead (§2.8). +inline bool TableJsonScanString( TableJsonIn & in, char * out, int32_t capacity, int32_t * length, + bool * clamped_out = NULL ) +{ + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + int32_t placed = 0; + bool clamped = false; + for ( ;; ) + { + char unit[4]; + int32_t unit_length = 0; + if ( !TableJsonScanUnit( in, unit, &unit_length ) ) { return false; } + if ( unit_length == 0 ) { break; } + if ( out == NULL ) + { + placed += unit_length; // measured and not kept: a byte buffer's read sizes its node this way (§2.5) + } + else if ( !clamped && placed + unit_length <= capacity ) + { + memcpy( out + placed, unit, (size_t) unit_length ); + placed += unit_length; + } + else + { + // A CLAMP IS A PREFIX. Once one code point does not fit, the scan + // stops placing: a later SHORTER code point sliding into the room + // the long one left would store a string the input never spelled, + // and one clamped count cannot tell the two apart. + clamped = true; + } + } + if ( clamped_out != NULL ) { *clamped_out = clamped; } + else if ( clamped ) { in.report->clamped++; } + if ( length != NULL ) { *length = placed; } + return true; +} + +// One UTF-8 sequence back to its CODE POINT, over bytes TableJsonScanUnit +// produced and therefore already well formed. It is the inverse of +// TableJsonEncodeUtf8 and nothing more. +inline uint32_t TableJsonDecodeUtf8( const char * unit, int32_t unit_length ) +{ + const unsigned char lead = (unsigned char) unit[0]; + if ( unit_length == 1 ) { return lead; } + uint32_t code = lead & ( unit_length == 2 ? 0x1fu : ( unit_length == 3 ? 0x0fu : 0x07u ) ); + for ( int32_t i = 1; i < unit_length; i++ ) + { + code = ( code << 6 ) | (uint32_t) ( (unsigned char) unit[i] & 0x3f ); + } + return code; +} + +// Scan one JSON string into a caller buffer of UTF-16 CODE UNITS: the wstring +// row of §16.2, the text TRANSCODED at the boundary. It shares +// TableJsonScanUnit with the narrow scan, so the escape grammar and the +// lone-surrogate rule are one grammar, and appends ONE CODE POINT AT A TIME — +// one unit below U+10000 and a surrogate PAIR above it. A string longer than +// the field is therefore clamped at N code units with a pair never split, and +// a high surrogate left without its low half is dropped with it, which is the +// same sentence the wire's clamp takes (§3). Clamping is counted, never fatal. +inline bool TableJsonScanWString( TableJsonIn & in, char16_t * out, int32_t capacity, int32_t * length ) +{ + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + int32_t placed = 0; + bool clamped = false; + for ( ;; ) + { + char unit[4]; + int32_t unit_length = 0; + if ( !TableJsonScanUnit( in, unit, &unit_length ) ) { return false; } + if ( unit_length == 0 ) { break; } + const uint32_t code = TableJsonDecodeUtf8( unit, unit_length ); + char16_t units[2]; + int32_t units_length = 1; + if ( code < 0x10000 ) + { + units[0] = (char16_t) code; + } + else + { + const uint32_t rest = code - 0x10000; + units[0] = (char16_t) ( 0xd800 + ( rest >> 10 ) ); + units[1] = (char16_t) ( 0xdc00 + ( rest & 0x3ff ) ); + units_length = 2; + } + if ( out == NULL ) + { + placed += units_length; + } + else if ( !clamped && placed + units_length <= capacity ) + { + for ( int32_t i = 0; i < units_length; i++ ) { out[placed + i] = units[i]; } + placed += units_length; + } + else + { + // A CLAMP IS A PREFIX, the narrow scan's own rule: once one code + // point does not fit, the scan stops placing. A pair is placed + // whole or not at all, so no clamp can leave an unpaired + // surrogate in storage. + clamped = true; + } + } + if ( clamped ) { in.report->clamped++; } + if ( length != NULL ) { *length = placed; } + return true; +} + + +// the numeric token at the cursor, copied out whole; false = not a number +// Scan one number, to JSON's OWN grammar (RFC 8259 §6) and not to a run of +// number-ish characters: +// +// number = [ "-" ] int [ frac ] [ exp ] +// int = "0" / ( digit1-9 *digit ) +// frac = "." 1*digit +// exp = ( "e" / "E" ) [ "-" / "+" ] 1*digit +// +// Scanning the production is what makes a typo in an authoring file a +// DIAGNOSTIC rather than a value: "1-2" scans as 1 and leaves "-2" where the +// object expects a comma, so the text is malformed — which is what §16.2 +// already promises. A permissive scan would hand "1-2" to a digit loop and +// report a clamp, and a config pipeline would never hear about it. Leading +// "+", leading zeros, ".5" and "3." are not JSON either. +inline bool TableJsonWalkNumber( TableJsonIn & in, bool * integral ) +{ + TableJsonSpace( in ); + bool whole = true; + if ( in.pos < in.size && in.text[in.pos] == '-' ) { in.pos++; } + // int: a lone zero, or a non-zero digit and any digits after it + if ( in.pos >= in.size ) { return false; } + if ( in.text[in.pos] == '0' ) + { + in.pos++; + } + else if ( in.text[in.pos] >= '1' && in.text[in.pos] <= '9' ) + { + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + } + else + { + return false; + } + // frac + if ( in.pos < in.size && in.text[in.pos] == '.' ) + { + in.pos++; + if ( in.pos >= in.size || in.text[in.pos] < '0' || in.text[in.pos] > '9' ) { return false; } + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + whole = false; + } + // exp + if ( in.pos < in.size && ( in.text[in.pos] == 'e' || in.text[in.pos] == 'E' ) ) + { + in.pos++; + if ( in.pos < in.size && ( in.text[in.pos] == '-' || in.text[in.pos] == '+' ) ) { in.pos++; } + if ( in.pos >= in.size || in.text[in.pos] < '0' || in.text[in.pos] > '9' ) { return false; } + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) { in.pos++; } + whole = false; + } + *integral = whole; + return true; +} + +// the same production, with the token kept for conversion +inline bool TableJsonScanNumber( TableJsonIn & in, char * token, int32_t capacity, int32_t * length, bool * integral ) +{ + TableJsonSpace( in ); + int64_t start = in.pos; + if ( !TableJsonWalkNumber( in, integral ) ) { return false; } + int64_t count = in.pos - start; + if ( count <= 0 || count >= capacity ) { return false; } + memcpy( token, in.text + start, (size_t) count ); + token[count] = 0; + *length = (int32_t) count; + return true; +} + +// the token's exact double, through the runtime's own converter — which +// speaks the LOCALE's decimal point, so the token crosses back over that +// character on its way in +inline double TableJsonTokenDouble( const char * token, int32_t length, bool single ) +{ + char work[kTableJsonMaxNumber]; + memcpy( work, token, (size_t) length ); + work[length] = 0; + char point = TableJsonDecimalPoint(); + if ( point != '.' ) + { + for ( int32_t i = 0; i < length; i++ ) + { + if ( work[i] == '.' ) { work[i] = point; } + } + } + if ( single ) { return (double) strtof( work, NULL ); } + return strtod( work, NULL ); +} + +// ---- ONE CHECKED NUMERIC INTERPRETATION (docs/SPEC-TABLES.md §16.2) ---- +// +// JSON HAS ONE NUMBER TYPE, so every integer target reads a token the same way +// and the VALUE decides rather than the spelling: 2, 2.0 and 1e3 are the +// integers 2, 2 and 1000. What comes out of a token is a SIGN, a MAGNITUDE and +// a STATUS, and nothing on the way is cast through a type that cannot hold what +// it is handed. A uint64 magnitude past INT64_MAX is a magnitude and never a +// negative, and a double is consulted only for a spelling the digit path cannot +// read exactly. +// +// TWO POLICIES SIT ON TOP OF THE ONE VALUE and neither reinterprets the token: +// an ordinary FIELD clamps to its domain and counts, and a MAP KEY rejects the +// whole entry, because a key is an identity and a clamped one is two entries +// merged. That difference is the only difference between them. +// +// THE KEY READS ITS TOKEN EXACTLY AND A FIELD READS IT THROUGH THE DOUBLE, and +// that is the one place the two interpretations part. A key is an IDENTITY, so +// two spellings a 53-bit mantissa cannot tell apart are two keys and the key +// path carries TableJsonInterpretExact below. A field's value is a quantity +// under a clamp, and its interpretation is the one the C, Go and Rust ports +// read the same texts with, so it lives here and reads as they read. +struct TableJsonInteger +{ + uint64_t magnitude; // |value|, exact for every integral token 64 bits hold + bool negative; + bool fractional; // a genuinely fractional VALUE: the wrong shape for an integer + bool saturated; // a magnitude past what 64 bits hold, held at that edge + bool finite; // false: no integer target holds it at all +}; + +// THE FIELD'S INTERPRETATION: the token, parsed digit by digit so no width and +// no locale can move it, and through the runtime's converter only where the +// spelling carries a fraction or an exponent +inline TableJsonInteger TableJsonInterpret( const char * token, int32_t length, bool integral ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = false; + out.fractional = false; + out.saturated = false; + out.finite = true; + if ( integral ) + { + int32_t i = 0; + if ( i < length && token[i] == '-' ) // WalkNumber refuses a leading plus + { + out.negative = true; + i++; + } + for ( ; i < length; i++ ) + { + const uint64_t digit = (uint64_t) ( token[i] - '0' ); + if ( out.magnitude > ( UINT64_MAX - digit ) / 10 ) + { + out.magnitude = UINT64_MAX; + out.saturated = true; + break; + } + out.magnitude = out.magnitude * 10 + digit; + } + if ( out.magnitude == 0 ) { out.negative = false; } // -0 IS zero + return out; + } + const double d = TableJsonTokenDouble( token, length, false ); + if ( !TableJsonFinite( d ) ) { out.finite = false; return out; } + out.negative = d < 0; + const double whole = out.negative ? -d : d; + // THE DOMAIN IS ESTABLISHED BEFORE THE CAST: a magnitude past what sixty-four + // bits hold is answered here, so no value ever reaches a conversion that is + // undefined for it + if ( whole >= 18446744073709551616.0 ) + { + out.magnitude = UINT64_MAX; + out.saturated = true; + return out; + } + const uint64_t truncated = (uint64_t) whole; + if ( (double) truncated != whole ) { out.fractional = true; return out; } + out.magnitude = truncated; + if ( out.magnitude == 0 ) { out.negative = false; } + return out; +} + +// THE DECIMAL BAND a token is answered in without arithmetic: 10^20 is above +// UINT64_MAX whatever the digits are, so a point past it saturates and a token +// spelling 1e999999999 costs nothing to refuse. +const int64_t kTableJsonDecimalBand = 20; + +// THE MAP KEY'S INTERPRETATION, and no other path's: the token's own digits, +// read where they stand, so no 53-bit mantissa decides the identity of a 64-bit +// key. The int and frac runs are one digit string with the point after "point" +// of them, and the exponent moves the point rather than the digits, which is +// the normalization the wide kinds already use over one 64-bit lane. A zero +// fraction is the integer the token spells at every magnitude the kind holds, +// so 9007199254740993.0 is that key rather than the one a double rounds it to. +// No exact reader has an infinity, so finite is true here always. +inline TableJsonInteger TableJsonInterpretExact( const char * token, int32_t length ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = false; + out.fractional = false; + out.saturated = false; + out.finite = true; + int32_t i = 0; + if ( i < length && token[i] == '-' ) { out.negative = true; i++; } // WalkNumber refuses a leading plus + const char * int_digits = token + i; + int32_t int_len = 0; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { int_len++; i++; } + const char * frac_digits = token + i; + int32_t frac_len = 0; + if ( i < length && token[i] == '.' ) + { + i++; + frac_digits = token + i; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { frac_len++; i++; } + } + int64_t exp = 0; + if ( i < length && ( token[i] == 'e' || token[i] == 'E' ) ) + { + i++; + bool exp_negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { exp_negative = token[i] == '-'; i++; } + while ( i < length && token[i] >= '0' && token[i] <= '9' ) + { + if ( exp < 100000 ) { exp = exp * 10 + ( token[i] - '0' ); } + i++; + } + if ( exp_negative ) { exp = -exp; } + } + // leading and trailing zeros stripped, so the last digit kept is significant + int32_t start = 0, end = int_len + frac_len; + int64_t point = (int64_t) int_len + exp; + while ( start < end && ( start < int_len ? int_digits[start] : frac_digits[start - int_len] ) == '0' ) { start++; point--; } + while ( end > start && ( end - 1 < int_len ? int_digits[end - 1] : frac_digits[end - 1 - int_len] ) == '0' ) { end--; } + const int64_t digits = end - start; + if ( digits == 0 ) { out.negative = false; return out; } // the value is zero, and -0 IS zero + if ( point < digits ) { out.fractional = true; return out; } // a significant digit below the point + if ( point > kTableJsonDecimalBand ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + for ( int32_t k = start; k < end; k++ ) + { + const uint64_t digit = (uint64_t) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ); + if ( out.magnitude > ( UINT64_MAX - digit ) / 10 ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + out.magnitude = out.magnitude * 10 + digit; + } + for ( int64_t k = digits; k < point; k++ ) // the point's own zeros, which no digit spells + { + if ( out.magnitude > UINT64_MAX / 10 ) { out.magnitude = UINT64_MAX; out.saturated = true; return out; } + out.magnitude *= 10; + } + return out; +} + +// a declared range bound as the same value. A bound is inside the field's own +// domain by construction, so nothing here saturates. +inline TableJsonInteger TableJsonIntegerOf( double bound ) +{ + TableJsonInteger out; + out.magnitude = 0; + out.negative = bound < 0; + out.fractional = false; + out.saturated = false; + out.finite = true; + const double whole = out.negative ? -bound : bound; + out.magnitude = whole >= 18446744073709551616.0 ? UINT64_MAX : (uint64_t) whole; + if ( out.magnitude == 0 ) { out.negative = false; } + return out; +} + +// THE TARGET DOMAIN, established before the value reaches storage: the bytes of +// storage, signed or not. Answers what the target holds and whether the domain +// MOVED it. An unsigned magnitude above INT64_MAX rides out as its bit pattern, +// which is the storage's own image of it and not a negative number. +inline int64_t TableJsonIntegerInDomain( const TableJsonInteger & number, bool is_signed, int32_t bytes, bool & moved ) +{ + moved = false; + uint64_t magnitude = number.magnitude; + if ( is_signed ) + { + const uint64_t high = bytes >= 8 ? (uint64_t) INT64_MAX : ( ( uint64_t( 1 ) << ( bytes * 8 - 1 ) ) - 1 ); + if ( number.negative ) + { + const uint64_t low = high + 1; // the floor's magnitude + if ( magnitude > low ) { magnitude = low; moved = true; } + return (int64_t) ( ~magnitude + 1 ); // two's complement, INT64_MIN included + } + if ( magnitude > high ) { magnitude = high; moved = true; } + return (int64_t) magnitude; + } + // A NEGATIVE TOKEN IN AN UNSIGNED FIELD CLAMPS TO ZERO, and -0 is zero, + // which is why the sign is dropped at a zero magnitude above + if ( number.negative ) { moved = true; return 0; } + const uint64_t high = bytes >= 8 ? UINT64_MAX : ( ( uint64_t( 1 ) << ( bytes * 8 ) ) - 1 ); + if ( magnitude > high ) { magnitude = high; moved = true; } + return (int64_t) magnitude; +} + +// A number token into a wide kind's raw storage (docs/SPEC-TABLES.md §16.2). A +// 128-bit integer takes any token whose VALUE is integral; a fixed field any +// token whose value is EXACTLY representable in its Q I.F — a finer fraction +// is the wrong shape for the field, counted as a kind mismatch and never +// rounded, the rule SPEC.md §4.6 gives a fixed default. A magnitude past 128 +// bits saturates and counts as a clamp, as an int64 field saturates at +// INT64_MAX; the declared range clamps after it, on the RAW scale, as it does +// for every bounded scalar. +// +// The token is normalized to its digits with the decimal point after "point" +// of them. An integer part past 40 digits is above 2^128 whatever the digits +// are, and a value below 10^-40 is finer than 2^-127, the finest fraction any +// F can spell — so outside that band the answer is known without the +// arithmetic, and a token spelling 1e999999999 costs nothing to refuse. +inline bool TableJsonReadWide( TableJsonIn & in, const char * token, int32_t length, void * storage, const TableFieldInfo * f ) +{ + bool is_signed = TableJsonKindWideSigned( f->kind ); + int frac = f->frac_bits; + int32_t i = 0; + bool negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { negative = token[i] == '-'; i++; } + const char * int_digits = token + i; + int32_t int_len = 0; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { int_len++; i++; } + const char * frac_digits = token + i; + int32_t frac_len = 0; + if ( i < length && token[i] == '.' ) + { + i++; + frac_digits = token + i; + while ( i < length && token[i] >= '0' && token[i] <= '9' ) { frac_len++; i++; } + } + int64_t exp = 0; + if ( i < length && ( token[i] == 'e' || token[i] == 'E' ) ) + { + i++; + bool exp_negative = false; + if ( i < length && ( token[i] == '-' || token[i] == '+' ) ) { exp_negative = token[i] == '-'; i++; } + while ( i < length && token[i] >= '0' && token[i] <= '9' ) + { + if ( exp < 100000 ) { exp = exp * 10 + ( token[i] - '0' ); } + i++; + } + if ( exp_negative ) { exp = -exp; } + } + // the digits, with the point after "point" of them; leading and trailing + // zeros stripped. digit( k ) reads the k-th of the int and frac runs. + int32_t start = 0, end = int_len + frac_len; + int64_t point = int_len + exp; + while ( start < end && ( start < int_len ? int_digits[start] : frac_digits[start - int_len] ) == '0' ) { start++; point--; } + while ( end > start && ( end - 1 < int_len ? int_digits[end - 1] : frac_digits[end - 1 - int_len] ) == '0' ) { end--; } + + TableJsonWide raw = { 0, 0 }; + bool saturated = false; + TableJsonWide signed_max = { ~uint64_t( 0 ), ~uint64_t( 0 ) >> 1 }; + TableJsonWide signed_min = { 0, uint64_t( 1 ) << 63 }; + TableJsonWide unsigned_max = { ~uint64_t( 0 ), ~uint64_t( 0 ) }; + if ( start == end ) + { + // zero, and -0 IS zero + } + else if ( point > 40 ) + { + saturated = true; + if ( !negative ) { raw = is_signed ? signed_max : unsigned_max; } + else if ( is_signed ) { raw = signed_min; } + } + else if ( point < -40 ) + { + in.report->kind_mismatch++; // finer than any F can spell + return true; + } + else + { + // the fraction FIRST, so an inexact value is the wrong shape whatever + // its magnitude: its digits, with the zeros a negative point puts in + // front, doubled F times; each doubling's carry is the next bit, and + // the value is exact iff nothing is left after the last one + char fd[kTableJsonMaxNumber + 48]; + int32_t fn = 0; + for ( int64_t z = point; z < 0; z++ ) { fd[fn++] = 0; } + for ( int32_t k = (int32_t) ( point > 0 ? point : 0 ) + start; k < end; k++ ) + { + fd[fn++] = (char) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ); + } + TableJsonWide fraction = { 0, 0 }; + for ( int b = 0; b < frac; b++ ) + { + int carry = 0; + for ( int32_t k = fn - 1; k >= 0; k-- ) + { + int d = fd[k] * 2 + carry; + fd[k] = (char) ( d % 10 ); + carry = d / 10; + } + fraction = TableJsonWideShl( fraction, 1 ); + fraction.lo |= (uint64_t) carry; + } + for ( int32_t k = 0; k < fn; k++ ) + { + if ( fd[k] != 0 ) + { + in.report->kind_mismatch++; + return true; + } + } + // then the whole part, saturating past 128 bits + TableJsonWide whole = { 0, 0 }; + for ( int64_t k = start; k < start + point && !saturated; k++ ) + { + uint32_t digit = k < end ? (uint32_t) ( ( k < int_len ? int_digits[k] : frac_digits[k - int_len] ) - '0' ) : 0; + if ( TableJsonWideMulAdd( &whole, 10, digit ) != 0 ) { saturated = true; } + } + if ( !saturated && frac > 0 && !TableJsonWideZero( TableJsonWideShr( whole, 128 - frac ) ) ) { saturated = true; } + if ( !saturated ) + { + raw = TableJsonWideShl( whole, frac ); + raw.lo |= fraction.lo; + raw.hi |= fraction.hi; + } + if ( is_signed ) + { + if ( !saturated && !negative && TableJsonWideNegative( raw ) ) { saturated = true; } + if ( !saturated && negative && TableJsonWideCompare( raw, signed_min, false ) > 0 ) { saturated = true; } + if ( saturated ) { raw = negative ? signed_min : signed_max; } + else if ( negative ) { raw = TableJsonWideNeg( raw ); } + } + else + { + if ( saturated ) { raw = unsigned_max; } + if ( negative && !TableJsonWideZero( raw ) ) { raw.lo = 0; raw.hi = 0; saturated = true; } + } + } + if ( saturated ) { in.report->clamped++; } + if ( f->wide != NULL ) + { + TableJsonWide lo = { f->wide->lo[0], f->wide->lo[1] }; + TableJsonWide hi = { f->wide->hi[0], f->wide->hi[1] }; + if ( TableJsonWideCompare( raw, lo, is_signed ) < 0 ) { raw = lo; in.report->clamped++; } + else if ( TableJsonWideCompare( raw, hi, is_signed ) > 0 ) { raw = hi; in.report->clamped++; } + } + TableJsonWideStore( storage, f->elem_size, raw ); + return true; +} + +inline bool TableJsonSkipValue( TableJsonIn & in, int32_t depth ); + +inline bool TableJsonSkipContainer( TableJsonIn & in, char close, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; // the opening bracket + bool first = true; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == close ) { in.pos++; return true; } + if ( c == 0 ) { in.bad = true; return false; } + if ( close == '}' ) + { + // the key is kept, because a skipped OBJECT may still be a + // pointer's: an `&node` opening it names a node the storage could + // not hold, and the numbering has to survive the drop (§16.7). + // Anywhere but first, the prefix is the reserved key out of place + // — in a pointered unit; a fixed unit skips the value whole. + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + if ( key[0] == '&' && in.graph != NULL ) + { + if ( !first ) { in.report->malformed = true; in.bad = true; return false; } + if ( !TableJsonSkippedAmpersand( in, key, depth ) ) { return false; } + first = false; + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == close ) { in.pos++; return true; } + in.bad = true; + return false; + } + } + first = false; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == close ) { in.pos++; return true; } + in.bad = true; + return false; + } +} + +inline bool TableJsonSkipValue( TableJsonIn & in, int32_t depth ) +{ + char c = TableJsonPeek( in ); + switch ( c ) + { + case '{': return TableJsonSkipContainer( in, '}', depth ); + case '[': return TableJsonSkipContainer( in, ']', depth ); + case '"': return TableJsonScanString( in, NULL, 0, NULL ); + case 't': return TableJsonLiteral( in, "true" ); + case 'f': return TableJsonLiteral( in, "false" ); + case 'n': return TableJsonLiteral( in, "null" ); + case 0: in.bad = true; return false; + default: + { + // consumed, never converted: skipping needs no buffer, and this + // is the one walk a hostile text drives to the depth cap. It is + // the SAME production the value path scans, so an unknown key + // cannot smuggle past a number a named key would refuse. + bool integral = false; + if ( !TableJsonWalkNumber( in, &integral ) ) { in.bad = true; return false; } + return true; + } + } +} + +inline bool TableJsonReadTable( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth ); +// a UNION ARM that names no declaration reads through the field walk one key +// down (docs/SPEC-TABLES.md §2.6, §16.2), which is defined below +inline bool TableJsonReadField( TableJsonIn & in, void * base, const TableFieldInfo * f, int32_t depth ); + +// place one scalar at one storage address +inline bool TableJsonReadScalar( TableJsonIn & in, void * storage, const TableFieldInfo * f, int32_t depth ) +{ + if ( f->arms != NULL ) + { + // a union is an object with ONE key, the arm's name; {} is None, and + // two keys is a text this walk will not guess at + const TableUnionInfo * arms = f->arms(); + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + TableJsonSetRaw( (uint8_t *) storage + arms->tag_offset, arms->tag_size, 0 ); + if ( TableJsonPeek( in ) == '}' ) { in.pos++; return true; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t tag = 0; + for ( int64_t t = 1; t <= f->enum_max; t++ ) + { + if ( strcmp( f->enum_name( (uint64_t) t ), key ) == 0 ) { tag = t; break; } + } + if ( tag == 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + void * payload = (uint8_t *) storage + arms->arms[tag].offset; + const TableFieldInfo * arm = arms->arms[tag].field; + bool placed = true; + if ( arm != NULL ) + { + // THE ARM'S VALUE TAKES THE ARM'S OWN ROW (§16.2). A value of + // the wrong shape for that row is a KIND MISMATCH: the union + // reads None, the event is counted, and the enclosing object + // continues — the rule a FIELD's value lives under, one key + // down. A pointer arm's null is a null pointer, not a shape + // error, exactly as a pointer field's is (§16.7). + char got = TableJsonValueShape( in ); + if ( arm->kind == 17 && !arm->is_array && got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + memset( payload, 0, (size_t) arms->arms[tag].size ); + } + else if ( got != TableJsonShape( arm ) ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else if ( arm->kind == 17 && !arm->is_array ) + { + // A POINTER ARM'S VALUE IS THE POINTEE IN PLACE, or a + // node reference to one (§16.7) — the read a pointer + // FIELD takes, which is not the scalar walk + memset( payload, 0, (size_t) arms->arms[tag].size ); + if ( !TableJsonReadPointer( in, payload, arm, depth + 1 ) ) { return false; } + } + else + { + // SELECTION ZERO-ESTABLISHES THE ARM (SPEC §5): an arm + // takes no specified default, so zero is the establish + memset( payload, 0, (size_t) arms->arms[tag].size ); + if ( !TableJsonReadField( in, storage, arm, depth + 1 ) ) { return false; } + } + } + else if ( arms->arms[tag].table != NULL ) + { + if ( TableJsonValueShape( in ) != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else + { + arms->arms[tag].table->reset( payload ); + if ( !TableJsonReadTable( in, payload, arms->arms[tag].table, depth + 1 ) ) { return false; } + } + } + else + { + // A PAYLOAD-FREE ARM'S VALUE IS null (§2.6): the arm name + // selects it and there is nothing to place + if ( TableJsonValueShape( in ) != 'z' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed = false; + } + else if ( !TableJsonLiteral( in, "null" ) ) + { + return false; + } + } + if ( placed ) + { + TableJsonSetRaw( (uint8_t *) storage + arms->tag_offset, arms->tag_size, (uint64_t) tag ); + } + } + char c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; c = TableJsonPeek( in ); } + if ( c == '}' ) { in.pos++; return true; } + in.bad = true; // a second key: a one-of with two arms is not a value + return false; + } + if ( f->kind == 13 ) + { + f->table->reset( storage ); + return TableJsonReadTable( in, storage, f->table, depth + 1 ); + } + if ( TableJsonIsEnum( f ) ) + { + char name[kTableJsonMaxKey]; + int32_t name_length = 0; + if ( !TableJsonScanString( in, name, kTableJsonMaxKey - 1, &name_length ) ) { return false; } + name[name_length] = 0; + for ( int64_t v = 0; v <= f->enum_max; v++ ) + { + if ( strcmp( f->enum_name( (uint64_t) v ), name ) == 0 ) + { + TableJsonSetRaw( storage, f->elem_size, (uint64_t) v ); + return true; + } + } + // a name this build cannot name reads as None and counts as unknown, + // exactly as an unknown variant id does on the wire (§4) + TableJsonSetRaw( storage, f->elem_size, 0 ); + in.report->unknown++; + return true; + } + if ( TableJsonIsFlags( f ) ) + { + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + in.pos++; + uint64_t bits = 0; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + if ( c != '"' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + char name[kTableJsonMaxKey]; + int32_t name_length = 0; + if ( !TableJsonScanString( in, name, kTableJsonMaxKey - 1, &name_length ) ) { return false; } + name[name_length] = 0; + bool found = false; + for ( int64_t bit = 0; bit <= f->enum_max; bit++ ) + { + if ( strcmp( f->enum_name( (uint64_t) bit ), name ) == 0 ) + { + bits |= uint64_t( 1 ) << bit; + found = true; + break; + } + } + if ( !found ) { in.report->unknown++; } + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + TableJsonSetRaw( storage, f->elem_size, bits ); + return true; + } + if ( f->kind == 1 ) + { + char c = TableJsonPeek( in ); + if ( c == 't' ) { if ( !TableJsonLiteral( in, "true" ) ) { return false; } TableJsonSetRaw( storage, f->elem_size, 1 ); return true; } + if ( !TableJsonLiteral( in, "false" ) ) { return false; } + TableJsonSetRaw( storage, f->elem_size, 0 ); + return true; + } + char token[kTableJsonMaxNumber]; + int32_t length = 0; + bool integral = false; + if ( !TableJsonScanNumber( in, token, kTableJsonMaxNumber, &length, &integral ) ) + { + in.bad = true; + return false; + } + if ( TableJsonKindWide( f->kind ) ) + { + return TableJsonReadWide( in, token, length, storage, f ); + } + if ( f->kind == 10 || f->kind == 11 ) + { + bool single = f->kind == 10; + double value = TableJsonTokenDouble( token, length, single ); + // A magnitude the field's format cannot hold is the WRONG SHAPE for + // the kind, and it never reaches storage: 1e400 is not a float64 and + // 1e300 is not a float32. Storing the infinity the conversion + // produced would leave an instance this walk called CLEAN that + // ToJsonMeasure then refuses forever (a non-finite float has no JSON + // spelling), and §16.1's one invariant is that a text which reads + // clean writes back. + if ( !TableJsonFinite( value ) ) + { + in.report->kind_mismatch++; + return true; + } + if ( f->has_range ) + { + if ( value < f->range_min ) { value = f->range_min; in.report->clamped++; } + else if ( value > f->range_max ) { value = f->range_max; in.report->clamped++; } + } + if ( single ) + { + float narrow = (float) value; + if ( !TableJsonFinite( (double) narrow ) ) + { + in.report->kind_mismatch++; + return true; + } + memcpy( storage, &narrow, sizeof( narrow ) ); + } + else + { + memcpy( storage, &value, sizeof( value ) ); + } + return true; + } + // AN ORDINARY FIELD'S POLICY over the one interpreted value: it CLAMPS to + // its domain and counts. JSON has one number type, so 2.0 IS the integer 2 + // and 1e3 IS 1000. A library that round-trips numbers through a double + // emits them that way, and this walker's own float writer emits 1e+21. Only + // a genuinely fractional value is the wrong shape for the kind. + const bool is_signed = f->kind >= 2 && f->kind <= 5; + const TableJsonInteger number = TableJsonInterpret( token, length, integral ); + if ( !number.finite || number.fractional ) + { + in.report->kind_mismatch++; + return true; + } + if ( number.saturated ) { in.report->clamped++; } // past what sixty-four bits hold + // THE DECLARED RANGE FIRST, THEN THE STORAGE WIDTH, the wire's order (§4), + // so a text and a wire loaded from the same data land the same instance. + // The comparison is on the value's OWN scale, correctly signed past + // INT64_MAX, where the storage's bit pattern is not a number to compare. + TableJsonInteger bounded = number; + if ( f->has_range ) + { + const double scale = number.negative ? -(double) number.magnitude : (double) number.magnitude; + if ( scale < f->range_min ) { bounded = TableJsonIntegerOf( f->range_min ); in.report->clamped++; } + else if ( scale > f->range_max ) { bounded = TableJsonIntegerOf( f->range_max ); in.report->clamped++; } + } + bool moved = false; + const int64_t value = TableJsonIntegerInDomain( bounded, is_signed, (int32_t) f->elem_size, moved ); + if ( moved ) { in.report->clamped++; } + TableJsonSetRaw( storage, f->elem_size, (uint64_t) value ); + return true; +} + +inline bool TableJsonReadField( TableJsonIn & in, void * base, const TableFieldInfo * f, int32_t depth ) +{ + uint8_t * storage = (uint8_t *) base + f->offset; + if ( TableJsonIsMap( f ) ) + { + return TableJsonReadMap( in, (void *) storage, f, depth ); + } + if ( TableJsonIsList( f ) ) + { + return TableJsonReadList( in, (void *) storage, f, depth ); + } + + if ( f->kind == 12 ) + { + int32_t length = 0; + if ( !TableJsonScanString( in, (char *) storage, f->array_bound, &length ) ) { return false; } + storage[length] = 0; + TableJsonSetCount( base, f, length ); + return true; + } + if ( f->kind == 33 ) + { + char16_t * units = (char16_t *) (void *) storage; + int32_t length = 0; + if ( !TableJsonScanWString( in, units, (int32_t) f->array_bound, &length ) ) { return false; } + units[length] = 0; // the terminating zero UNIT at index length (§7.2, SPEC.md §4.12) + TableJsonSetCount( base, f, length ); + return true; + } + if ( TableJsonIsBytes( f ) ) + { + // base64 decodes STRAIGHT INTO the field's storage, six bits at a + // time — no window, no temporary, so a bytes(N) of any declared + // extent reads the same way. A base64 body carries no escapes, so a + // backslash in one is simply not an alphabet character. + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + in.pos++; + memset( storage, 0, (size_t) f->array_bound ); + TableJsonSetCount( base, f, 0 ); + const char * alphabet = TableJsonBase64Alphabet(); + int32_t placed = 0; + uint32_t accumulator = 0; + int32_t held = 0; + bool clamped = false; + bool malformed = false; + for ( ;; ) + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos++]; + if ( c == '"' ) { break; } + if ( c == '=' || malformed ) { continue; } + const char * at = c != 0 ? strchr( alphabet, c ) : NULL; + if ( at == NULL ) { malformed = true; continue; } + accumulator = ( accumulator << 6 ) | (uint32_t) ( at - alphabet ); + held += 6; + if ( held >= 8 ) + { + held -= 8; + if ( placed < f->array_bound ) + { + storage[placed++] = (uint8_t) ( ( accumulator >> held ) & 0xff ); + } + else + { + clamped = true; + } + } + } + if ( malformed ) + { + // a body that is not base64 is the wrong shape for the kind: the + // field keeps its default and the event is counted + in.report->kind_mismatch++; + return true; + } + if ( clamped ) { in.report->clamped++; } + TableJsonSetCount( base, f, placed ); + return true; + } + if ( TableJsonIsKeyed( f ) ) + { + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + // every slot back to its declared defaults first, so a key the text + // omits keeps them and a repeated field key cannot leave an earlier + // occurrence's slots standing + for ( int32_t i = 0; i < f->array_bound; i++ ) + { + void * slot = storage + (int64_t) i * f->elem_size; + if ( f->kind == 13 ) { f->table->reset( slot ); } + else { memset( slot, 0, (size_t) f->elem_size ); } + } + char shape = TableJsonElementShape( f ); + // A KEYED OBJECT'S KEYS ARE KEYS: a variant named twice is a duplicate + // key like any other, last-wins and counted (§16.2). Tracked the way + // a table's own field keys are — a bounded, allocation-free bitmask; + // a vocabulary wider than this still reads, its repeats simply stop + // being counted. + uint64_t seen[8] = {}; + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t slot = -1; + for ( int64_t v = 0; v < f->array_bound; v++ ) + { + // nothing is stored for None, so "None" finds no slot and is + // an unknown key like any other name this reader cannot place + if ( !TableJsonKeyedSlotValid( f, v ) ) { continue; } + if ( strcmp( f->key_name( TableJsonKeyedSlotKey( v ) ), key ) == 0 ) { slot = v; break; } + } + if ( slot >= 0 && slot < 512 ) + { + uint64_t bit = uint64_t( 1 ) << ( slot & 63 ); + if ( ( seen[slot >> 6] & bit ) != 0 ) { in.report->duplicate++; } + seen[slot >> 6] |= bit; + } + if ( slot < 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( TableJsonValueShape( in ) != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadScalar( in, storage + slot * f->elem_size, f, depth + 1 ) ) + { + return false; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; + } + if ( f->is_array ) + { + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + in.pos++; + // LAST WINS has to be true of a repeated ARRAY key too, and it is + // wire-visible: a fixed array writes every slot, so a second, shorter + // occurrence overlaying a prefix would leave the first occurrence's + // tail standing. The field goes back to its declared defaults before + // this occurrence's elements are placed — the re-establishment a nested + // table and a union arm already get. A table element's defaults are + // its own (the reset hook); every other element kind's storage + // default is zero, which is what the generated array declares. + if ( f->kind == 13 ) + { + for ( int32_t i = 0; i < f->array_bound; i++ ) + { + f->table->reset( storage + (int64_t) i * f->elem_size ); + } + } + else + { + memset( storage, 0, (size_t) f->array_bound * (size_t) f->elem_size ); + } + TableJsonSetCount( base, f, 0 ); + int32_t placed = 0; + char shape = TableJsonElementShape( f ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + if ( placed >= f->array_bound ) + { + // more elements than the reader's bound: the bounded prefix + // is kept and the excess counts, the wire's rule (§4) + in.report->clamped++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( f->kind == 17 ) + { + // an element of an ARRAY OF POINTERS (§2.1): null is a null slot, an + // object is the pointee in place or an `&node` reference (§16.7) + char got = TableJsonValueShape( in ); + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( storage + (int64_t) placed * f->elem_size, f->elem_size, 0 ); + } + else if ( got != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, storage + (int64_t) placed * f->elem_size, f, depth + 1 ) ) { return false; } + placed++; + } + else if ( TableJsonValueShape( in ) != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + placed++; + } + else + { + if ( !TableJsonReadScalar( in, storage + (int64_t) placed * f->elem_size, f, depth + 1 ) ) { return false; } + placed++; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + // a fixed array's tail keeps the defaults the prefill left there, + // exactly as a short wire count does + TableJsonSetCount( base, f, placed ); + return true; + } + return TableJsonReadScalar( in, storage, f, depth ); +} + +inline bool TableJsonReadTableKeys( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth, const char * first_key ); + +// ONE table object: keys are field keys, unknown ones are skipped and +// counted, a repeated key is last-wins and counted. The instance is already +// at its declared defaults when this is entered, so a key the text never +// mentions keeps the default an absent field takes on the wire (§4). +inline bool TableJsonReadTable( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth ) +{ + if ( depth > kTableJsonMaxDepth ) { in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + return TableJsonReadTableKeys( in, base, info, depth, NULL ); +} + +// The keys of an object whose brace is already consumed. A pointer's object +// opens the same way a table's does, but its FIRST key may be `&node` (§16.7) +// and the adapter that reads it has to scan the key to know — so it hands the +// key it scanned in as `first_key`, with the colon consumed, and this places +// it before scanning the rest. +inline bool TableJsonReadTableKeys( TableJsonIn & in, void * base, const TableTypeInfo * info, int32_t depth, const char * first_key ) +{ + // duplicate tracking, bounded and allocation-free: a table with more + // fields than this still reads, its repeats simply stop being counted + uint64_t seen[8] = {}; + for ( ;; ) + { + char key[kTableJsonMaxKey]; + char c = 0; + if ( first_key != NULL ) + { + memcpy( key, first_key, strlen( first_key ) + 1 ); // scanned into a buffer this size by the caller + first_key = NULL; + } + else + { + c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; return true; } + if ( c == 0 ) { in.bad = true; return false; } + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + } + int32_t index = -1; + for ( int32_t i = 0; i < info->num_fields; i++ ) + { + if ( strcmp( info->fields[i].json, key ) == 0 ) { index = i; break; } + } + if ( key[0] == '&' ) + { + // THE AMPERSAND PREFIX IS RESERVED TO THE FORM (docs/SPEC-TABLES.md + // §16.7). No declaration may take a key beginning with it, so this + // is never a field this build lacks — it is the sharing construct + // somewhere it cannot stand: `&node` is the FIRST key of a pointer's + // object and nothing else, and the adapter that reads a pointer + // has consumed it before these keys are read. MALFORMED, refused + // and counted; never counted as unknown, never skipped. + in.report->malformed = true; + in.bad = true; + return false; + } + if ( index < 0 ) + { + in.report->unknown++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else + { + const TableFieldInfo * f = &info->fields[index]; + if ( index < 512 ) + { + uint64_t bit = uint64_t( 1 ) << ( index & 63 ); + if ( ( seen[index >> 6] & bit ) != 0 ) { in.report->duplicate++; } + seen[index >> 6] |= bit; + } + // PRESENCE OF THE KEY IS THE PRESENCE (§16.2): reaching this line + // is the key being present, so an optional is set present + // whatever its value — with one exception the page names: a JSON + // null, which reads as ABSENT rather than as a value. + char got = TableJsonValueShape( in ); + if ( f->kind == 17 && !f->is_array ) + { + // a pointer: null is a null pointer, an object is the pointee + // in place or an `&node` reference to one (§16.7), a string is + // a BYTE BUFFER's bytes (§2.5), and anything else is the wrong + // shape for the kind + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) base + f->offset, f->elem_size, 0 ); + } + else if ( got != TableJsonShape( f ) ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, (uint8_t *) base + f->offset, f, depth ) ) + { + return false; + } + } + else if ( f->optional && got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + // absent, and back at its defaults: a repeated key whose last + // occurrence is null must not leave an earlier value standing + if ( f->table != NULL ) { f->table->reset( (uint8_t *) base + f->offset ); } + else { memset( (uint8_t *) base + f->offset, 0, (size_t) f->elem_size ); } + TableJsonSetRaw( (uint8_t *) base + f->present_offset, 1, 0 ); + } + else + { + if ( got != TableJsonShape( f ) ) + { + // the wrong JSON type for the kind: skipped, never coerced + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadField( in, base, f, depth ) ) + { + return false; + } + if ( f->optional ) + { + TableJsonSetRaw( (uint8_t *) base + f->present_offset, 1, 1 ); + } + } + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; return true; } + in.bad = true; + return false; + } +} + +// ---- the two entry points the per-table wrappers name ---- + +inline bool TableJsonRead( void * value, const TableTypeInfo * info, const char * text, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableJsonIn in; + in.text = text; + in.size = bytes; + in.pos = 0; + in.report = report != NULL ? report : &ignored; + in.bad = false; + in.graph = NULL; + info->reset( value ); + if ( text == NULL || bytes < 0 ) + { + in.report->malformed = true; + return false; + } + bool ok = TableJsonReadTable( in, value, info, 0 ); + if ( ok ) + { + TableJsonSpace( in ); + if ( in.pos != in.size ) { in.bad = true; } // trailing rubbish is not one text + } + if ( in.bad || !ok ) + { + in.report->malformed = true; + return false; + } + return true; +} + +inline int64_t TableJsonWrite( const void * value, const TableTypeInfo * info, char * buffer, int64_t capacity ) +{ + TableJsonOut out; + out.buffer = buffer; + out.capacity = capacity; + out.offset = 0; + out.overflow = false; + out.graph = NULL; + if ( !TableJsonWriteValue( out, value, info, 0 ) ) { return -1; } + // THE CANONICAL TEXT ENDS WITH EXACTLY ONE NEWLINE (docs/SPEC-TABLES.md + // §16.1). Every writer emits it — this walk, the C# walk and + // "schema unpack" — and every reader accepts a text with or without one, + // because the trailing whitespace a read already skips is what makes the + // two the same text. It is a byte of the FORM rather than a file + // convention: a text that is written to a file, pasted into a diff and + // handed back through a pipe has to be one text in all three places, and a + // buffer whose last byte is a closing brace is the one shape that is not. + out.put( '\n' ); + if ( out.overflow ) { return -1; } + return out.offset; +} + +// ---- json walk: end ---- + +// ---- json graph walk: begin ---- +// +// THE VARIABLE CLASS's half of the text form (docs/SPEC-TABLES.md §16.7). The +// walk above places every kind but one; this defines the three adapters it +// calls for that one, and the two entry points a pointered table's wrappers +// name. The text is the fixed class's — a pointee is an object in place — and a +// node named more than once carries `&node`: defined once, with its fields, +// and referenced after by `{ "&node": N }` alone. + +// ---- the identity map ---- +// +// ONE map shape serves both directions. Writing keys it by a node's ADDRESS and +// counts the slots that name the node, so the second pass knows at a node's +// first occurrence whether it will be named again; reading keys it by the +// text's own label and answers the node it defined. Open addressing, a +// multiply-shift hash and quadrupling growth — TablePackMap's shape (§6.2), on +// the same terms: proportional to nodes, never to bytes, on the authoring +// side, and released before the call returns. + +struct TableJsonGraphEntry +{ + uint64_t key; // a node's address (write) or a label (read); 0 is an empty slot + int64_t count; // write: how many slots name this node + int64_t label; // write: the `&node` label assigned at its first write, 0 until then + uint8_t open; // the descent is still open: a reference here is a cycle (write), a self-reference (read) + uint32_t node; // read: the node's arena offset; 0 for a definition the reader dropped + const TableTypeInfo * type; // read: the node's table; NULL for a dropped one +}; + +struct TableJsonGraphMap +{ + TableJsonGraphEntry * entries; + int64_t capacity; // a power of two, or zero while empty + int64_t count; + TableAllocator allocator; // the caller's pair (§6.5): the builder's on read, the one handed to ToJson on write +}; + +inline void TableJsonGraphMapInit( TableJsonGraphMap & map, TableAllocator allocator ) +{ + map.entries = NULL; + map.capacity = 0; + map.count = 0; + map.allocator = allocator; +} + +inline void TableJsonGraphMapShutdown( TableJsonGraphMap & map ) +{ + map.allocator.free( map.allocator.context, map.entries ); + TableJsonGraphMapInit( map, map.allocator ); +} + +inline int64_t TableJsonGraphMapSlot( const TableJsonGraphMap & map, uint64_t key ) +{ + uint64_t hash = key * 0x9E3779B97F4A7C15ull; + hash ^= hash >> 29; + int64_t mask = map.capacity - 1; + int64_t at = (int64_t) ( hash & (uint64_t) mask ); + while ( map.entries[at].key != 0 && map.entries[at].key != key ) + { + at = ( at + 1 ) & mask; + } + return at; +} + +inline TableJsonGraphEntry * TableJsonGraphMapFind( TableJsonGraphMap & map, uint64_t key ) +{ + if ( map.capacity == 0 ) { return NULL; } + TableJsonGraphEntry * entry = &map.entries[ TableJsonGraphMapSlot( map, key ) ]; + return entry->key == key ? entry : NULL; +} + +inline bool TableJsonGraphMapGrow( TableJsonGraphMap & map ) +{ + TableJsonGraphMap grown; + grown.allocator = map.allocator; + grown.capacity = map.capacity != 0 ? map.capacity * 4 : 64; + grown.count = 0; + grown.entries = (TableJsonGraphEntry *) map.allocator.alloc( map.allocator.context, grown.capacity * (int64_t) sizeof( TableJsonGraphEntry ) ); // zeroed, by the pair's contract + if ( grown.entries == NULL ) { return false; } + for ( int64_t i = 0; i < map.capacity; i++ ) + { + if ( map.entries[i].key == 0 ) { continue; } + grown.entries[ TableJsonGraphMapSlot( grown, map.entries[i].key ) ] = map.entries[i]; + grown.count++; + } + map.allocator.free( map.allocator.context, map.entries ); + map = grown; + return true; +} + +// the entry for a key, made if it was not there; `taken` says which. NULL is the +// allocator refusing, and the walk refuses with it. +inline TableJsonGraphEntry * TableJsonGraphMapReach( TableJsonGraphMap & map, uint64_t key, bool & taken ) +{ + if ( ( map.count + 1 ) * 4 >= map.capacity * 3 ) // keep the load factor under three quarters + { + if ( !TableJsonGraphMapGrow( map ) ) { return NULL; } + } + TableJsonGraphEntry * entry = &map.entries[ TableJsonGraphMapSlot( map, key ) ]; + taken = entry->key != key; + if ( taken ) + { + entry->key = key; + map.count++; + } + return entry; +} + +// ---- reading: into a builder ---- + +struct TableJsonGraphIn +{ + TableWorker * worker; // where every node comes from + TableJsonGraphMap labels; // a label -> the node it defined +}; + +// `&node`'s value, the LABEL: a positive integer spelled as one — digits, no sign, no +// fraction, no exponent, no leading zero (§16.7). Anything else is malformed. +inline bool TableJsonScanLabel( TableJsonIn & in, uint64_t & label ) +{ + TableJsonSpace( in ); + if ( in.pos >= in.size || in.text[in.pos] < '1' || in.text[in.pos] > '9' ) + { + in.report->malformed = true; + in.bad = true; + return false; + } + uint64_t value = 0; + while ( in.pos < in.size && in.text[in.pos] >= '0' && in.text[in.pos] <= '9' ) + { + uint64_t digit = (uint64_t) ( in.text[in.pos] - '0' ); + if ( value > ( UINT64_MAX - digit ) / 10 ) + { + in.report->malformed = true; + in.bad = true; + return false; + } + value = value * 10 + digit; + in.pos++; + } + label = value; + return true; +} + +// A BYTE BUFFER's text (docs/SPEC-TABLES.md §2.5, §16.2): a string. For a +// *string the string's bytes become the blob; for a *bytes the string is base64 +// and its decoded bytes do. The blob is allocated at EXACTLY the decoded +// length — the string is scanned once without keeping it to learn the length, +// and once into the node — so a blob of any size reads with no window and no +// bound to clamp against. A *bytes body that is not base64 is the wrong shape +// for the kind: the reference stays null and the event is counted. +inline bool TableJsonReadBlob( TableJsonIn & in, void * slot, const TableFieldInfo * f ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '"' ) { in.bad = true; return false; } + TableRef * ref = (TableRef *) slot; + ref->value = 0; + if ( strcmp( f->type_name, "string" ) == 0 ) + { + const int64_t mark = in.pos; + int32_t length = 0; + if ( !TableJsonScanString( in, NULL, 0, &length ) ) { return false; } + in.pos = mark; + char * data = TableStringEmplace( *graph->worker, *ref, NULL, (int64_t) length ); + if ( data == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + int32_t placed = 0; + return TableJsonScanString( in, data, length, &placed ); + } + // base64: the alphabet characters decide the length, six bits apiece + const char * alphabet = TableJsonBase64Alphabet(); + const int64_t mark = in.pos + 1; + int64_t symbols = 0; + bool malformed = false; + in.pos++; + for ( ;; ) + { + if ( in.pos >= in.size ) { in.bad = true; return false; } + char c = in.text[in.pos++]; + if ( c == '"' ) { break; } + if ( c == '=' || malformed ) { continue; } + if ( c == 0 || strchr( alphabet, c ) == NULL ) { malformed = true; continue; } + symbols++; + } + if ( malformed ) + { + in.report->kind_mismatch++; + return true; + } + const int64_t length = ( symbols * 6 ) / 8; + uint8_t * data = TableBytesEmplace( *graph->worker, *ref, length ); + if ( data == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + int64_t placed = 0; + uint32_t accumulator = 0; + int32_t held = 0; + for ( int64_t at = mark; ; at++ ) + { + char c = in.text[at]; + if ( c == '"' ) { break; } + const char * symbol = c != '=' ? strchr( alphabet, c ) : NULL; + if ( symbol == NULL ) { continue; } + accumulator = ( accumulator << 6 ) | (uint32_t) ( symbol - alphabet ); + held += 6; + if ( held >= 8 ) + { + held -= 8; + if ( placed < length ) { data[placed++] = (uint8_t) ( ( accumulator >> held ) & 0xff ); } + } + } + return true; +} + +// A pointer's object. Its FIRST key decides what it is: `&node` naming a label not +// yet defined, with fields after it, is a DEFINITION; `&node` naming one already +// defined, alone, is a REFERENCE; any other key is a node named once, its +// object in place. The node comes from the +// builder's arena, and the slot holds its arena offset (§6.3). A pointer whose +// target is a BYTE BUFFER — no table — takes a string instead (§2.5). +inline bool TableJsonReadPointer( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( f->table == NULL ) { return TableJsonReadBlob( in, slot, f ); } + // the pointee nests one level down, exactly as a by-value table does, and + // takes the same cap: a chain nests as deep as it is long (§16.7) + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + in.pos++; + char c = TableJsonPeek( in ); + if ( c == '}' ) + { + // an empty object: a node at its defaults, named once + in.pos++; + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } // the arena refused + return true; + } + if ( c == 0 ) { in.bad = true; return false; } + char key[kTableJsonMaxKey]; + int32_t key_length = 0; + if ( !TableJsonScanString( in, key, kTableJsonMaxKey - 1, &key_length ) ) { return false; } + key[key_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + if ( strcmp( key, "&node" ) != 0 ) + { + // a node named once: the pointee's object in place, and this key is + // its first field — unless it is the reserved prefix under a spelling + // this form does not have, which ReadTableKeys refuses + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } + return TableJsonReadTableKeys( in, node, f->table, depth + 1, key ); + } + uint64_t label = 0; + if ( !TableJsonScanLabel( in, label ) ) { return false; } + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph->labels, label, taken ); + if ( entry == NULL ) { in.report->malformed = true; in.bad = true; return false; } + // ONE SPELLING, and what follows the label says which half it is: fields + // after a label the text has not defined DEFINE it, and a label alone that + // the text has defined REFERS to it. The other two are malformed — a label + // alone that the text never defined, which would otherwise read as a default + // node under a silent report, and a field after a label already defined, + // which would be a second definition. That is what keeps a typo loud. + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; c = TableJsonPeek( in ); } + bool bare = c == '}'; + if ( bare == taken ) { in.report->malformed = true; in.bad = true; return false; } + if ( bare ) + { + // A REFERENCE. A label is defined when its object CLOSES, so a + // reference met inside its own definition — at any depth of by-value + // nesting — names a node whose descent is still open: the cycle the + // wire refuses (§3.1), refused here where it is written. A definition + // the reader dropped names no node, so the slot stays null with + // nothing more counted — the drop was counted where it happened. A + // node of another table than the slot declares is a kind mismatch, as + // on the wire. + in.pos++; + if ( entry->open != 0 ) { in.report->malformed = true; in.bad = true; return false; } + TableRef ref; + if ( entry->type == NULL ) + { + memcpy( slot, &ref, sizeof( ref ) ); + return true; + } + if ( entry->type != f->table ) + { + memcpy( slot, &ref, sizeof( ref ) ); + in.report->kind_mismatch++; + return true; + } + ref.value = (int64_t) entry->node; + memcpy( slot, &ref, sizeof( ref ) ); + return true; + } + // A DEFINITION: the node is allocated, the label is its, and the keys after + // `&node` are its fields. The entry is OPEN until the object closes, so a + // reference to the label from inside the node's own fields is refused as + // the cycle it is; the node and its table are filled in at the close. + void * node = f->emplace( *graph->worker, slot ); + if ( node == NULL ) { in.report->malformed = true; in.bad = true; return false; } + entry->open = 1; + if ( !TableJsonReadTableKeys( in, node, f->table, depth + 1, NULL ) ) { return false; } + entry = TableJsonGraphMapFind( graph->labels, label ); // the map may have grown under the descent + if ( entry == NULL ) { in.report->malformed = true; in.bad = true; return false; } + TableRef ref; + memcpy( &ref, slot, sizeof( ref ) ); + entry->node = (uint32_t) ref.value; + entry->type = f->table; + entry->open = 0; + return true; +} + +// An `&`-prefixed key opening an object the walk is SKIPPING — a value past an +// array's bound, an unknown key's value, a value of the wrong shape. A +// definition in there still takes its label, so the numbering survives whatever +// the storage could not hold (§16.7): the label is registered with no node, and a +// reference to it reads null. Any other prefixed key is the reserved prefix +// out of place. +inline bool TableJsonSkippedAmpersand( TableJsonIn & in, const char * key, int32_t ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL || strcmp( key, "&node" ) != 0 ) { in.report->malformed = true; in.bad = true; return false; } + uint64_t label = 0; + if ( !TableJsonScanLabel( in, label ) ) { return false; } + bool taken = false; + if ( TableJsonGraphMapReach( graph->labels, label, taken ) == NULL ) { in.report->malformed = true; in.bad = true; return false; } + return true; // a fresh entry is node 0, type NULL: a definition with no node +} + +// ---- writing: from a region's const root ---- + +struct TableJsonGraphOut +{ + TableJsonGraphMap nodes; // a node's address -> how many slots name it, and its `&node` once assigned + bool counting; // PASS ONE: count the references, refuse a cycle, emit nothing + int64_t next_label; +}; + +// The node a slot names: null as `null`, a node named once as its object in +// place, and a node named more than once under the construct. Which of the +// last two it is was learned in pass one; pass two spells it. +inline bool TableJsonWritePointer( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphOut * graph = (TableJsonGraphOut *) out.graph; + if ( graph == NULL ) { return false; } + const void * node = f->resolve( slot ); + if ( node == NULL ) + { + out.raw( "null", 4 ); + return true; + } + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph->nodes, (uint64_t) (uintptr_t) node, taken ); + if ( entry == NULL ) { return false; } + if ( f->table == NULL ) + { + // A BYTE BUFFER (§2.5, §16.7): its text is a string, which has no + // first key to carry `&node`, so a blob named from more than one + // slot has no spelling this form can carry and the graph is refused — + // as a shared node with nothing to write is. A blob named once is its + // bytes in place: base64 for a *bytes, the string itself for a *string. + if ( graph->counting ) { entry->count++; return true; } + if ( entry->count > 1 ) { return false; } + const TableBlob * blob = (const TableBlob *) node; + if ( blob->length > (uint32_t) 0x7fffffff ) { return false; } + if ( strcmp( f->type_name, "string" ) == 0 ) { TableJsonWriteString( out, (const char *) ( blob + 1 ), (int32_t) blob->length ); } + else { TableJsonWriteBase64( out, (const uint8_t *) ( blob + 1 ), (int32_t) blob->length ); } + return true; + } + if ( graph->counting ) + { + // PASS ONE: one visit per node, every slot that names it counted, and + // a reference to a node whose descent is still open is a cycle — + // refused here as the wire refuses it (§3.1) + entry->count++; + if ( !taken ) { return entry->open == 0; } + entry->open = 1; + if ( !TableJsonWriteValue( out, node, f->table, depth ) ) { return false; } + entry = TableJsonGraphMapFind( graph->nodes, (uint64_t) (uintptr_t) node ); // the map may have grown under the descent + if ( entry == NULL ) { return false; } + entry->open = 0; + return true; + } + // PASS TWO: a node named once is its object in place; a node named more + // than once is DEFINED at its first occurrence — `&node` first, then its + // fields — and REFERENCED by `&node` alone after that, spelled the same way at + // every site. Labels run from 1 in first-write order and are the text's own, + // so a stray number in a hand-edited text is most often one never defined. + if ( entry->count <= 1 ) + { + return TableJsonWriteValue( out, node, f->table, depth ); + } + if ( depth > kTableJsonMaxDepth ) { return false; } + if ( entry->label != 0 ) + { + out.put( '{' ); + out.line( depth + 1 ); + out.raw( "\"&node\": ", 9 ); + TableJsonWriteUnsigned( out, (uint64_t) entry->label ); + out.line( depth ); + out.put( '}' ); + return true; + } + entry->label = ++graph->next_label; + out.put( '{' ); + out.line( depth + 1 ); + out.raw( "\"&node\": ", 9 ); + TableJsonWriteUnsigned( out, (uint64_t) entry->label ); + bool any = true; + int64_t before = out.offset; + if ( !TableJsonWriteFields( out, node, f->table, depth, any ) ) { return false; } + // a definition carries at least one field, because a label alone is a + // reference: a shared node with nothing to write has no definition this + // form can spell, and the writer refuses it as it refuses any value it + // cannot spell (§16.3) + if ( out.offset == before ) { return false; } + out.line( depth ); + out.put( '}' ); + return true; +} + +// ---- the two entry points a pointered table's wrappers name ---- + +// The text into the builder's root. Every node the text names is allocated in +// the builder's arena through the field's own Emplace; the label map is the +// walk's, released before this returns. The root itself takes no label — nothing +// may name it (§16.7) — so an `&node` at the root is refused like any other key +// of the prefix. +inline bool TableJsonReadGraph( TableWorker & worker, void * root, const TableTypeInfo * info, const char * text, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + if ( worker.arena == NULL ) { if ( report != NULL ) { report->malformed = true; } return false; } + TableJsonGraphIn graph; + graph.worker = &worker; + TableJsonGraphMapInit( graph.labels, worker.arena->allocator ); + TableJsonIn in; + in.text = text; + in.size = bytes; + in.pos = 0; + in.report = report != NULL ? report : &ignored; + in.bad = false; + in.graph = &graph; + info->reset( root ); + if ( text == NULL || bytes < 0 ) + { + in.report->malformed = true; + return false; + } + bool ok = TableJsonReadTable( in, root, info, 0 ); + if ( ok ) + { + TableJsonSpace( in ); + if ( in.pos != in.size ) { in.bad = true; } // trailing rubbish is not one text + } + TableJsonGraphMapShutdown( graph.labels ); + if ( in.bad || !ok ) + { + in.report->malformed = true; + return false; + } + return true; +} + +// The text of a region's const root: measured when the buffer is NULL, written +// when it is not, over one code path. Two passes over one walk — the first +// counts how many slots name each node and refuses a cycle, the second writes +// — so a node's first occurrence knows whether it will be named again. The +// ROOT's entry is open for the whole first pass, so a reference back at it is +// the cycle it is (§3.1), and it takes no label. +inline int64_t TableJsonWriteGraph( const void * root, const TableTypeInfo * info, char * buffer, int64_t capacity, TableAllocator allocator ) +{ + if ( root == NULL ) { return -1; } + TableJsonGraphOut graph; + TableJsonGraphMapInit( graph.nodes, allocator ); + graph.counting = true; + graph.next_label = 0; + bool taken = false; + TableJsonGraphEntry * entry = TableJsonGraphMapReach( graph.nodes, (uint64_t) (uintptr_t) root, taken ); + if ( entry == NULL ) { TableJsonGraphMapShutdown( graph.nodes ); return -1; } + entry->open = 1; + TableJsonOut count; + count.buffer = NULL; + count.capacity = 0; + count.offset = 0; + count.overflow = false; + count.graph = &graph; + bool ok = TableJsonWriteValue( count, root, info, 0 ); + graph.counting = false; + TableJsonOut out; + out.buffer = buffer; + out.capacity = capacity; + out.offset = 0; + out.overflow = false; + out.graph = &graph; + if ( ok ) { ok = TableJsonWriteValue( out, root, info, 0 ); } + TableJsonGraphMapShutdown( graph.nodes ); + if ( !ok ) { return -1; } + out.put( '\n' ); // the canonical text ends with exactly one newline (§16.1) + if ( out.overflow ) { return -1; } + return out.offset; +} + +// ---- json graph walk: end ---- + +// ---- the out-of-line array's slot (docs/SPEC-TABLES.md §8.1) ---- + +inline int32_t TableJsonExtentCount( const void * slot ) +{ + int32_t count = 0; + memcpy( &count, (const uint8_t *) slot + 8, sizeof( count ) ); + return count < 0 ? 0 : count; +} + +inline const uint8_t * TableJsonExtentElements( const void * slot ) +{ + int64_t delta = 0; + memcpy( &delta, slot, sizeof( delta ) ); + return delta != 0 ? (const uint8_t *) slot + delta : NULL; +} + +// ---- json map walk: begin ---- + +inline bool TableJsonIsMap( const TableFieldInfo * f ) +{ + return f->is_array && f->array_bound == 0 && strncmp( f->type_name, "map[", 4 ) == 0; +} + +// the entry's two rows: fields[0] IS the key and fields[1] IS the value, which +// is what makes a user's own table of pairs the same bytes (§2.8) +inline const TableFieldInfo * TableJsonMapKeyField( const TableFieldInfo * f ) { return &f->table->fields[0]; } +inline const TableFieldInfo * TableJsonMapValueField( const TableFieldInfo * f ) { return &f->table->fields[1]; } + +inline bool TableJsonMapKeyIsString( const TableFieldInfo * key ) { return key->kind == 12; } +inline bool TableJsonMapKeySigned( const TableFieldInfo * key ) { return key->kind >= 2 && key->kind <= 5; } + +// AN INTEGER KEY IS THE INTEGER'S DECIMAL SPELLING, QUOTED, because a JSON +// object's keys are strings. Written digit by digit so no locale can move it. +inline void TableJsonWriteMapIntegerKey( TableJsonOut & out, const void * storage, const TableFieldInfo * key ) +{ + uint64_t magnitude = 0; + bool negative = false; + if ( TableJsonMapKeySigned( key ) ) + { + int64_t value = 0; + switch ( key->kind ) + { + case 2: value = (int64_t) *(const int8_t *) storage; break; + case 3: value = (int64_t) *(const int16_t *) storage; break; + case 4: value = (int64_t) *(const int32_t *) storage; break; + default: value = *(const int64_t *) storage; break; + } + negative = value < 0; + magnitude = negative ? ( ~(uint64_t) value ) + 1 : (uint64_t) value; + } + else + { + switch ( key->kind ) + { + case 6: magnitude = (uint64_t) *(const uint8_t *) storage; break; + case 7: magnitude = (uint64_t) *(const uint16_t *) storage; break; + case 8: magnitude = (uint64_t) *(const uint32_t *) storage; break; + default: magnitude = *(const uint64_t *) storage; break; + } + } + char digits[24]; + int32_t at = (int32_t) sizeof( digits ); + do { digits[--at] = (char) ( '0' + ( magnitude % 10 ) ); magnitude /= 10; } while ( magnitude != 0 ); + if ( negative ) { digits[--at] = '-'; } + TableJsonWriteString( out, digits + at, (int32_t) sizeof( digits ) - at ); +} + +inline void TableJsonWriteMapKey( TableJsonOut & out, const void * entry, const TableFieldInfo * key ) +{ + const uint8_t * storage = (const uint8_t *) entry + key->offset; + if ( TableJsonMapKeyIsString( key ) ) + { + // A STRING KEY IS THE STRING (§2.8): every JSON key of a map object is + // a KEY OF THE MAP and none is a field key, so the `&` prefix §16.7 + // reserves for field keys is ordinary data here. + TableJsonWriteString( out, (const char *) storage, *(const int32_t *) ( (const uint8_t *) entry + key->count_offset ) ); + return; + } + TableJsonWriteMapIntegerKey( out, (const void *) storage, key ); +} + +// ToJson WRITES ENTRIES IN ASCENDING KEY ORDER, so unpack then pack is +// byte-stable and a diff of two texts is a diff of two maps (§2.8, §17.2). +// A region holds them in that order already, so this is the array in place. +inline bool TableJsonWriteMap( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + const int32_t count = TableJsonExtentCount( slot ); + if ( count == 0 ) { out.raw( "{}", 2 ); return true; } + const TableFieldInfo * key = TableJsonMapKeyField( f ); + const TableFieldInfo * value = TableJsonMapValueField( f ); + const uint8_t * entries = TableJsonExtentElements( slot ); + out.put( '{' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + const void * entry = (const void *) ( entries + (int64_t) i * f->elem_size ); + TableJsonWriteMapKey( out, entry, key ); + out.raw( ": ", 2 ); + if ( !TableJsonWriteField( out, entry, value, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( '}' ); + return true; +} + +// AN INTEGER KEY IS READ BY §16.2's INTEGER RULE AND BY NOTHING ELSE, so +// "2.0" and "1e3" are the integers 2 and 1000 and "-0" is zero. The token is +// walked as a JSON number over its own bytes; a token that rule calls +// malformed makes the KEY malformed, and a genuinely fractional value, or one +// outside the key kind's range, is kind_mismatch for that entry. +// +// THE KEY IS THE SPELLING AND NOTHING AROUND IT. The number walk steps over +// leading whitespace and comments, which is right BETWEEN tokens and wrong +// INSIDE one: a key is an identity, and a padded spelling that resolved to the +// same integer would be a second name for one entry. So the walk must begin at +// the token's first byte, and a token with anything before the number is not a +// JSON number at all, which is malformed on the terms "1-2" is. +inline bool TableJsonMapKeyValue( const char * token, int32_t length, const TableFieldInfo * key, + int64_t & value, bool & fits ) +{ + fits = false; + TableReport scratch; + TableJsonIn probe = { token, (int64_t) length, 0, &scratch, false, NULL }; + bool integral = false; + TableJsonSpace( probe ); + if ( probe.pos != 0 ) { return false; } // whitespace is never part of a key + if ( !TableJsonWalkNumber( probe, &integral ) ) { return false; } + if ( probe.pos != (int64_t) length ) { return false; } // trailing bytes: not a number + // A MAP KEY'S POLICY over the one interpreted value: it REJECTS THE WHOLE + // ENTRY. A key is an identity, so a clamped one is two entries merged, and + // a value the key kind does not hold is kind_mismatch for that entry, + // dropped and counted, never clamped. + const TableJsonInteger number = TableJsonInterpretExact( token, length ); + if ( number.fractional || number.saturated ) { return true; } + bool moved = false; + value = TableJsonIntegerInDomain( number, TableJsonMapKeySigned( key ), (int32_t) key->elem_size, moved ); + fits = !moved; + return true; +} + +// FromJson READS KEYS IN WHATEVER ORDER THE TEXT GIVES THEM. A repeated key is +// last-wins and counted duplicate, the object rule (§16.2) applied inside the +// map. An empty object is an empty map, and null is kind_mismatch. +inline bool TableJsonReadMap( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '{' ) { in.bad = true; return false; } + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; + const TableFieldInfo * key = TableJsonMapKeyField( f ); + const TableFieldInfo * value = TableJsonMapValueField( f ); + const char shape = TableJsonShape( value ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == '}' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + char token[kTableJsonMaxKey]; + int32_t token_length = 0; + bool key_over = false; // longer than THIS buffer: never truncated into a key + if ( !TableJsonScanString( in, token, kTableJsonMaxKey - 1, &token_length, &key_over ) ) { return false; } + token[token_length] = 0; + if ( TableJsonPeek( in ) != ':' ) { in.bad = true; return false; } + in.pos++; + int64_t key_value = 0; + bool place = true; + if ( !TableJsonMapKeyIsString( key ) ) + { + // AN INTEGER KEY PAST THIS SCAN'S BUFFER DROPS AS kind_mismatch, + // here and in the tool's walker. The bytes kept are a PREFIX, and a + // prefix is a different token, so the entry drops rather than a + // truncation being read as a value. The length alone does not + // settle it: a token that long can still spell a number an integer + // kind holds, "1" padded by an exponent of zeroes for one, and this + // read declines to find out. + bool fits = false; + if ( key_over ) { in.report->kind_mismatch++; place = false; } + else if ( !TableJsonMapKeyValue( token, token_length, key, key_value, fits ) ) + { + // A MALFORMED KEY STOPS THE READ where §16.1's rule stops it, + // with the instance holding what was placed before the stop. + in.report->malformed = true; + in.bad = true; + return false; + } + else if ( !fits ) { in.report->kind_mismatch++; place = false; } + } + else if ( key_over || token_length > key->array_bound ) + { + // A KEY LONGER THAN N DROPS ITS ENTRY AND COUNTS clamped, the + // wire's rule, because a clamped key is a merged entry (§2.8). The + // BOUND IS THE WALKER'S, tested here against the key field's own + // descriptor, so placement is left with one failure to report. A + // key past this scan's own buffer is the SAME event, because a + // truncated key is the merged entry the rule exists to prevent. + in.report->clamped++; + place = false; + } + const int32_t before = TableJsonExtentCount( (const void *) slot ); + void * entry = place ? f->place( *graph->worker, slot, token, token_length, key_value ) : NULL; + if ( place && entry == NULL ) + { + // AN ALLOCATION FAILURE IS NOT AN OVERSIZED KEY (§2.8, §16.1). The + // key was checked above, so the arena is what refused, and the read + // stops where the list, blob and pointer paths stop on one rather + // than handing back an instance short of entries the text spelled + // and calling itself clean. + in.report->malformed = true; + in.bad = true; + return false; + } + else if ( entry != NULL && TableJsonExtentCount( (const void *) slot ) == before ) + { + in.report->duplicate++; // last-wins, the object rule inside the map + } + const char got = TableJsonValueShape( in ); + if ( entry == NULL ) + { + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( value->kind == 17 && !value->is_array ) + { + // A POINTER VALUE IS SHARED EXACTLY AS A POINTER FIELD IS (§2.8): + // null is a null slot, an object is the pointee in place or an + // &node reference to one (§16.7), anything else is the wrong shape — + // the same three the field-key loop gives a pointer field, because + // an entry's value IS a field line. + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) entry + value->offset, value->elem_size, 0 ); + } + else if ( got != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, (uint8_t *) entry + value->offset, value, depth + 1 ) ) + { + return false; + } + } + else if ( got != shape ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadField( in, entry, value, depth + 1 ) ) + { + return false; + } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == '}' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; +} + +// ---- json map walk: end ---- + +// ---- json list walk: begin ---- + +// an unbounded array is the out-of-line array that is not a map (§8.1) +inline bool TableJsonIsList( const TableFieldInfo * f ) +{ + return f->is_array && f->array_bound == 0 && !TableJsonIsMap( f ); +} + +// ToJson WRITES THE ELEMENTS IN INDEX ORDER, which is the only order there is, +// so unpack then pack is byte-stable without a rule of its own (§2.9, §17.2). +// A region holds the array in place, so this steps it at the descriptor's pitch. +inline bool TableJsonWriteList( TableJsonOut & out, const void * slot, const TableFieldInfo * f, int32_t depth ) +{ + const int32_t count = TableJsonExtentCount( slot ); + if ( count == 0 ) { out.raw( "[]", 2 ); return true; } + const uint8_t * elements = TableJsonExtentElements( slot ); + out.put( '[' ); + for ( int32_t i = 0; i < count; i++ ) + { + if ( i > 0 ) { out.put( ',' ); } + out.line( depth + 1 ); + const uint8_t * element = elements + (int64_t) i * f->elem_size; + if ( f->kind == 17 ) + { + // a []*T's elements take the pointer row (§16.7): the pointee's + // object in place, null, or `&node` for a shared one + if ( !TableJsonWritePointer( out, element, f, depth + 1 ) ) { return false; } + } + else if ( !TableJsonWriteScalar( out, element, f, depth + 1 ) ) { return false; } + } + out.line( depth ); + out.put( ']' ); + return true; +} + +// FromJson READS EVERY ELEMENT THE TEXT CARRIES, appending each through the +// descriptor's place resolver: `[]` is an empty list, and null is +// kind_mismatch, the array row's own rule (§16.2). LAST WINS holds for a +// repeated key: the list goes back to EMPTY before this occurrence's elements +// land, the builder's storage being reclaimed at reset (§2.9). +inline bool TableJsonReadList( TableJsonIn & in, void * slot, const TableFieldInfo * f, int32_t depth ) +{ + TableJsonGraphIn * graph = (TableJsonGraphIn *) in.graph; + if ( graph == NULL ) { in.report->malformed = true; in.bad = true; return false; } + if ( TableJsonPeek( in ) != '[' ) { in.bad = true; return false; } + if ( depth + 1 > kTableJsonMaxDepth ) { in.bad = true; return false; } + in.pos++; + TableJsonSetRaw( (uint8_t *) slot, 8, 0 ); + TableJsonSetRaw( (uint8_t *) slot + 8, 4, 0 ); + const char shape = TableJsonElementShape( f ); + for ( ;; ) + { + char c = TableJsonPeek( in ); + if ( c == ']' ) { in.pos++; break; } + if ( c == 0 ) { in.bad = true; return false; } + void * element = f->place( *graph->worker, slot, NULL, 0, 0 ); + if ( element == NULL ) + { + // NOT ADDED: the arena could not carve another segment, or the + // count met the int32 cap. The text cannot be placed whole, and + // the read stops where §16.1's rule stops it. + in.report->malformed = true; + in.bad = true; + return false; + } + if ( f->kind == 17 ) + { + // an element of a []*T (§2.9): null is a null slot, an object is the + // pointee in place or an `&node` reference (§16.7) + char got = TableJsonValueShape( in ); + if ( got == 'z' ) + { + if ( !TableJsonLiteral( in, "null" ) ) { return false; } + TableJsonSetRaw( (uint8_t *) element, f->elem_size, 0 ); + } + else if ( got != 'o' ) + { + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadPointer( in, element, f, depth + 1 ) ) { return false; } + } + else if ( TableJsonValueShape( in ) != shape ) + { + // the wrong shape for the element kind: the slot keeps its + // defaults and the event counts, the array row's rule (§16.2) + in.report->kind_mismatch++; + if ( !TableJsonSkipValue( in, depth + 1 ) ) { return false; } + } + else if ( !TableJsonReadScalar( in, element, f, depth + 1 ) ) { return false; } + c = TableJsonPeek( in ); + if ( c == ',' ) { in.pos++; continue; } // a trailing comma is accepted + if ( c == ']' ) { in.pos++; break; } + in.bad = true; + return false; + } + return true; +} + +// ---- json list walk: end ---- + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_JSON + +namespace mapdemo { + +bool TrailsFromJson( TrailsBuilder & builder, const char * text, int64_t bytes, TableReport * report ) +{ + Trails * root = builder.GetRoot(); + if ( root == NULL ) { if ( report != NULL ) { report->malformed = true; } return false; } // locked, or the root allocation failed + return TableJsonReadGraph( builder.main, root, TrailsTableType(), text, bytes, report ); +} + +int64_t TrailsToJsonMeasure( const Trails * root, TableAllocator allocator ) +{ + return TableJsonWriteGraph( root, TrailsTableType(), NULL, 0, allocator ); +} + +int64_t TrailsToJson( const Trails * root, char * buffer, int64_t capacity, TableAllocator allocator ) +{ + return TableJsonWriteGraph( root, TrailsTableType(), buffer, capacity, allocator ); +} + +} // namespace mapdemo diff --git a/testdata/golden/tables/maps/TrailsTable.h b/testdata/golden/tables/maps/TrailsTable.h new file mode 100644 index 000000000..6ad9b9906 --- /dev/null +++ b/testdata/golden/tables/maps/TrailsTable.h @@ -0,0 +1,9623 @@ +// Code generated by the schema compiler from Trails.schema. DO NOT EDIT. +// SPDX-License-Identifier: NONE — this generated output is yours, under terms of +// your choice. See the LICENSE exception in the schema compiler; the compiler is +// AGPL-3.0, its output is not. +// package mapdemo — protocol id 0x1ac124decde5b2aa (packets only: tables version by field id, not by protocol id) +// The TABLE wire (evolution-tolerant, docs/SPEC-TABLES.md): no serialize +// dependency — includable from any TU. + +#pragma once + +#include +#include // the prefill's scalar-array fills +#include // offsetof, for the reflection descriptors + +// ---- the hooks (docs/USAGE.md, "the C++ table runtime's hooks") ---- +// +// schema_assert — the runtime's own assert, and the refusal a debugger reads. +// NDEBUG removes it, exactly as it removes assert. A caller who already routes +// serialize's asserts writes `#define schema_assert serialize_assert` before +// including this header and both halves land in one handler. +#ifndef schema_assert +#include +#define schema_assert assert +#endif // #ifndef schema_assert + +// schema_fatal — what stands after the assert on a path that cannot continue. +// NDEBUG does not remove it. Supply it and is never included. +#ifndef schema_fatal +#include // abort +#define schema_fatal abort +#endif // #ifndef schema_fatal + +// schema_allocate / schema_release — what "no allocator handed in" means for +// this program. schema_allocate hands back ZEROED bytes and NULL on failure: +// an arena segment is copied whole, padding included, so anything left +// uninitialized here would reach a packed region. Supply both and +// is never included; hand a TableAllocator to a builder to route one +// structure's allocations somewhere else again. +#ifndef schema_allocate +#include // calloc, free +#define schema_allocate( bytes ) calloc( (size_t) 1, (size_t) ( bytes ) ) +#define schema_release( pointer ) free( pointer ) +#endif // #ifndef schema_allocate +#include // a node's lifetime starts in arena storage (placement new) +#include // one atomic per slab: the arena is lock-free by ownership + +#include "Trails.h" +#include "FleetTable.h" + +#ifndef MAPDEMO_SCHEMA_TABLE_PRIMITIVES +#define MAPDEMO_SCHEMA_TABLE_PRIMITIVES + +// THE CODEC DOES NOT DEPEND ON THE COMPILER'S INLINING BUDGET. A table of a +// realistic field count emits one large body per type, and the cursor a body +// writes through lives in the caller's `TableWriter`: across a call boundary +// that cursor round-trips through memory, and a `uint8_t *` store may alias the +// writer itself, so every put reloads it. When a budget runs out mid-body the +// codec silently degrades to that shape. Forcing the primitives and the +// fixed-class bodies inline is what keeps the cursor in registers and lets +// adjacent constant framing bytes merge into one store. +#if defined( _MSC_VER ) +#define MAPDEMO_TABLE_INLINE __forceinline +#elif defined( __GNUC__ ) || defined( __clang__ ) +#define MAPDEMO_TABLE_INLINE inline __attribute__(( always_inline )) +#else +#define MAPDEMO_TABLE_INLINE inline +#endif + +namespace mapdemo { + +// WHY A READ WAS REFUSED, by name (docs/SPEC-TABLES.md §3.3, §11). A REFUSAL +// is not one of §4's events: nothing is decoded, no counter moves and no +// damage is reported, so five zero counters and a false flag are what a clean +// read prints too and only the verdict tells them apart. The reason says which +// refusal it was. +// +// This is the MESSAGE PATH's vocabulary and not the cooked form's (§7.4): a +// caller meeting one of these has been refused a MESSAGE on a connection, +// which is a different recovery with a different owner than a file a header +// match turned down. +enum TableMessageReason +{ + newer_form, // a FORM BYTE this reader does not carry (§3) + no_vocabulary, // no table for this connection: the message arrived before the announcement, or after a refused one + second_announcement, // a second announcement on a connection: it sets nothing, amends nothing, and the connection closes + vocabulary_too_large, // an announcement above the receiver's declared bound, refused before an entry is touched + message_form_as_file, // a form 2 wire where a FILE was expected: its table is somewhere else + batch_too_large // a batch of more than 256 bodies on the write side, or of more than the caller has room for on the read side: nothing is written or decoded, and the count says what the wire carries +}; + +// The table-wire read report — the permissive contract's ledger. Silence +// (all zero) means the data matched this reader's schema exactly. +struct TableReport +{ + int32_t unknown = 0; // unknown field ids skipped (newer data) + int32_t kind_mismatch = 0; // known id, changed type — skipped, never misdecoded + // a kind that GREW since the writer (docs/SPEC-TABLES.md §4): an integer + // kind read into a wider one of the same signedness, or f32 into f64, + // decoded EXACTLY. One count per field or per map. It is the one counter + // that names no loss: the bytes were not the shape this reader declares, + // and the number survived. + int32_t widened = 0; + int32_t clamped = 0; // out-of-range values clamped to declared bounds + // a key the TEXT form saw twice: last wins, and the repeat is counted + // (docs/SPEC-TABLES.md §16.2). The wire never raises it — a body carrying an + // id twice is legal input whose last occurrence wins, silently (§3). + int32_t duplicate = 0; + bool malformed = false; // framing damage; decode stopped, partial result kept + // THE REFUSAL VERDICT, which is not one of §4's events and moves no counter + // (docs/SPEC-TABLES.md §3): a FORM BYTE this reader does not carry. Five + // zero counters and a false flag are what a clean read prints too, so the + // verdict is what tells the two apart. + bool refused = false; + // WHICH refusal, and it is read only when refused is set: a read that + // was not refused has no reason, and this member is the one the caller + // must not look at then (docs/SPEC-TABLES.md §3.3). + TableMessageReason reason = newer_form; + // RETAIN-UNKNOWN's pair (docs/SPEC-TABLES.md §6.6), on the same struct for + // the reason duplicate is: a caller has one report type and not two. Both + // are ZERO in every read that did not opt in, and retention moves no + // counter above. A retained field still counts unknown, because unknown + // says what a READER could not name and that stays true. + int32_t retained = 0; // unknown fields whose bytes were kept + int32_t retain_lost = 0; // every unknown this load or save could not keep +}; + + +// WHY A FILE WAS REFUSED, by name (docs/SPEC-TABLES.md §6.5, §7, §19.2): the +// one vocabulary a cook's Open, a block's BlockOpen and a load measure's -1 +// share, because a caller asking "why can I not have this file" is +// asking one question whichever call refused it. The FIRST failing clause names the +// reason, in the order §7 enumerates, so one file answers one value in every +// language. A refusal moves no counter, and a match writes nothing: the +// out-parameter is touched on the refusal path only. +// +// It is not the MESSAGE FORM's vocabulary (TableMessageReason, §3.3): a caller +// meeting one of these has been refused a FILE, by a header match or by a +// measure. +enum TableRefuseReason +{ + ok, // no clause failed: the only value beside a non-null root (§7) + not_a_cook, // the magic is neither this build's constant nor its byte reversal, or the byte-order word contradicts the magic + foreign_order, // the magic byte-reversed: a cook of the other byte order (§7.1) + wrong_build_version, // the build_version word is not this build's (§20) + reserved_not_zero, // a reserved header word is not zero (§7.1) + bad_alignment, // the alignment word is not a power of two, is below eight, is above sixty-four, or is not a multiple of the root's own alignof + truncated, // the part lengths against the caller's length, or a data part too short to hold the root + unaligned_base, // the pointer the caller passed is not aligned for the region: the caller's defect, not the file's + bad_layout, // BlockOpen (§19.2): a pitch, a count, an offset or an extent that disagrees with this build's or leaves the block + unknown_form, // at a MEASURE (§3, §6.5): a form byte this build does not carry, refused before any read + count_over_length, // an array or map count whose elements cannot fit the field's own L (§2.8, §2.9) + count_over_extent_cap, // a count above the int32 extent cap (§2.2), which no region can hold whatever its size + blob_over_size_cap, // a blob whose length is past the derived-size cap (§3.1, §11) + data_cycle // a data cycle reached from a builder: the AUTHORING side's -1 (§3.1, §7.6) +}; +// ---- reflection (tables only, docs/SPEC-TABLES.md) ---- +// +// Static field descriptors for every type in the table closure: name, wire +// id/kind, storage offset, bounds, ranges, enum names and branch guards — +// enough to walk, print, diff, edit or bind any table value at runtime with +// no RTTI and no schema files. TableType() returns X's descriptor. + +struct TableTypeInfo; + +// One arm of a union field: where its payload sits inside the union's storage +// and what its payload looks like. The arm's NAME and its table-wire id come +// from the field's enum_name/variant_id functions at the same tag, so nothing +// is spelled twice (docs/SPEC-TABLES.md §8). +struct TableFieldInfo; + +struct TableUnionArmInfo +{ + uint32_t offset; // offsetof the arm's payload within the union storage + const TableTypeInfo * table; // the arm payload's descriptor, or NULL + // AN ARM IS A FIELD LINE (docs/SPEC-TABLES.md §2.6): an arm that names no + // declared type or table carries the FIELD descriptor a field of that + // type would carry instead — offsets taken within the union storage — so + // a generic walk meets an arm's kind, width, bounds and companions where + // it meets a field's. Exactly one of the two is non-NULL on a set arm. + const TableFieldInfo * field; + uint32_t size; // the arm's whole storage, which selection zero-establishes +}; + +// A union field's shape: the tag, and the arms indexed by it. Arms run +// [0, enum_max]; index 0 is the EMPTY arm and carries no payload. +struct TableUnionInfo +{ + uint32_t tag_offset; // offsetof the tag within the union storage + uint32_t tag_size; // sizeof the tag + const TableUnionArmInfo * arms; +}; + +// The exact raw range of a wide-kind field (docs/SPEC-TABLES.md §8.2): two 128-bit +// values as 64-bit lanes, low lane first, two's complement for the signed kinds. +struct TableWideRange +{ + uint64_t lo[2]; + uint64_t hi[2]; +}; + +// THE SHARED EMPTY DOC (docs/SPEC-TABLES.md §8.1): a declaration with no /// +// block carries a doc column pointing at this one object, so absence costs a +// unit no string data and a printer concatenates doc columns with no null +// test. One definition for the whole unit: every absent doc compares equal by +// address. +inline const char TableDocNone[1] = ""; + +// the arena's allocation front, defined with the variable-length runtime +// below; a descriptor names it only through a pointer parameter. +struct TableWorker; + +struct TableFieldInfo +{ + const char * name; // schema field name, e.g. "health" + const char * json; // the TEXT form's key: the json = "key" attribute, else name (§16.3) + const char * type_name; // schema type name, e.g. "float32", "Grade" + uint64_t id; // table-wire field id: fnv1a64 of the name, of the was alias after a rename (§5) + uint8_t kind; // table-wire kind; for arrays/strings/bytes, the ELEMENT kind + bool is_array; // fixed or counted array (bytes included) + bool is_pointer; // a *T pointer field: storage is an 8-byte TableRef; the target is a table + // THE TWO THE TEXT FORM NEEDS (docs/SPEC-TABLES.md §16.7), and they + // are here for the same reason is_pointer is: the walk is ONE walk + // over descriptors and cannot spell a target's own At or + // Emplace. `resolve` reads a slot in a REGION and answers the + // node it names, or NULL; `emplace` allocates one in a BUILDER's + // arena and points the slot at it. NULL on every field that is not + // a pointer, and emitted only in a unit that declares one. + const void * (*resolve)( const void * slot ); + void * (*emplace)( TableWorker & worker, void * slot ); + bool counted; // a _count/_length int32 companion exists (counted arrays, strings, bytes) + bool optional; // a ?T field: a _present bool companion decides whether it rides + int32_t array_bound; // array capacity / string max length; 0 for plain scalars + uint32_t offset; // offsetof the storage member + uint32_t elem_size; // sizeof the member (element size for arrays) + uint32_t count_offset; // offsetof the _count/_length companion, or 0xffffffff + uint32_t present_offset; // offsetof the _present companion, or 0xffffffff + const TableTypeInfo * table; // nested table's descriptor, or NULL + bool has_range; // a declared [min, max] (int or float) + double range_min; // NOTE: int64 ranges beyond 2^53 lose precision here + double range_max; + // the WIDE kinds (18-29, docs/SPEC-TABLES.md §3, §8.2): frac_bits is a fixed + // field's F — its storage holds units × 2^F — and wide is the declared + // range on that RAW scale, exact, as two 128-bit two's-complement values + // in 64-bit lanes (low lane first). NULL where the declaration bounds + // nothing (a bare uint128) and for every other kind; frac_bits is 0 for + // every kind that is not fixed-point. range_min/range_max still carry + // the declared bounds as doubles — whole units for a fixed field — for + // a walker that only shows them. + uint8_t frac_bits; + const TableWideRange * wide; + int64_t enum_max; // enums: highest valid value (None = 0 always valid); + // unions: the arm count (tag range [0, enum_max]); + // flags: the highest declared BIT INDEX; else -1 + // the vocabulary's names, indexed the same way enum_max bounds: an enum's + // value -> name, a union's tag -> arm name, a FLAGS field's bit index -> + // variant name. NULL for every other kind. + const char * (*enum_name)( uint64_t value ); + // the TABLE-WIRE id of one variant (docs/SPEC-TABLES.md §5): for an enum, the + // hash of the variant's name; for a union, the hash of the arm's name. + // 0 is the reserved id — an enum's None, a union's empty. NULL for every + // other kind — a FLAGS field's variants have no per-variant wire id (§4), + // so a NULL here beside a non-NULL enum_name is what says "flags". + // Walk [0, enum_max] to enumerate a vocabulary and its ids. + uint64_t (*variant_id)( uint64_t value ); + // an ENUM-KEYED array (docs/SPEC-TABLES.md §2.4): the array has one slot per + // variant of key_type_name, indexed by the variant's value, and its slots + // ride under variant ids rather than positions. key_name and key_id are + // the key's vocabulary — walk [0, array_bound) to print slots by name. + // NULL on every other field. + const char * key_type_name; + const char * (*key_name)( uint64_t value ); + uint64_t (*key_id)( uint64_t value ); + // union fields: the tag and its arms, behind a function so the whole + // descriptor stays CONSTANT-INITIALISED (a captureless lambda converts to + // a function pointer at compile time; the arms themselves are a static + // inside it). NULL for every other kind. + const TableUnionInfo * (*arms)(); + // an OUT-OF-LINE array (docs/SPEC-TABLES.md §8.1): place one element and + // hand it back at its defaults. A MAP places BY KEY, a string key comes + // in as the bytes and the length, an integer key as the value, and NULL + // is NOT INSERTED: a key past the bound, or an arena that could not carve + // another segment. A LIST ignores the key and APPENDS, NULL at the arena + // or the int32 cap. NULL on every field that is neither. + void * ( * place )( TableWorker & worker, void * slot, const char * key, int32_t key_length, int64_t key_value ); + const char * guard; // branch guard, e.g. "at_rest" or "!at_rest"; "" if unguarded + // what a PERSON wrote about the field (docs/SPEC-TABLES.md §8.1): the /// + // block above it, verbatim (SPEC §4.1). It is TableDocNone when there is + // none, never NULL. Its tags (SPEC §4.2) follow in declared order, and an + // untagged field is 0 beside NULL. Static, constant-initialized, + // allocating nothing. + const char * doc; + int32_t num_tags; + const char * const * tags; +}; + +struct TableTypeInfo +{ + const char * name; // schema type name + uint32_t size; // sizeof the storage struct + int32_t num_fields; + const TableFieldInfo * fields; + // put one instance back at its declared defaults, in place. A generic + // walker that fills a value has to be able to establish the defaults an + // absent field takes, and it holds no type to spell — this is the one + // thing the descriptors could not express without it. Placement-new + // value-init, exactly what the wire's read path does, and no temporary. + void (*reset)( void * storage ); + // the DERIVED mode (docs/SPEC-TABLES.md): false = fixed-size, a plain + // relocatable struct; true = variable-length, built through a Builder + // and read through a region root. Nobody declares it; the compiler + // works it out. + bool variable; + // the declaration's own doc and tags, on the same terms as a field's + // (docs/SPEC-TABLES.md §8.1) + const char * doc; + int32_t num_tags; + const char * const * tags; +}; + +struct TableWriter +{ + uint8_t * buffer; + int64_t capacity; + int64_t offset = 0; + bool overflow = false; + + // the parameters do not repeat the member names: a parameter that hides a + // member is a warning the estate's compilers disagree about (gcc's + // -Wshadow and cl's C4458 refuse it, clang's -Wshadow does not), and this + // is a header a consumer compiles under its OWN flags + TableWriter( uint8_t * to_buffer, int64_t to_capacity ) : buffer( to_buffer ), capacity( to_capacity ) {} + + MAPDEMO_TABLE_INLINE void raw( const void * data, int64_t bytes ) + { + if ( offset + bytes > capacity ) { overflow = true; return; } + memcpy( buffer + offset, data, (size_t) bytes ); + offset += bytes; + } + MAPDEMO_TABLE_INLINE void put8( uint8_t v ) { raw( &v, 1 ); } + MAPDEMO_TABLE_INLINE void put16( uint16_t v ) { uint8_t b[2] = { uint8_t( v ), uint8_t( v >> 8 ) }; raw( b, 2 ); } + MAPDEMO_TABLE_INLINE void put32( uint32_t v ) { uint8_t b[4] = { uint8_t( v ), uint8_t( v >> 8 ), uint8_t( v >> 16 ), uint8_t( v >> 24 ) }; raw( b, 4 ); } + MAPDEMO_TABLE_INLINE void put64( uint64_t v ) { put32( uint32_t( v ) ); put32( uint32_t( v >> 32 ) ); } + // a 128-bit value as two lanes, the low half first (docs/SPEC-TABLES.md §3) + MAPDEMO_TABLE_INLINE void put128( uint64_t lo, uint64_t hi ) { put64( lo ); put64( hi ); } + // EVERY LENGTH, COUNT, INDEX AND ID REFERENCE IS ONE CANONICAL UNSIGNED + // LEB128 (docs/SPEC-TABLES.md §3): seven value bits a byte, the lowest + // group first, the high bit set on every byte but the last. One value has + // one spelling, so two conforming writers agree byte for byte. + MAPDEMO_TABLE_INLINE void putleb( uint64_t v ) + { + while ( v >= 0x80 ) { put8( uint8_t( v ) | 0x80 ); v >>= 7; } + put8( uint8_t( v ) ); + } +}; + +// TableLebBytes is one value's spelling length, which a MEASURE needs before +// the bytes exist — the length of a body has to be known before it is written, +// because a length whose own width moves cannot be patched in place. +inline int64_t TableLebBytes( uint64_t v ) +{ + int64_t n = 1; + while ( v >= 0x80 ) { v >>= 7; n++; } + return n; +} + +// THE ID TABLE, WRITER SIDE (docs/SPEC-TABLES.md §3). It holds every id the +// body used, once each, in FIRST-USE order over the whole wire, and the body +// names them by position: reference k is the kth entry, counted from 1, and +// reference 0 names NO ID. +// +// Its capacity is a COMPILE-TIME fact of the unit — the distinct names its +// table closure can spell — so a save allocates nothing: the table is a local +// of Measure and of Save. The bucket chain makes ref constant time and makes +// truncate constant time too, which is what an ELIDED field needs: a field +// that turns out not to ride costs nothing in the id table either, so the walk +// interns its id, builds the payload that decides, and undoes the entry when +// nothing rides. +struct TableIds +{ + static const int32_t kCapacity = 76; + static const int32_t kBuckets = 256; + + uint64_t ids[ kCapacity ]; + int32_t chain[ kCapacity ]; + int32_t head[ kBuckets ]; + int32_t count; + bool overflow; + + TableIds() : count( 0 ), overflow( false ) + { + for ( int32_t i = 0; i < kBuckets; i++ ) { head[i] = -1; } + } + + static MAPDEMO_TABLE_INLINE uint32_t bucket_of( uint64_t id ) + { + return uint32_t( ( id * 0x9E3779B97F4A7C15ull ) >> 56 ) & uint32_t( kBuckets - 1 ); + } + + // the reference an id takes: the file's own first-use entry, appended on + // first use. The MESSAGE form names no id at all: its references are + // compile-time slots of the announced vocabulary (docs/SPEC-TABLES.md §3.3). + MAPDEMO_TABLE_INLINE uint64_t ref( uint64_t id ) + { + const uint32_t b = bucket_of( id ); + for ( int32_t i = head[b]; i >= 0; i = chain[i] ) + { + if ( ids[i] == id ) { return uint64_t( i ) + 1; } + } + if ( count >= kCapacity ) { overflow = true; return 1; } + ids[count] = id; chain[count] = head[b]; head[b] = count; count++; + return uint64_t( count ); + } + + // undo every entry appended since mark. An entry removed is the most + // recent one in its bucket, so it sits at that bucket's head. + void truncate( int32_t mark ) + { + while ( count > mark ) + { + count--; + head[ bucket_of( ids[count] ) ] = chain[count]; + } + } +}; + +// TableIdsBytes is the trailer's own size: the entries, each a fixed +// little-endian u64, and the ENTRY COUNT, the one fixed-width number on the +// wire (docs/SPEC-TABLES.md §3). +inline int64_t TableIdsBytes( const TableIds & ids ) { return int64_t( ids.count ) * 8 + 8; } + +// TableIdsWrite puts the trailer where the walk ended: a writer never patches, +// because first-use order is known only when the walk ends. +inline void TableIdsWrite( TableWriter & w, const TableIds & ids ) +{ + for ( int32_t i = 0; i < ids.count; i++ ) { w.put64( ids.ids[i] ); } + w.put64( uint64_t( ids.count ) ); +} + +// THE ID TABLE, READER SIDE (docs/SPEC-TABLES.md §3). A reader locates it from +// the END of the wire and resolves it ONCE, at open: the entries are eight +// bytes each and a body names them by position, so every field dispatches +// through an index rather than through a search over hashes. +struct TableIdTable +{ + const uint8_t * entries = NULL; + int64_t count = 0; + + // the id a reference names. ref is 1-based and bounds-checked by the + // caller: a reference ABOVE the entry count is framing damage on the body + // that carries it, and 0 names no id at all. + uint64_t at( uint64_t ref ) const + { + const uint8_t * e = entries + ( ref - 1 ) * 8; + uint64_t lo = uint64_t( e[0] ) | uint64_t( e[1] ) << 8 | uint64_t( e[2] ) << 16 | uint64_t( e[3] ) << 24; + uint64_t hi = uint64_t( e[4] ) | uint64_t( e[5] ) << 8 | uint64_t( e[6] ) << 16 | uint64_t( e[7] ) << 24; + return lo | ( hi << 32 ); + } +}; + +struct TableReader +{ + const uint8_t * buffer; + int64_t size; + int64_t offset = 0; + TableReport * report; + const TableIdTable * ids = NULL; + // ONLY THE ROOT BODY CARRIES THE NODE TABLE (docs/SPEC-TABLES.md §3.1), so + // a body has to know which it is: the reserved id inside a NESTED body is + // malformed, because a second numbering cannot exist. Every reader made + // for a payload is nested; the two the wire surfaces make for a root say so. + bool nested = true; + + TableReader( const uint8_t * from_buffer, int64_t from_size, TableReport * to_report ) + : buffer( from_buffer ), size( from_size ), report( to_report ) {} + + TableReader( const uint8_t * from_buffer, int64_t from_size, TableReport * to_report, const TableIdTable * to_ids ) + : buffer( from_buffer ), size( from_size ), report( to_report ), ids( to_ids ) {} + + MAPDEMO_TABLE_INLINE bool has( int64_t bytes ) const { return offset + bytes <= size; } + // A LENGTH IS A 64-BIT NUMBER AND A BUFFER IS NOT (docs/SPEC-TABLES.md + // §3): every length, count and index on this wire has sixty-four bits of + // capability, so one past what remains must be compared UNSIGNED. Casting + // it to int64 first turns 0xFFFFFFFFFFFFFFFF into -1, and a negative + // length looks like room. + MAPDEMO_TABLE_INLINE bool room( uint64_t bytes ) const { return bytes <= (uint64_t) ( size - offset ); } + MAPDEMO_TABLE_INLINE uint8_t get8() { return buffer[offset++]; } + MAPDEMO_TABLE_INLINE uint16_t get16() { uint16_t v = uint16_t( buffer[offset] ) | uint16_t( buffer[offset+1] ) << 8; offset += 2; return v; } + MAPDEMO_TABLE_INLINE uint32_t get32() { uint32_t v = uint32_t( buffer[offset] ) | uint32_t( buffer[offset+1] ) << 8 | uint32_t( buffer[offset+2] ) << 16 | uint32_t( buffer[offset+3] ) << 24; offset += 4; return v; } + MAPDEMO_TABLE_INLINE uint64_t get64() { uint64_t lo = get32(); uint64_t hi = get32(); return lo | ( hi << 32 ); } + MAPDEMO_TABLE_INLINE void get128( uint64_t & lo, uint64_t & hi ) { lo = get64(); hi = get64(); } + + // ONE CANONICAL UNSIGNED LEB128 (docs/SPEC-TABLES.md §3), and a + // non-minimal spelling is MALFORMED: 0x80 0x00 and 0x00 both spell zero, + // and only the second is legal input. An encoding past ten bytes, or a + // tenth byte with a bit above the 64th value bit, is malformed on the same + // rule. false = framing damage on the body carrying it. + bool getleb( uint64_t & value ) + { + // A NUMBER THIS READER REFUSES LEAVES THE CURSOR WHERE IT WAS. The + // caller's next question is often "did this body end exactly at its + // L", and a rejected number that had moved the cursor would answer + // that question with the damage already stepped over. + const int64_t at = offset; + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( !has( 1 ) ) { offset = at; return false; } + const uint8_t b = get8(); + if ( i == 9 && b > 1 ) { offset = at; return false; } + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) + { + if ( i > 0 && b == 0 ) { offset = at; return false; } // a redundant continuation + return true; + } + shift += 7; + } + offset = at; + return false; + } + + // resolve one id reference against the file's table. false = a reference + // ABOVE the entry count, or a 0 where an id is required, both of which + // are framing damage on the body that carries it. + bool getid( uint64_t & id ) + { + uint64_t ref = 0; + if ( !getleb( ref ) ) { return false; } + if ( ref == 0 || ids == NULL || ref > (uint64_t) ids->count ) { return false; } + id = ids->at( ref ); + return true; + } + + // skip one payload by kind; false = framing damage. FOUR RULES COVER THE + // SET (docs/SPEC-TABLES.md §3), and a kind outside it is not skippable — + // which is why the set is closed and why kind 31 exists. + bool skip( uint8_t kind ) + { + switch ( kind ) + { + // the fixed-width kinds, each by its width: 18-29 are the 128-bit integers and + // the fixed-point family at every storage width (docs/SPEC-TABLES.md §3) + case 1: case 2: case 6: case 20: case 25: return has( 1 ) ? ( offset += 1, true ) : false; + case 3: case 7: case 21: case 26: return has( 2 ) ? ( offset += 2, true ) : false; + case 4: case 8: case 10: case 22: case 27: return has( 4 ) ? ( offset += 4, true ) : false; + case 5: case 9: case 11: case 23: case 28: return has( 8 ) ? ( offset += 8, true ) : false; + case 18: case 19: case 24: case 29: return has( 16 ) ? ( offset += 16, true ) : false; + case 17: case 30: // a NODE INDEX (§3.1) and an ENUM's variant reference: one LEB128 and stop + { + uint64_t ignored = 0; + return getleb( ignored ); + } + case 12: case 13: case 14: case 16: case 31: case 32: case 33: // 31 is the ESCAPE, 32 the payload-free kind, 33 wide text + { + uint64_t n = 0; + if ( !getleb( n ) ) return false; + return room( n ) ? ( offset += (int64_t) n, true ) : false; + } + case 15: // union: the arm id reference, then its kind, its L and its payload (reference 0 = empty) + { + uint64_t arm = 0; + if ( !getleb( arm ) ) return false; + if ( arm == 0 ) return true; + if ( !has( 1 ) ) return false; + offset += 1; // the arm's kind byte + uint64_t n = 0; + if ( !getleb( n ) ) return false; + return room( n ) ? ( offset += (int64_t) n, true ) : false; + } + // KIND 34 IS RESERVED FOR float16 AND IS NOT PART OF THIS MAJOR (§3): + // no writer emits it and no reader has a rule for it, so a reader + // meets it only as DAMAGE, exactly as it meets 35 or 200. A bare 34 + // is a writer that ignored the escape kind 31. + case 34: return false; + } + return false; + } +}; + + +// WIDENING (docs/SPEC-TABLES.md §4): a payload under a kind BELOW the reader's +// on the same ladder decodes exactly. The signed ladder is kinds 2, 3, 4, 5, +// 18, the unsigned one 6, 7, 8, 9, 19, and 10 into 11 is the float rung. Every +// other pair is a kind mismatch. The declared kind is a constant at every call +// site, so this folds to one or two comparisons on the mismatch path and to +// nothing on the matching one. +inline bool TableKindWidens( uint8_t kind, uint8_t declared ) +{ + switch ( declared ) + { + case 3: case 4: case 5: return kind >= 2 && kind < declared; + case 18: return kind >= 2 && kind <= 5; + case 7: case 8: case 9: return kind >= 6 && kind < declared; + case 19: return kind >= 6 && kind <= 9; + case 11: return kind == 10; + } + return false; +} + +// a fixed-width kind's payload width, for the one place the width is a +// runtime fact: an arm whose kind byte the reader widens, whose L must be the +// wire kind's own width (§3) +inline int64_t TableKindWidth( uint8_t kind ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: return 1; + case 3: case 7: case 21: case 26: return 2; + case 4: case 8: case 10: case 22: case 27: return 4; + case 5: case 9: case 11: case 23: case 28: return 8; + case 18: case 19: case 24: case 29: return 16; + } + return 0; +} + +// the payload of a kind on the SIGNED ladder (2 to 5), sign-extended to +// sixty-four bits; false = the body cannot cover it, which is framing damage +inline bool TableReadSignedAt( TableReader & r, uint8_t kind, int64_t & out ) +{ + switch ( kind ) + { + case 2: if ( !r.has( 1 ) ) { return false; } out = (int8_t) r.get8(); return true; + case 3: if ( !r.has( 2 ) ) { return false; } out = (int16_t) r.get16(); return true; + case 4: if ( !r.has( 4 ) ) { return false; } out = (int32_t) r.get32(); return true; + default: if ( !r.has( 8 ) ) { return false; } out = (int64_t) r.get64(); return true; + } +} + +// the payload of a kind on the UNSIGNED ladder (6 to 9), zero-extended +inline bool TableReadUnsignedAt( TableReader & r, uint8_t kind, uint64_t & out ) +{ + switch ( kind ) + { + case 6: if ( !r.has( 1 ) ) { return false; } out = r.get8(); return true; + case 7: if ( !r.has( 2 ) ) { return false; } out = r.get16(); return true; + case 8: if ( !r.has( 4 ) ) { return false; } out = r.get32(); return true; + default: if ( !r.has( 8 ) ) { return false; } out = r.get64(); return true; + } +} + +// f32 into f64, exact: a NaN's payload is data and rides on the bits, since +// the hardware conversion would set the quiet bit (§4) +inline double TableWidenF32( uint32_t bits ) +{ + if ( ( bits & 0x7F800000u ) == 0x7F800000u && ( bits & 0x007FFFFFu ) != 0 ) + { + const uint64_t sign = (uint64_t) ( bits >> 31 ) << 63; + const uint64_t payload = (uint64_t) ( bits & 0x007FFFFFu ) << 29; + const uint64_t nan_bits = sign | 0x7FF0000000000000ull | payload; + double d; memcpy( &d, &nan_bits, 8 ); return d; + } + float f; memcpy( &f, &bits, 4 ); return (double) f; +} + +// ILL-FORMED TEXT IS DAMAGE (docs/SPEC-TABLES.md §3, §4): a kind 12 payload is +// well-formed UTF-8 with no zero byte among its bytes, checked AS IT ARRIVES +// and before the reader's own bound, because a payload that is not text is not +// text at whatever length the reader would have kept. Rejects a zero byte, a +// truncated sequence, a bare continuation, an overlong encoding, a surrogate +// and a code point past U+10FFFF, which is SPEC.md §4.7's rule in this wire's +// idiom: the field reads its declared default, one malformed counts, and the +// parent reads on past L. +// +// A LENGTH IS A 64-BIT NUMBER (§3), so it arrives as one: a payload length is +// whatever the wire spelled, and narrowing it to a signed count would read +// 0xFFFFFFFFFFFFFFFF as an empty payload. +inline bool TableUtf8Valid( const uint8_t * bytes, uint64_t length ) +{ + uint64_t i = 0; + while ( i < length ) + { + const uint8_t lead = bytes[i]; + uint64_t continuations; + uint32_t code_point; + if ( lead == 0 ) { return false; } + if ( lead < 0x80 ) { i++; continue; } + else if ( ( lead & 0xE0 ) == 0xC0 ) { continuations = 1; code_point = lead & 0x1F; } + else if ( ( lead & 0xF0 ) == 0xE0 ) { continuations = 2; code_point = lead & 0x0F; } + else if ( ( lead & 0xF8 ) == 0xF0 ) { continuations = 3; code_point = lead & 0x07; } + else { return false; } + if ( i + continuations >= length ) { return false; } + for ( uint64_t k = 1; k <= continuations; k++ ) + { + if ( ( bytes[i + k] & 0xC0 ) != 0x80 ) { return false; } + code_point = ( code_point << 6 ) | uint32_t( bytes[i + k] & 0x3F ); + } + if ( continuations == 1 && code_point < 0x80 ) { return false; } + if ( continuations == 2 && ( code_point < 0x800 || ( code_point >= 0xD800 && code_point <= 0xDFFF ) ) ) { return false; } + if ( continuations == 3 && ( code_point < 0x10000 || code_point > 0x10FFFF ) ) { return false; } + i += 1 + continuations; + } + return true; +} + +// A CLAMP CUTS AT A CODE POINT BOUNDARY (§3, §16.2): the last whole code point +// that fits within the bound, over a payload the check above already accepted, +// so a clamp can never invent ill-formed storage. +// +// THE ANSWER IS NEVER ABOVE THE BOUND. The length arrives as the wire's own +// 64-bit number and the caller turns the answer back into the size of a copy, +// so a length no reader could have bounded has to leave here bounded: taken as +// a signed count, 0xFFFFFFFFFFFFFFFF is -1, -1 is under every bound, and the +// copy would run at SIZE_MAX. +inline int64_t TableUtf8Clamp( const uint8_t * bytes, uint64_t length, int64_t bound ) +{ + if ( length <= (uint64_t) bound ) { return (int64_t) length; } + int64_t cut = bound; + while ( cut > 0 && ( bytes[cut] & 0xC0 ) == 0x80 ) { cut--; } + return cut; +} + +// ONE CODE UNIT off the wire: two bytes LITTLE-ENDIAN, this wire's order for +// every fixed-width number (docs/SPEC-TABLES.md §3). No unit can exceed +// 0xFFFF, because two bytes cannot spell one. +inline uint16_t TableUtf16Unit( const uint8_t * bytes, int64_t index ) +{ + return uint16_t( uint16_t( bytes[index * 2] ) | ( uint16_t( bytes[index * 2 + 1] ) << 8 ) ); +} + +// ILL-FORMED WIDE TEXT IS DAMAGE (docs/SPEC-TABLES.md §3, §4): a kind 33 +// payload carrying an UNPAIRED SURROGATE or a ZERO CODE UNIT among its units, +// checked AS IT ARRIVES and before the reader's own bound, on the rule kind 12 +// takes for UTF-8. An ODD L is framing damage and the caller rejects it ahead +// of this, because units is L / 2. SPEC.md §4.12 refuses the same content +// TERMINALLY on the packet wire; here the field reads its declared default, +// one malformed counts, and the parent reads on past L. +inline bool TableUtf16Valid( const uint8_t * bytes, int64_t units ) +{ + int64_t i = 0; + while ( i < units ) + { + const uint16_t unit = TableUtf16Unit( bytes, i ); + if ( unit == 0 ) { return false; } + if ( unit >= 0xD800 && unit <= 0xDBFF ) + { + if ( i + 1 >= units ) { return false; } // a high surrogate with no low half + const uint16_t low = TableUtf16Unit( bytes, i + 1 ); + if ( low < 0xDC00 || low > 0xDFFF ) { return false; } + i += 2; + continue; + } + if ( unit >= 0xDC00 && unit <= 0xDFFF ) { return false; } // a low surrogate first + i++; + } + return true; +} + +// A CLAMP CUTS AT A CODE UNIT BOUNDARY AND NEVER SPLITS A PAIR (§3, §16.2): +// the first bound units of a payload the check above already accepted, and +// where the last kept unit is a HIGH SURROGATE whose low half did not fit, +// that unit is dropped with it. So a clamp can never invent an unpaired +// surrogate, exactly as kind 12's clamp can never invent a broken sequence. +inline int64_t TableUtf16Clamp( const uint8_t * bytes, int64_t units, int64_t bound ) +{ + if ( units <= bound ) { return units; } + int64_t cut = bound; + if ( cut > 0 ) + { + const uint16_t last = TableUtf16Unit( bytes, cut - 1 ); + if ( last >= 0xD800 && last <= 0xDBFF ) { cut--; } + } + return cut; +} + +// The RESERVED node-table id, the one id the language holds back +// (docs/SPEC-TABLES.md §3.1, §5). It rides in every unit, pointered or not, +// because every body has to know that a NESTED body claiming one is damaged. +static const uint64_t kTableNodeTableFieldId = 0xFFFFFFFFFFFFFFFFull; + +// TableWireForm is the FORM BYTE, and it is the whole header +// (docs/SPEC-TABLES.md §3). A reader that meets a byte it does not know +// refuses the wire by name and never reports damage. +const uint8_t kTableWireForm = 1; + +// TableOpen reads the form byte and the trailer, in that order, and hands back +// the ROOT BODY. It answers one of three verdicts, because five zero counters +// and a false flag are what a clean read prints too: +// +// TableOpenOk the form is known and the table read whole +// TableOpenRefused a FORM BYTE this reader does not carry: nothing is +// decoded, nothing is counted, and no damage is reported +// TableOpenDamaged a table that cannot be read whole — fewer than eight +// bytes, a count whose entries run past the front of the +// file, a count that leaves no room for the form byte, or +// ONE ID IN TWO ENTRIES. The whole wire is malformed, +// nothing is decoded, and one event is counted. +// TableOpenBodyStopped the form and the table were good and the ROOT BODY +// could not be walked to its own terminator. What it +// decoded before that is kept, as everywhere on this wire. +enum TableOpenVerdict { TableOpenOk, TableOpenRefused, TableOpenDamaged, TableOpenBodyStopped }; + +inline TableOpenVerdict TableOpen( const uint8_t * buffer, int64_t bytes, TableIdTable & table, int64_t & body_bytes ) +{ + if ( bytes < 1 ) { return TableOpenDamaged; } + if ( buffer[0] != kTableWireForm ) { return TableOpenRefused; } + if ( bytes < 9 ) { return TableOpenDamaged; } + const uint8_t * tail = buffer + bytes - 8; + uint64_t lo = uint64_t( tail[0] ) | uint64_t( tail[1] ) << 8 | uint64_t( tail[2] ) << 16 | uint64_t( tail[3] ) << 24; + uint64_t hi = uint64_t( tail[4] ) | uint64_t( tail[5] ) << 8 | uint64_t( tail[6] ) << 16 | uint64_t( tail[7] ) << 24; + uint64_t count = lo | ( hi << 32 ); + if ( count > (uint64_t) ( bytes / 8 ) ) { return TableOpenDamaged; } + const int64_t span = (int64_t) count * 8 + 8; + if ( span + 1 > bytes ) { return TableOpenDamaged; } + table.entries = buffer + bytes - span; + table.count = (int64_t) count; + // THE ENTRIES ARE DISTINCT: a table that carries one id twice is malformed + // for the whole wire, because no wire this schema writes carries a repeat + // and it would leave one more shape of table for a hostile writer to aim + // at (docs/SPEC-TABLES.md §3). + for ( int64_t i = 1; i < table.count; i++ ) + { + const uint64_t id = table.at( uint64_t( i ) + 1 ); + for ( int64_t j = 0; j < i; j++ ) + { + if ( table.at( uint64_t( j ) + 1 ) == id ) { return TableOpenDamaged; } + } + } + body_bytes = bytes - span - 1; + return TableOpenOk; +} + +// TableBodyExtent walks a body's framing to the zero reference that ends it, +// so a reader can tell a body that ENDED EARLY — leaving bytes no field claims +// — from one that is merely damaged. ANY BYTE BETWEEN THE ROOT'S TERMINATOR +// AND THE TABLE'S FIRST ENTRY IS MALFORMED, because no field claims it and the +// two ends of the file have met (docs/SPEC-TABLES.md §3). +inline bool TableBodyEndsEarly( const uint8_t * body, int64_t bytes, const TableIdTable & table ) +{ + TableReport ignored; + TableReader r( body, bytes, &ignored, &table ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.getleb( ref ) ) { return false; } + if ( ref == 0 ) { return r.offset != bytes; } + if ( ref > (uint64_t) table.count ) { return false; } + if ( !r.has( 1 ) ) { return false; } + if ( !r.skip( r.get8() ) ) { return false; } + } +} + +// THE MESSAGE FORM (docs/SPEC-TABLES.md §3.3): a batch of BITPACKED bodies +// under one announced vocabulary. +// +// A form 2 wire is THREE PARTS: the form byte, the body count, and the bodies +// as one continuous bit stream, zero-padded to the next byte at the end and +// nowhere else. A body is a sequence of fields, each a REFERENCE followed by a +// PAYLOAD and nothing else: no kind byte and no length, because the +// announcement carries the kind and the shape of every entry. +const uint8_t kTableWireMessageForm = 2; + +// THE COUNT IS A RANGED INTEGER OVER [1, 256], eight bits carrying M - 1. 256 +// is a WIRE CONSTANT of this form rather than a receiver's policy, because the +// count's WIDTH depends on it and two peers that disagreed on the width would +// not be reading the same wire. A batch of zero is not spellable. +static const int64_t kTableMessageBatchMax = 256; + +// The RESERVED ids of the announcement's own two fields (§5, §11), beside the +// node table's. They are the announcement's transport, they never appear in a +// body, and they take no slot in the vocabulary. +static const uint64_t kTableBuildVersionFieldId = 0xFFFFFFFFFFFFFFFEull; +static const uint64_t kTableMessageVocabularyFieldId = 0xFFFFFFFFFFFFFFFDull; + +// THE WIDEST COUNT THIS FORM SPELLS, which is the count an UNBOUNDED array +// announces (§2.9): an unbounded array states no bound, so the announcement +// states the widest one a batch could carry. It is the ceiling an array's or a +// keyed entry's announced min and max are checked against. +static const uint64_t kTableMessageListMax = 0xFFFFFFFFull; + +// THIS UNIT'S OWN REFERENCE WIDTH: the bits a writer spends on every reference +// of every body it writes, which is a compile-time constant because the +// vocabulary is. A READER spends the width the SENDER's vocabulary settles. +static const int64_t kTableMessageRefBitsHere = 7; + +// THIS UNIT'S OWN ENTRY COUNT, which is the CAPACITY a receiver declares for +// its resolved vocabulary when it talks only to peers of this schema (§3.3). +// The vocabulary is a pure function of the build version, so a peer at this +// build announces exactly this many entries; a receiver that means to meet +// OTHER builds declares more, and an announcement above whatever it declared +// is refused as vocabulary_too_large. +static const int64_t kTableMessageEntriesHere = 75; + +// The reserved NODE-TABLE id's own slot in this unit's vocabulary (§3.3). A +// pointered body names the node table through it, and the node table is the +// ROOT body's FIRST field because a pointer index's width is settled by the +// node count it carries. +static const uint64_t kTableNodeTableFieldSlot = 54; + +// THE BIT STREAM the bodies ride on (§3.3). It is the packet wire's own +// layout, bit i of the stream in byte i/8 at bit position i%8 low bit first, +// so a value written here and a value written by a generated packet writer are +// the same bits in the same places. + +// EIGHT BYTES OF THE STREAM AS ONE WORD, and the word is LITTLE-END-FIRST +// whatever order this host is in, because the stream's own definition puts +// bit i in byte i/8: byte 0 of the run holds the word's low eight bits. That +// is what lets one value of any width move in one unaligned load or store +// instead of one touch a byte, and the BITS ON THE WIRE do not move. +inline uint64_t table_message_byteswap64( uint64_t v ) +{ + return ( v >> 56 ) | ( ( v >> 40 ) & 0xff00ull ) | ( ( v >> 24 ) & 0xff0000ull ) | ( ( v >> 8 ) & 0xff000000ull ) + | ( ( v << 8 ) & 0xff00000000ull ) | ( ( v << 24 ) & 0xff0000000000ull ) | ( ( v << 40 ) & 0xff000000000000ull ) + | ( v << 56 ); +} + +inline uint64_t table_message_load64( const uint8_t * p ) +{ + uint64_t v = 0; + memcpy( &v, p, 8 ); +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + v = table_message_byteswap64( v ); +#endif + return v; +} + +inline void table_message_store64( uint8_t * p, uint64_t v ) +{ +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + v = table_message_byteswap64( v ); +#endif + memcpy( p, &v, 8 ); +} + +struct TableBitWriter +{ + uint8_t * buffer; + int64_t capacity; // bytes + int64_t bits; + bool overflow; + + TableBitWriter() : buffer( NULL ), capacity( 0 ), bits( 0 ), overflow( false ) {} + TableBitWriter( uint8_t * to_buffer, int64_t to_capacity ) : buffer( to_buffer ), capacity( to_capacity ), bits( 0 ), overflow( false ) {} + + // ONE WORD AT A TIME, never one bit and never one byte of arithmetic: the + // value is shifted into place in a REGISTER once, and the bytes it + // occupies are stored from that register with no read back. A sixty-four + // bit field costs one shift rather than nine masked read-modify-writes. + // The word is assembled little-end-first, so the BITS ON THE WIRE are the + // same bits in the same places, bit i in byte i/8 at position i%8 with the + // low bit first, which is what the pinned goldens hold. IT WRITES EXACTLY + // THE BYTES THE VALUE OCCUPIES and never one past them, so a caller's + // buffer beyond the batch is its own. + void put( uint64_t value, int64_t n ) + { + if ( n <= 0 ) { return; } + if ( ( bits + n + 7 ) / 8 > capacity ) { overflow = true; bits += n; return; } + if ( n < 64 ) { value &= ( uint64_t( 1 ) << n ) - 1; } // a caller's high bits never leak + const int64_t index = bits >> 3; + const int64_t bit = bits & 7; + // the byte the write STARTS in keeps the bits already written to it, + // and every byte after it is this value's own + const uint64_t head = bit != 0 ? ( uint64_t( buffer[index] ) & ( ( uint64_t( 1 ) << bit ) - 1 ) ) : 0; + const uint64_t word = head | ( value << bit ); + const int64_t need = ( bit + n + 7 ) >> 3; // 1 to 9 bytes + if ( need >= 8 ) + { + table_message_store64( buffer + index, word ); + if ( need > 8 ) { buffer[index + 8] = uint8_t( value >> ( 64 - bit ) ); } + } + else + { + for ( int64_t i = 0; i < need; i++ ) { buffer[index + i] = uint8_t( word >> ( 8 * i ) ); } + } + bits += n; + } + + // THE ALIGN IS WHAT BUYS THIS (docs/SPEC-TABLES.md §3.3): a string(N), a + // bytes(N) and a blob record align before their bytes precisely so the + // largest payload on the wire moves as ONE memcpy. Off a boundary there is + // nothing to memcpy and the bytes go through put. + void putbytes( const uint8_t * data, int64_t n ) + { + if ( n <= 0 ) { return; } + if ( ( bits & 7 ) == 0 ) + { + if ( ( bits >> 3 ) + n > capacity ) { overflow = true; bits += n * 8; return; } + memcpy( buffer + ( bits >> 3 ), data, (size_t) n ); + bits += n * 8; + return; + } + for ( int64_t i = 0; i < n; i++ ) { put( (uint64_t) data[i], 8 ); } + } + + // a string's or a bytes' payload ALIGNS before its bytes, and a batch + // aligns once at its end. Both are zero fill, spent in one call. + void align() { put( 0, ( 8 - ( bits & 7 ) ) & 7 ); } +}; + +// TableAlignBits is what an align costs from a bit position, which a measure +// spends exactly where a save does. +inline int64_t TableAlignBits( int64_t bits ) { return ( 8 - ( bits % 8 ) ) % 8; } + +struct TableBitReader +{ + const uint8_t * buffer; + int64_t bits; // the stream's extent, in bits + int64_t offset; // bits consumed + + TableBitReader() : buffer( NULL ), bits( 0 ), offset( 0 ) {} + TableBitReader( const uint8_t * from_buffer, int64_t from_bytes ) : buffer( from_buffer ), bits( from_bytes * 8 ), offset( 0 ) {} + + bool has( int64_t n ) const { return n >= 0 && offset + n <= bits; } + + // the primitive is sixty-four bits, and a width above it is refused + // here as well as at the announcement: no field on any body can ask this + // reader to move more bits than it holds + // ONE WORD OF THE BUFFER AT A TIME, the mirror of the writer's put: the + // eight bytes the value starts in load as one little-end-first word and a + // ninth byte carries the spill a value that straddles the word needs. + // Within nine bytes of the stream's end there is no room for a word load + // and the bytes come one at a time, by the same arithmetic. + bool get( uint64_t & value, int64_t n ) + { + if ( n > 64 || !has( n ) ) { return false; } + value = 0; + if ( n == 0 ) { return true; } + const int64_t index = offset >> 3; + const int64_t bit = offset & 7; + const int64_t bytes = ( bits + 7 ) >> 3; + if ( index + 9 <= bytes ) + { + uint64_t v = table_message_load64( buffer + index ) >> bit; + if ( bit != 0 && bit + n > 64 ) { v |= uint64_t( buffer[index + 8] ) << ( 64 - bit ); } + value = n == 64 ? v : ( v & ( ( uint64_t( 1 ) << n ) - 1 ) ); + offset += n; + return true; + } + int64_t got = 0; + while ( got < n ) + { + const int64_t byte = offset >> 3; + const int64_t off = offset & 7; + const int64_t room = 8 - off; + const int64_t take = ( n - got ) < room ? ( n - got ) : room; + const uint64_t chunk = ( uint64_t( buffer[byte] ) >> off ) & ( ( uint64_t( 1 ) << take ) - 1 ); + value |= chunk << got; + offset += take; + got += take; + } + return true; + } + + // the bytes of an ALIGNED payload, which is the read side of the memcpy + // the align buys (docs/SPEC-TABLES.md §3.3) + bool getbytes( uint8_t * out, int64_t n ) + { + if ( n < 0 || !has( n * 8 ) ) { return false; } + if ( ( offset & 7 ) == 0 ) + { + memcpy( out, buffer + ( offset >> 3 ), (size_t) n ); + offset += n * 8; + return true; + } + for ( int64_t i = 0; i < n; i++ ) + { + uint64_t by = 0; + if ( !get( by, 8 ) ) { return false; } + out[i] = (uint8_t) by; + } + return true; + } + + bool skip( int64_t n ) { if ( !has( n ) ) { return false; } offset += n; return true; } + + // the pad to the next byte boundary is VERIFIED ZERO, which is the packet + // wire's rule for the same reason (SPEC.md §4.3) + bool align() + { + const int64_t pad = ( 8 - ( offset & 7 ) ) & 7; + if ( pad == 0 ) { return true; } + uint64_t bits_read = 0; + return get( bits_read, pad ) && bits_read == 0; + } +}; + +// TableBitsRequired is bits_required( min, max ): the bit length of max - min, +// and zero where the two are equal, which is a value that spends no bit at all. +inline int64_t TableBitsRequired( int64_t min, int64_t max ) +{ + if ( max <= min ) { return 0; } + uint64_t span = (uint64_t) ( max - min ); + int64_t n = 0; + while ( span > 0 ) { n++; span >>= 1; } + return n; +} + +// THE ANNOUNCED ENTRY (§3.3): an id, a kind, and a SHAPE, which is the width +// and range facts a reader needs to SKIP a field exactly and to DECODE one +// whose own declaration has moved. One name may take TWO entries, at two kinds or two +// shapes, and a body names the one it means. +// +// The ELEMENT's own facts ride beside the field's because an array's element +// is the one nesting this wire has: an array of arrays is not a table-wire +// construct, so one level is every level. +// +// IT IS THE RESOLVED ENTRY AND THE CALLER SIZES AN ARRAY OF THEM, so it +// carries what a DECODE takes and nothing a decode does not: qmin, qdelta and +// qcount are what SPEC.md §4.3's rule leaves behind, and the qmax and qres +// that rule CONSUMES are locals of the parse. The widths are int16 because a +// width is bounded by the kind it came under and no kind holds more than 128 +// bits. +struct TableMessageEntry +{ + uint64_t id = 0; + int64_t min = 0; // an array's minimum count + int64_t max = 0; // an array's maximum count, a string's capacity, a keyed array's slots + int64_t base_lo = 0; // the ranged base, low half: a signed kind's sign-extends, an unsigned kind's is whole + int64_t base_hi = 0; // its high half, for a 128-bit kind + int64_t elem_max = 0; + int64_t elem_base_lo = 0; + int64_t elem_base_hi = 0; + // what SPEC.md §4.3's derivation leaves: the base, the step and the count + float qmin = 0.0f; + float qdelta = 0.0f; + uint32_t qcount = 0; + float elem_qmin = 0.0f; + float elem_qdelta = 0.0f; + uint32_t elem_qcount = 0; + // THE PAYLOAD'S WIDTH, RESOLVED: what the kind, the packing and the + // announced bits together say, computed once at AnnounceRead, and -1 + // where the payload is not a fixed-width value at all + int16_t value_bits = -1; + int16_t elem_value_bits = -1; + uint8_t kind = 0; + uint8_t packing = 0; + uint8_t elem_kind = 0; + uint8_t elem_packing = 0; +}; + +// TableMessageEntrySame reports whether two RESOLVED entries carry the same +// shape, which is every fact of the entry but its id and its kind. It is what +// the announcement's duplicate rule is asked in: two entries that agree on all +// three parts are malformed (§3.3). +inline bool TableMessageEntrySame( const TableMessageEntry & a, const TableMessageEntry & b ) +{ + return a.min == b.min && a.max == b.max && a.base_lo == b.base_lo && a.base_hi == b.base_hi + && a.elem_max == b.elem_max && a.elem_base_lo == b.elem_base_lo && a.elem_base_hi == b.elem_base_hi + && a.qmin == b.qmin && a.qdelta == b.qdelta && a.qcount == b.qcount + && a.elem_qmin == b.elem_qmin && a.elem_qdelta == b.elem_qdelta && a.elem_qcount == b.elem_qcount + && a.value_bits == b.value_bits && a.elem_value_bits == b.elem_value_bits + && a.packing == b.packing && a.elem_kind == b.elem_kind && a.elem_packing == b.elem_packing; +} + +// TableMessageKindBits is the widest RANGED value a kind can carry, its own +// storage width: a width above it is a hostile width on the announcement. +inline int64_t TableMessageKindBits( uint8_t kind ) +{ + switch ( kind ) + { + case 2: case 6: case 20: case 25: return 8; + case 3: case 7: case 21: case 26: return 16; + case 4: case 8: case 22: case 27: return 32; + case 5: case 9: case 23: case 28: return 64; + default: return 128; + } +} + +// TableMessageQuantization is SPEC.md §4.3's derivation over an announced +// triple, in float32 and by nothing else: delta, the step count and the +// width. False is a triple SPEC.md calls non-conforming, which on the +// announcement is a hostile width like any other (§3.3). +inline bool TableMessageQuantization( float qmin, float qmax, float qres, float & delta, uint32_t & count, int64_t & bits ) +{ + if ( !( qmin < qmax ) || !( qres > 0.0f ) ) { return false; } + delta = qmax - qmin; + float values = delta / qres; + if ( !( delta - delta == 0.0f ) || !( values - values == 0.0f ) ) { return false; } // Inf - Inf is NaN + if ( !( values >= 1.0f ) ) { values = 1.0f; } + else if ( values > 4294967040.0f ) { values = 4294967040.0f; } // the largest float below 2^32 + count = (uint32_t) values; + if ( (float) count < values ) { count++; } // ceil, on a value the cast holds exactly + bits = TableBitsRequired( 0, (int64_t) count ); + return true; +} + +// The two roundings on each side of the rule (SPEC.md §7.2): the product +// rounds to float32 BEFORE the add, which a compiler permitted to contract +// would otherwise fuse into one rounding and move the wire. +#if ( defined( __GNUC__ ) || defined( __clang__ ) ) && ( defined( __aarch64__ ) || defined( _M_ARM64 ) ) +#define TABLE_FLOAT_FORCE_ROUND( x ) __asm__ ( "" : "+w" ( x ) ) +#elif ( defined( __GNUC__ ) || defined( __clang__ ) ) && ( defined( __x86_64__ ) || defined( __i386__ ) ) +#define TABLE_FLOAT_FORCE_ROUND( x ) __asm__ ( "" : "+x" ( x ) ) +#else +#define TABLE_FLOAT_FORCE_ROUND( x ) do { volatile float table_float_force_round_slot = ( x ); ( x ) = table_float_force_round_slot; } while ( 0 ) +#endif + +// TableMessageQuantize is the writer's half: the index a value takes. +inline uint32_t TableMessageQuantize( float value, float qmin, float delta, uint32_t count ) +{ + float normalized = ( value - qmin ) / delta; + if ( !( normalized >= 0.0f ) ) { normalized = 0.0f; } + else if ( !( normalized <= 1.0f ) ) { normalized = 1.0f; } + float scaled = normalized * (float) count; + TABLE_FLOAT_FORCE_ROUND( scaled ); + uint32_t index = (uint32_t) ( scaled + 0.5f ); // floor of a non-negative value + if ( index > count ) { index = count; } + return index; +} + +// TableMessageDequantize is the reader's half: the float an index names. +inline float TableMessageDequantize( uint32_t index, float qmin, float delta, uint32_t count ) +{ + if ( index > count ) { index = count; } + const float normalized = index / (float) count; + float scaled = normalized * delta; + TABLE_FLOAT_FORCE_ROUND( scaled ); + return scaled + qmin; +} + +inline bool TableMessageIntegerKind( uint8_t kind ) +{ + return ( kind >= 2 && kind <= 9 ) || kind == 18 || kind == 19; +} + +inline bool TableMessageFixedKind( uint8_t kind ) { return kind >= 20 && kind <= 29; } + +inline bool TableMessageKnownKind( uint8_t kind ) +{ + return kind == 0 || ( kind >= 1 && kind <= 17 ) || ( kind >= 18 && kind <= 29 ) || ( kind >= 30 && kind <= 33 ); +} + +// A CANONICAL LEB128, which is the announcement's own integer: the +// announcement is a form 1 FILE and takes §3's rule. +inline bool TableMessageLeb( const uint8_t * in, int64_t size, int64_t & at, uint64_t & value ) +{ + value = 0; + for ( int64_t shift = 0; at < size; shift += 7 ) + { + if ( shift >= 64 ) { return false; } + const uint8_t by = in[ at++ ]; + value |= uint64_t( by & 0x7F ) << shift; + if ( ( by & 0x80 ) == 0 ) { return !( shift > 0 && by == 0 ); } + } + return false; +} + +// TableMessageShapeFacts is where one shape's facts land: the field's own, +// or its element's, which is the one nesting this wire has. +struct TableMessageShapeFacts +{ + uint8_t & packing; int64_t & value_bits; int64_t & base_lo; int64_t & base_hi; + float & qmin; float & qmax; float & qres; float & qdelta; uint32_t & qcount; + int64_t & min; int64_t & max; uint8_t & elem_kind; +}; + +inline bool TableMessageShapeRead( const uint8_t * in, int64_t size, int64_t & at, uint8_t kind, TableMessageShapeFacts f ); +inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t value_bits ); + +// TableMessageEntryRead parses ONE entry, and answers false for a HOSTILE +// SHAPE: bits above the kind's own domain, an array whose min exceeds its +// max, an element kind outside the closed set, a quantized triple SPEC.md +// calls non-conforming, or a shape running past the vocabulary's own bytes. +inline bool TableMessageEntryRead( const uint8_t * in, int64_t size, int64_t & at, TableMessageEntry & entry ) +{ + if ( at + 9 > size ) { return false; } + entry = TableMessageEntry(); + for ( int i = 0; i < 8; i++ ) { entry.id |= uint64_t( in[ at + i ] ) << ( 8 * i ); } + entry.kind = in[ at + 8 ]; + at += 9; + if ( !TableMessageKnownKind( entry.kind ) ) { return false; } + // The parse lands in LOCALS and the entry keeps what a decode reads: the + // quantized max and res are the derivation's inputs and never a field's. + uint8_t packing = 0, elem_kind = 0; + int64_t bits = 0, base_lo = 0, base_hi = 0, min = 0, max = 0; + float qmin = 0.0f, qmax = 0.0f, qres = 0.0f, qdelta = 0.0f; + uint32_t qcount = 0; + TableMessageShapeFacts own = { packing, bits, base_lo, base_hi, + qmin, qmax, qres, qdelta, qcount, + min, max, elem_kind }; + if ( !TableMessageShapeRead( in, size, at, entry.kind, own ) ) { return false; } + entry.packing = packing; + entry.value_bits = (int16_t) TableMessageValueBits( entry.kind, packing, bits ); + entry.base_lo = base_lo; + entry.base_hi = base_hi; + entry.qmin = qmin; + entry.qdelta = qdelta; + entry.qcount = qcount; + entry.min = min; + entry.max = max; + entry.elem_kind = elem_kind; + if ( entry.kind == 14 || entry.kind == 16 ) + { + uint8_t elem_packing = 0, inner_kind = 0; + int64_t elem_bits = 0, elem_base_lo = 0, elem_base_hi = 0, elem_min = 0, elem_max = 0; + float elem_qmin = 0.0f, elem_qmax = 0.0f, elem_qres = 0.0f, elem_qdelta = 0.0f; + uint32_t elem_qcount = 0; + TableMessageShapeFacts elem = { elem_packing, elem_bits, elem_base_lo, elem_base_hi, + elem_qmin, elem_qmax, elem_qres, elem_qdelta, elem_qcount, + elem_min, elem_max, inner_kind }; + if ( !TableMessageShapeRead( in, size, at, entry.elem_kind, elem ) ) { return false; } + entry.elem_packing = elem_packing; + entry.elem_value_bits = (int16_t) TableMessageValueBits( entry.elem_kind, elem_packing, elem_bits ); + entry.elem_base_lo = elem_base_lo; + entry.elem_base_hi = elem_base_hi; + entry.elem_qmin = elem_qmin; + entry.elem_qdelta = elem_qdelta; + entry.elem_qcount = elem_qcount; + entry.elem_max = elem_max; + } + return true; +} + +// TableMessageShapeRead is one shape, by the kind that names it (§3.3's shape +// table). Every number in it is a canonical LEB128 except where the row says +// otherwise: a RANGED BASE IS ENCODED BY ITS KIND'S SIGNEDNESS, zigzag for the +// signed kinds, unsigned for the unsigned kinds and sixteen bytes for the +// 128-bit and fixed-point kinds, and a QUANTIZED f32 carries min, max and res +// as float32, from which the step count and the width derive by SPEC.md +// §4.3's rule and by nothing else. +inline bool TableMessageShapeRead( const uint8_t * in, int64_t size, int64_t & at, uint8_t kind, TableMessageShapeFacts f ) +{ + uint64_t v = 0; + if ( TableMessageIntegerKind( kind ) || TableMessageFixedKind( kind ) || kind == 10 ) + { + if ( at >= size ) { return false; } + f.packing = in[ at++ ]; + if ( f.packing == 0 ) { return true; } + if ( f.packing == 1 && kind != 10 ) + { + if ( !TableMessageLeb( in, size, at, v ) || (int64_t) v > TableMessageKindBits( kind ) ) { return false; } + f.value_bits = (int64_t) v; + if ( kind == 18 || kind == 19 || TableMessageFixedKind( kind ) ) + { + if ( at + 16 > size ) { return false; } + uint64_t lo = 0, hi = 0; + for ( int i = 0; i < 8; i++ ) { lo |= uint64_t( in[ at + i ] ) << ( 8 * i ); } + for ( int i = 0; i < 8; i++ ) { hi |= uint64_t( in[ at + 8 + i ] ) << ( 8 * i ); } + f.base_lo = (int64_t) lo; f.base_hi = (int64_t) hi; + at += 16; + return true; + } + if ( !TableMessageLeb( in, size, at, v ) ) { return false; } + if ( kind >= 2 && kind <= 5 ) { f.base_lo = (int64_t) ( v >> 1 ) ^ -(int64_t) ( v & 1 ); } // zigzag + else { f.base_lo = (int64_t) v; } // the unsigned domain, whole + return true; + } + if ( f.packing == 2 && kind == 10 ) + { + if ( at + 12 > size ) { return false; } + uint32_t raw[3] = { 0, 0, 0 }; + for ( int k = 0; k < 3; k++ ) { for ( int i = 0; i < 4; i++ ) { raw[k] |= uint32_t( in[ at + 4 * k + i ] ) << ( 8 * i ); } } + at += 12; + memcpy( &f.qmin, &raw[0], 4 ); + memcpy( &f.qmax, &raw[1], 4 ); + memcpy( &f.qres, &raw[2], 4 ); + return TableMessageQuantization( f.qmin, f.qmax, f.qres, f.qdelta, f.qcount, f.value_bits ); + } + return false; // a packing outside the closed set + } + // A MAX ABOVE WHAT THE KIND CAN HOLD IS A HOSTILE WIDTH (§3.3). A string + // and a wide string are bounded by the int32 storage cap the checker + // applies to every N (SPEC §4.3, §6.1), and an array and a keyed entry by + // the 32-bit count an unbounded array announces (§2.9), which is the + // widest count this form spells. A larger bound is a shape no conforming + // declaration can produce, and a reader that carried it would do its + // length arithmetic in a range that overflows. + if ( kind == 12 || kind == 33 ) + { + if ( !TableMessageLeb( in, size, at, v ) || v > (uint64_t) INT32_MAX ) { return false; } + f.max = (int64_t) v; + return true; + } + if ( kind == 14 || kind == 16 ) + { + if ( kind == 14 ) + { + if ( !TableMessageLeb( in, size, at, v ) || v > kTableMessageListMax ) { return false; } + f.min = (int64_t) v; + } + if ( !TableMessageLeb( in, size, at, v ) || v > kTableMessageListMax ) { return false; } + if ( (int64_t) v < f.min ) { return false; } + f.max = (int64_t) v; + if ( at >= size ) { return false; } + f.elem_kind = in[ at++ ]; + if ( !TableMessageKnownKind( f.elem_kind ) ) { return false; } + // AND AN ELEMENT KIND OF 12 OR 33 IS REFUSED HERE, at the + // announcement, rather than at the skip that would meet it (§3.3): no + // declaration this language accepts is an array of string(N) or of + // wstring(N), so a shape announcing one is one rule's business and not + // two. + if ( f.elem_kind == 12 || f.elem_kind == 33 ) { return false; } + return true; + } + return true; +} + +// TableMessageValueBits is one value's width under a shape, and -1 where the +// kind's payload is not a fixed-width value at all. +inline int64_t TableMessageValueBits( uint8_t kind, uint8_t packing, int64_t value_bits ) +{ + if ( kind == 1 ) { return 1; } + if ( kind == 11 ) { return 64; } + if ( kind == 10 ) { return packing == 2 ? value_bits : 32; } + if ( TableMessageIntegerKind( kind ) || TableMessageFixedKind( kind ) ) + { + if ( packing == 1 ) { return value_bits; } + switch ( kind ) + { + case 2: case 6: case 20: case 25: return 8; + case 3: case 7: case 21: case 26: return 16; + case 4: case 8: case 22: case 27: return 32; + case 5: case 9: case 23: case 28: return 64; + default: return 128; + } + } + return -1; +} + +// THE UNIT'S ANNOUNCEMENT, byte for byte: 75 entries and 901 bytes. It is an +// ordinary form 1 FILE: the form byte, a body carrying the BUILD VERSION under +// the reserved id at kind 9 and the VOCABULARY under the reserved id at kind 14 +// over element kind 6, and a trailer of those two reserved ids. +// +// THE VOCABULARY IS A FIELD AND NOT THE TRAILER, and that buys three things: +// §3's writer rule that an id no body references is never written is restored +// unbroken, an entry can carry a KIND and a SHAPE which a trailer of bare ids +// cannot, and one NAME can appear at two shapes. +// +// The order is the COOK PROJECTION's (§20.2): each record in the order the +// projection renders it and each record's fields in the order the projection +// renders them, then each enum's variants and each union's arms. Then comes +// the tail the projection does not name: the reserved node-table id, the three +// blob type ids as bytes, string and wstring, and every table's own name id in +// the projection's sorted record order. The tail is UNCONDITIONAL, so an +// ordinary edit only ever grows it at its end and never moves a slot a +// generated field header carries as a literal. +static const int64_t kTableAnnounceBytes = 901; +static const uint8_t kTableAnnounce[ kTableAnnounceBytes ] = { + 0x01, 0x01, 0x09, 0xdd, 0x70, 0xc9, 0xd2, 0x68, 0x20, 0x4c, 0x19, 0x02, + 0x0e, 0xdd, 0x06, 0x06, 0xda, 0x06, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0xc9, 0x3d, 0x06, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, 0x36, + 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x08, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, + 0xe4, 0x7c, 0x11, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, + 0x20, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0d, 0xec, 0x10, + 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0x10, 0xec, 0x10, 0x5b, 0x36, + 0x19, 0x4a, 0xc9, 0x3d, 0x03, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, + 0xc9, 0x3d, 0x08, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x02, 0x02, 0x11, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, + 0x0e, 0x03, 0x03, 0x04, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, + 0x3d, 0x04, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, + 0x00, 0x0a, 0x06, 0x00, 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, + 0x07, 0x00, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x21, 0x06, + 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0c, 0x10, 0x38, 0x81, + 0x0a, 0xf1, 0x1f, 0x06, 0xa7, 0xa3, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0xe9, 0xea, 0x71, 0x6f, 0x0f, 0x01, 0x82, 0xbf, 0x04, 0x00, + 0x5b, 0x92, 0xde, 0x9c, 0xab, 0xea, 0xe2, 0x14, 0x0e, 0x00, 0xff, 0xff, + 0xff, 0xff, 0x0f, 0x0d, 0x90, 0x10, 0x39, 0x5e, 0x67, 0x94, 0xd5, 0x79, + 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, 0x30, + 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x02, 0x11, 0xaf, 0x5c, 0xca, 0x21, + 0x19, 0xaa, 0x08, 0x1a, 0x0d, 0x02, 0xfc, 0xa1, 0xce, 0xa2, 0x59, 0x64, + 0x1f, 0x0e, 0x00, 0x03, 0x0d, 0x5d, 0xf1, 0x50, 0x95, 0xf2, 0x1f, 0x55, + 0x70, 0x10, 0x02, 0x0d, 0xb5, 0xcc, 0x70, 0x05, 0x19, 0xc0, 0x56, 0xe7, + 0x0f, 0x9f, 0x76, 0x48, 0x6a, 0xa7, 0xda, 0x01, 0xab, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x4f, 0x81, 0x68, 0xf2, 0xff, 0x28, 0xb7, + 0xaf, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xec, 0x10, 0x5b, + 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x0c, 0xac, 0x02, 0x29, 0xbe, 0xb7, 0x2b, + 0x19, 0xea, 0x7d, 0x2b, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, + 0xec, 0x10, 0x5b, 0x36, 0x19, 0x4a, 0xc9, 0x3d, 0x09, 0x00, 0x44, 0xad, + 0xe1, 0x13, 0x49, 0x5c, 0x4a, 0x29, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, + 0x0f, 0x0d, 0x04, 0x34, 0x8d, 0xe9, 0x46, 0x4c, 0x02, 0x7b, 0x0e, 0x00, + 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x5d, 0x81, 0xb3, 0xa4, 0xc4, 0xa0, + 0xdf, 0x63, 0x11, 0x70, 0xf0, 0xf5, 0xf0, 0xb3, 0xa1, 0x4f, 0x29, 0x0e, + 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xce, 0xe3, 0xda, 0x5f, 0x6c, + 0xdc, 0xd8, 0x6d, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x74, + 0xa2, 0x79, 0x44, 0x8e, 0xe2, 0xe5, 0xb1, 0x04, 0x00, 0x46, 0x56, 0xee, + 0xb1, 0x6b, 0x2e, 0x8c, 0xe6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x53, 0xa2, 0x45, 0x08, 0x2c, 0xa7, 0xb2, 0xc5, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x16, 0x68, 0x56, 0xb2, 0x8a, 0xfc, 0x7d, + 0x43, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0x04, 0x0d, 0x86, 0x1b, 0x63, + 0x8e, 0xba, 0xad, 0xbc, 0xc4, 0x0c, 0x40, 0xcf, 0xa9, 0x8b, 0x28, 0xb5, + 0xd4, 0x69, 0x7f, 0x04, 0x00, 0x61, 0xb0, 0x72, 0x30, 0x30, 0x48, 0x65, + 0x7f, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xea, 0x0c, 0xe8, + 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x10, 0x02, 0x04, 0x00, 0xbd, 0x0f, 0x47, + 0x9c, 0x60, 0x32, 0x53, 0x75, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0x42, 0x4f, 0x4f, 0x30, 0x0d, 0x39, 0x84, 0x1c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x0d, 0xca, 0x5c, 0x71, 0x55, 0xf6, 0xf1, 0x33, + 0xa6, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, 0x0d, 0x14, 0x6d, 0x5b, + 0x5a, 0xad, 0x50, 0x42, 0x12, 0x0e, 0x00, 0xff, 0xff, 0xff, 0xff, 0x0f, + 0x0d, 0xea, 0x0c, 0xe8, 0x30, 0x94, 0xfd, 0xe4, 0x7c, 0x0e, 0x00, 0xff, + 0xff, 0xff, 0xff, 0x0f, 0x11, 0x8b, 0xe1, 0x45, 0xc2, 0xcb, 0x04, 0xda, + 0x4f, 0x00, 0x87, 0x94, 0xb6, 0x96, 0xa7, 0x62, 0xb5, 0xa0, 0x00, 0x31, + 0x63, 0x3e, 0xd6, 0x95, 0xbb, 0xc2, 0xd5, 0x0d, 0x07, 0xb2, 0x52, 0x16, + 0x4e, 0x19, 0x4d, 0xfd, 0x04, 0x00, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xff, 0x00, 0xe4, 0x4f, 0x1c, 0x4f, 0x47, 0xc0, 0x2e, 0x2f, 0x00, + 0x58, 0xfc, 0xaf, 0xfa, 0xd8, 0xe0, 0x4b, 0x70, 0x00, 0xc7, 0xd4, 0x7b, + 0x26, 0xb0, 0x9d, 0x29, 0x5f, 0x00, 0x52, 0x51, 0x2b, 0x1a, 0xc0, 0x1d, + 0x78, 0x1f, 0x00, 0x83, 0x39, 0x21, 0xa6, 0x7b, 0x20, 0x6c, 0x81, 0x00, + 0x51, 0x86, 0x08, 0x60, 0x00, 0x94, 0x5b, 0xc8, 0x00, 0x4a, 0x0d, 0xe3, + 0x6f, 0xdc, 0xd0, 0x31, 0x32, 0x00, 0x52, 0x68, 0x82, 0x60, 0x73, 0xfe, + 0x6f, 0x03, 0x00, 0xf8, 0x36, 0xa0, 0x45, 0xf0, 0x0a, 0x13, 0xe8, 0x00, + 0xaf, 0x79, 0xa2, 0xfb, 0x0a, 0xe0, 0x53, 0x0a, 0x00, 0x06, 0x68, 0x47, + 0x98, 0xd1, 0xa1, 0xcf, 0x52, 0x00, 0x86, 0x70, 0x33, 0xab, 0x0d, 0x20, + 0x04, 0x14, 0x00, 0xfb, 0x06, 0xc9, 0xfe, 0x19, 0xe1, 0x13, 0xa0, 0x00, + 0xbb, 0xc2, 0xc8, 0x70, 0x2b, 0xdd, 0x7b, 0xea, 0x00, 0x0d, 0x4f, 0xb1, + 0xd1, 0xd2, 0x52, 0x82, 0x75, 0x00, 0xa6, 0xd4, 0x21, 0x39, 0xcd, 0x15, + 0x6b, 0xf9, 0x00, 0x76, 0xba, 0x6b, 0x24, 0xf1, 0xe3, 0x3d, 0x03, 0x00, + 0x91, 0x0a, 0x55, 0x60, 0xf7, 0xa2, 0x07, 0xec, 0x00, 0x5e, 0xb4, 0x05, + 0x1b, 0xfb, 0xf5, 0x92, 0x24, 0x00, 0x50, 0xb1, 0x8f, 0xa7, 0x74, 0x87, + 0x57, 0xb4, 0x00, 0x16, 0xa3, 0x71, 0x35, 0x4e, 0x96, 0x13, 0xb4, 0x00, + 0x00, 0xfe, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xff, 0xff, + 0xff, 0xff, 0xff, 0xff, 0xff, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, +}; + +// TableVocabulary is ONE DIRECTION's announced vocabulary (§3.3): the entries +// an announcement carried, RESOLVED ONCE, under one numbering. +// +// THE RECEIVER RESOLVES ONCE (§3.3), so this holds the entries themselves and +// not the announcement's bytes: every entry is parsed at AnnounceRead, and +// every body after it dispatches through ONE ARRAY INDEX with nothing to +// re-read and nothing to decide. The announcement is free the moment +// AnnounceRead returns. +// +// THE STORAGE IS THE CALLER'S and this library never allocates. The caller +// declares an array of entries wherever it wants it, static, on a heap, in an +// arena or beside its connection, and hands it here with its CAPACITY. The +// announcement holds for the life of the connection (§3.3), so the array does +// too, and a peer holds TWO for a connection, the one it writes with and the +// one it reads with. A restart opens a fresh connection with an empty +// vocabulary and nothing is cached across connections. +// +// kTableMessageEntriesHere is the capacity a receiver that talks only to peers +// of THIS schema declares, and a receiver meeting other builds declares more. +struct TableVocabulary +{ + // THE CONFORMING DEFAULT BYTE BOUND (§3.3). The ENTRY bound has no default + // because it IS the caller's capacity: an announcement naming more entries + // than the caller made room for is refused as vocabulary_too_large before + // an entry is touched, and the byte bound is read off the vocabulary + // field's own length before that. + static const int64_t kDefaultMaxBytes = 64 * 1024; + + TableVocabulary( TableMessageEntry * storage, int64_t capacity ) + : entries( storage ), max_entries( capacity ) {} + + TableMessageEntry * entries; // THE CALLER'S, capacity max_entries + int64_t max_entries; + int64_t count = 0; + int64_t ref_bits = 0; + uint64_t build_version = 0; + bool announced = false; + // REFUSAL IS TERMINAL (§3.3): a connection whose first announcement was + // refused, for any reason, carries no vocabulary for its life, and every + // announcement after it is refused as second_announcement + bool refused = false; + int64_t max_bytes = kDefaultMaxBytes; +}; + +// TableVocabularyEntryAt is the entry a reference names, counted from 1: ONE +// ARRAY INDEX into the caller's resolved storage, no parse and no branch. +inline const TableMessageEntry & TableVocabularyEntryAt( const TableVocabulary & vocabulary, uint64_t slot ) +{ + return vocabulary.entries[ slot - 1 ]; +} + +// AnnounceRead reads an announcement into one direction's vocabulary (§3.3). +// +// The announcement IS a file, so every malformed rule of §3 already covers it. +// Over its body there are EXACTLY TWO STRICT CHECKS: the BUILD VERSION +// present, exactly once, under kind 9, eight bytes wide, and the VOCABULARY +// present, exactly once, under kind 14 over element kind 6. Everything else is +// ordinary and tolerant, so an unknown field is skipped and counted and the +// announcement can GAIN a field in a later minor without a lockstep redeploy. +// +// The FIRST announcement sets the vocabulary and it is the only one that can. +// A SECOND is refused by name: it does not replace it, does not amend it and +// changes nothing. A refused announcement sets NO VOCABULARY, and the refusal +// is TERMINAL: every announcement after it, whether or not the first set +// anything, is second_announcement, so a peer holds no retry on the +// connection and cannot buy a second resolve by having its first refused. +inline bool AnnounceReadOnce( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * to ); + +inline bool AnnounceRead( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * to = report != NULL ? report : &ignored; + if ( vocabulary.announced || vocabulary.refused ) + { + to->refused = true; + to->reason = second_announcement; + return false; + } + const bool set = AnnounceReadOnce( vocabulary, buffer, bytes, to ); + if ( !set ) { vocabulary.refused = true; } + return set; +} + +inline bool AnnounceReadOnce( TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * to ) +{ + if ( bytes < 1 ) { to->malformed = true; return false; } + if ( buffer[0] != kTableWireForm ) + { + to->refused = true; + to->reason = buffer[0] == kTableWireMessageForm ? message_form_as_file : newer_form; + return false; + } + if ( bytes < 9 ) { to->malformed = true; return false; } + TableIdTable table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( buffer, bytes, table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { to->malformed = true; } + else { to->refused = true; to->reason = newer_form; } + return false; + } + if ( TableBodyEndsEarly( buffer + 1, body_bytes, table ) ) { to->malformed = true; return false; } + TableReader r( buffer + 1, body_bytes, to, &table ); + uint64_t version = 0; + const uint8_t * words = NULL; + int64_t words_bytes = 0; + int32_t seen_version = 0, seen_vocabulary = 0; + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.getleb( ref ) ) { to->malformed = true; return false; } + if ( ref == 0 ) { break; } + if ( ref > (uint64_t) table.count || !r.has( 1 ) ) { to->malformed = true; return false; } + const uint64_t id = table.at( ref ); + const uint8_t kind = r.get8(); + if ( id == kTableBuildVersionFieldId ) + { + if ( kind != 9 || !r.has( 8 ) ) { to->malformed = true; return false; } + version = r.get64(); + // THE BUILD VERSION IS KEPT THE MOMENT IT IS READ, refusal or not, so + // that a refusal on this connection NAMES IT (§3.3). It is not the + // vocabulary, and a refused announcement still sets none. + vocabulary.build_version = version; + seen_version++; + continue; + } + if ( id == kTableMessageVocabularyFieldId ) + { + // kind 14 over element kind 6, which is §3's spelling for an + // opaque run of bytes + uint64_t framed = 0; + if ( kind != 14 || !r.getleb( framed ) || !r.has( (int64_t) framed ) ) { to->malformed = true; return false; } + const int64_t begin = r.offset, end = r.offset + (int64_t) framed; + r.offset = end; + if ( begin >= end || r.buffer[ begin ] != 6 ) { to->malformed = true; return false; } + int64_t at = begin + 1; + uint64_t length = 0; + if ( !TableMessageLeb( r.buffer, end, at, length ) || at + (int64_t) length != end ) { to->malformed = true; return false; } + if ( (int64_t) length > vocabulary.max_bytes ) { to->refused = true; to->reason = vocabulary_too_large; return false; } + words = r.buffer + at; + words_bytes = (int64_t) length; + seen_vocabulary++; + continue; + } + to->unknown++; + if ( !r.skip( kind ) ) { to->malformed = true; return false; } + } + if ( seen_version != 1 || seen_vocabulary != 1 ) { to->malformed = true; return false; } + + // THE ENTRIES, RESOLVED ONCE into the caller's storage (§3.3): every width + // is checked here and never again, and no body after this parses a byte of + // an announcement. An entry count above the caller's CAPACITY is refused + // by name before the entry is touched. + int64_t at = 0, count = 0, node_table_slots = 0; + while ( at < words_bytes ) + { + if ( count >= vocabulary.max_entries ) { to->refused = true; to->reason = vocabulary_too_large; return false; } + TableMessageEntry & parsed = vocabulary.entries[ count ]; + if ( !TableMessageEntryRead( words, words_bytes, at, parsed ) ) { to->malformed = true; return false; } + // THE RESERVED IDS WHERE THEY DO NOT BELONG (§3.3): the announcement's + // own two never take a slot, and the node-table id takes exactly one, + // so a vocabulary carrying either of the first or a SECOND node-table + // id is malformed whole and sets nothing + if ( parsed.id == kTableBuildVersionFieldId || parsed.id == kTableMessageVocabularyFieldId ) { to->malformed = true; return false; } + if ( parsed.id == kTableNodeTableFieldId ) { if ( node_table_slots++ > 0 ) { to->malformed = true; return false; } } + // A TRIPLE ALREADY PLACED IS NEVER PLACED TWICE, so two entries that + // agree on the id, the kind and every fact of the shape are malformed + // (§3.3): no writer this wire has produces one, and a reader that took + // it would carry two slots naming one thing. The scan is quadratic in + // the entry count, and the entry count is bounded above at 4096, so it + // is at most eight million compares on a path that runs ONCE a + // connection and never again. + for ( int64_t seen = 0; seen < count; seen++ ) + { + const TableMessageEntry & other = vocabulary.entries[ seen ]; + if ( other.id == parsed.id && other.kind == parsed.kind && TableMessageEntrySame( other, parsed ) ) { to->malformed = true; return false; } + } + count++; + } + vocabulary.count = count; + vocabulary.ref_bits = TableBitsRequired( 0, count ); + vocabulary.build_version = version; + vocabulary.announced = true; + return true; +} + +// TableMessageReserved is one of the three ids the language holds back (§3.1, +// §3.3, §5): each is malformed anywhere but its own transport, and the rule +// OUTRANKS the wrong-sort rule below. +inline bool TableMessageReserved( uint64_t id ) +{ + // THE THREE ARE THE TOP THREE VALUES a uint64 holds, so the test is ONE + // comparison: 0xFFFFFFFFFFFFFFFD, FE and FF and nothing else is at or + // above the vocabulary's own id, and a declaration hashing to any of them + // is refused by name (§11) + return id >= kTableMessageVocabularyFieldId; +} + +// TableMessageNameEntry resolves a reference used as a VALUE, which is an +// enum's variant, a keyed array's slot key or a node record's type id, and +// which must name a kind-0 entry (§3.3). A reference of 0 where an entry is required, one +// above E, one naming a reserved id and one naming an entry that carries a +// payload are each damage: the reader RESOLVED the entry and it contradicts +// the position it was used in, so the next bit's meaning is what is in doubt. +inline bool TableMessageNameEntry( const TableVocabulary & vocabulary, uint64_t ref, TableMessageEntry & entry ) +{ + if ( ref == 0 || ref > (uint64_t) vocabulary.count ) { return false; } + entry = TableVocabularyEntryAt( vocabulary, ref ); + return !TableMessageReserved( entry.id ) && entry.kind == 0; +} + +// TableMessageArmEntry resolves a UNION's arm reference, which must name an +// entry carrying the arm's own kind and shape: a kind-0 entry frames nothing, +// and a reserved id belongs to no arm (§3.3). +inline bool TableMessageArmEntry( const TableVocabulary & vocabulary, uint64_t ref, TableMessageEntry & entry ) +{ + if ( ref == 0 || ref > (uint64_t) vocabulary.count ) { return false; } + entry = TableVocabularyEntryAt( vocabulary, ref ); + return !TableMessageReserved( entry.id ) && entry.kind != 0; +} + +// TableMessageSkipVariant steps over an ENUM's variant reference on a SKIP +// path and RESOLVES it while it is there: 0 is None and the whole payload, and +// every other reference must name a kind-0 entry, because every reference +// above E is damage and one naming an entry that carries a payload +// contradicts the position it was used in, whether or not this reader was +// going to keep the value (§3.3). +inline bool TableMessageSkipVariant( TableBitReader & r, const TableVocabulary & vocabulary ) +{ + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + TableMessageEntry named; + return TableMessageNameEntry( vocabulary, ref, named ); +} +// TableMessageSkip steps over one field's payload without decoding it, using +// the announced ENTRY alone (§3.3). It is what makes an unknown entry +// skippable on a body with no kind byte, and it is ONE function over every +// table, because a shape says everything a skipper needs. +inline bool TableMessageSkipBody( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits ); +inline bool TableMessageSkip( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ); + +// TableMessageSkipElement steps over ONE element of an array or keyed entry +// by the element's own announced shape: a nested body to its zero reference, +// a variant or a node index at its reference width, a union arm by its own +// entry, and a fixed-width value at its bits. +inline bool TableMessageSkipElement( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ) +{ + switch ( entry.elem_kind ) + { + case 13: return TableMessageSkipBody( r, vocabulary, index_bits ); + case 30: return TableMessageSkipVariant( r, vocabulary ); + case 17: return index_bits > 0 && r.skip( index_bits ); + case 15: + { + TableMessageEntry inner; + inner.kind = 15; + return TableMessageSkip( r, vocabulary, index_bits, inner ); + } + default: + { + const int64_t elem = entry.elem_value_bits; + return elem >= 0 && r.skip( elem ); + } + } +} + +// TableMessageElementRunBits is the bits ONE element of an array or a keyed +// entry occupies on the SKIP path, where nothing is resolved and a run of them +// is one multiplication, and -1 where the element's width is its own +// content's. A ZERO is a real answer, and it is why this exists: a ranged +// element whose min equals its max rides no bits at all (§3.3). +inline int64_t TableMessageElementRunBits( const TableVocabulary & vocabulary, const TableMessageEntry & entry ) +{ + int64_t elem = 0; + switch ( entry.elem_kind ) + { + // a nested body, a union arm, an enum's variant and a node index each + // RESOLVE something, and a resolve that contradicts its position is + // damage this reader must still find, so they are walked + case 13: case 15: case 30: case 17: return -1; + default: elem = entry.elem_value_bits; break; + } + if ( elem < 0 ) { return -1; } + if ( entry.kind == 16 ) { elem += vocabulary.ref_bits; } // a keyed slot's own key reference + return elem; +} + +// TableMessageSkipRun steps over n elements of one fixed width in a single +// arithmetic step. A FIXED-WIDTH ELEMENT IS ARITHMETIC (§3.3), and a loop here +// would be the one superlinear thing in this form: a zero-width element under +// a count of 2^31 is six bytes of wire. +inline bool TableMessageSkipRun( TableBitReader & r, uint64_t n, int64_t width ) +{ + if ( width < 0 ) { return false; } + if ( width == 0 ) { return true; } + if ( n > (uint64_t) ( INT64_MAX / width ) ) { return false; } + return r.skip( (int64_t) ( n * (uint64_t) width ) ); +} + +inline bool TableMessageSkip( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, const TableMessageEntry & entry ) +{ + switch ( entry.kind ) + { + case 0: case 32: return true; // a name, and a payload-free arm + case 30: return TableMessageSkipVariant( r, vocabulary ); + case 13: return TableMessageSkipBody( r, vocabulary, index_bits ); + case 17: return index_bits > 0 && r.skip( index_bits ); // a node index, at the width the body's node count settled + case 15: + { + uint64_t arm = 0; + if ( !r.get( arm, vocabulary.ref_bits ) ) { return false; } + if ( arm == 0 ) { return true; } + TableMessageEntry arm_entry; + if ( !TableMessageArmEntry( vocabulary, arm, arm_entry ) ) { return false; } + return TableMessageSkip( r, vocabulary, index_bits, arm_entry ); + } + case 12: + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) || !r.align() ) { return false; } + return r.skip( (int64_t) n * 8 ); + } + case 33: + { + // the length, NO align, then SIXTEEN bits a code unit (§3.3) + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( 0, entry.max ) ) ) { return false; } + return r.skip( (int64_t) n * 16 ); + } + case 31: + { + // THE ESCAPE: align, a thirty-two bit L, then L bytes, opaque. It is + // the one path a later-major writer has on this form (§3.3) + uint64_t n = 0; + if ( !r.align() || !r.get( n, 32 ) ) { return false; } + return r.skip( (int64_t) n * 8 ); + } + case 14: case 16: + { + uint64_t n = (uint64_t) entry.min; + const int64_t width = entry.kind == 16 ? TableBitsRequired( 0, entry.max ) : TableBitsRequired( entry.min, entry.max ); + if ( entry.kind == 16 ) { n = 0; } + if ( width > 0 ) + { + uint64_t raw = 0; + if ( !r.get( raw, width ) ) { return false; } + n = entry.kind == 16 ? raw : raw + (uint64_t) entry.min; + } + if ( entry.kind == 14 && entry.elem_kind == 6 && !r.align() ) { return false; } + // A RUN OF FIXED-WIDTH ELEMENTS IS ONE MULTIPLICATION (§3.3), and + // only an element whose width is its own content's is walked + const int64_t run = TableMessageElementRunBits( vocabulary, entry ); + if ( run >= 0 ) { return TableMessageSkipRun( r, n, run ); } + for ( uint64_t i = 0; i < n; i++ ) + { + if ( entry.kind == 16 && !r.skip( vocabulary.ref_bits ) ) { return false; } + if ( !TableMessageSkipElement( r, vocabulary, index_bits, entry ) ) { return false; } + } + return true; + } + } + const int64_t width = entry.value_bits; + return width >= 0 && r.skip( width ); +} + +inline bool TableMessageSkipBody( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + if ( !TableMessageSkip( r, vocabulary, index_bits, TableVocabularyEntryAt( vocabulary, ref ) ) ) { return false; } + } +} + +// TableMessageNodeTableOpen reads the node table's opening when a body has +// one: the reserved id's reference and the count at thirty-two raw bits. A +// body whose first reference is anything else has no node table, and the +// reader is left where it was. False is damage: a reference past E, or bits +// that run out. +inline bool TableMessageNodeTableOpen( TableBitReader & r, const TableVocabulary & vocabulary, int64_t & count ) +{ + count = 0; + const int64_t at = r.offset; + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { r.offset = at; return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + if ( TableVocabularyEntryAt( vocabulary, ref ).id != kTableNodeTableFieldId ) { r.offset = at; return true; } + uint64_t n = 0; + if ( !r.get( n, 32 ) ) { return false; } + count = (int64_t) n; + return true; +} +// AnnounceMeasure is the announcement's byte count, which is a constant of the +// unit and not a walk. +inline int64_t AnnounceMeasure() { return kTableAnnounceBytes; } + +// Announce writes the announcement into the caller's buffer and answers the +// bytes written, which is exactly AnnounceMeasure's answer, or -1 when the +// buffer is too small. It allocates nothing and walks nothing. +inline int64_t Announce( uint8_t * buffer, int64_t capacity ) +{ + if ( buffer == NULL || capacity < kTableAnnounceBytes ) { return -1; } + memcpy( buffer, kTableAnnounce, (size_t) kTableAnnounceBytes ); + return kTableAnnounceBytes; +} + +// THE PRIMITIVE IS A BATCH (§3.3): a number of bodies of ONE ROOT in one +// buffer, one count and one continuous bit stream with no alignment between +// them. A single message is the batch of one. +// +// The count rides ahead of the bodies, so a writer declares it at Begin and +// End refuses a batch that wrote a different number: a count the bodies do not +// match is not a wire this writer will hand anyone. +struct TableMessageBatch +{ + TableBitWriter w; + int64_t declared = 0; + int64_t written = 0; +}; + +inline bool TableMessageBatchBegin( TableMessageBatch & batch, uint8_t * buffer, int64_t capacity, int64_t bodies ) +{ + if ( buffer == NULL || capacity < 1 || bodies < 1 || bodies > kTableMessageBatchMax ) { return false; } + buffer[0] = kTableWireMessageForm; // the FORM BYTE is read first, always + batch.w = TableBitWriter( buffer + 1, capacity - 1 ); + batch.declared = bodies; + batch.written = 0; + batch.w.put( (uint64_t) ( bodies - 1 ), 8 ); // a ranged integer over [1, 256] + return true; +} + +// TableMessageBatchEnd zero-fills to the next byte, the one alignment a batch +// spends at its end, and answers the whole batch's byte count, or -1. +inline int64_t TableMessageBatchEnd( TableMessageBatch & batch ) +{ + if ( batch.written != batch.declared || batch.w.overflow ) { return -1; } + batch.w.align(); + if ( batch.w.overflow ) { return -1; } + return 1 + batch.w.bits / 8; +} + +// TableMessageBatchBytes is a batch's byte count from its bodies' BIT count, +// which is what every MeasureMessages answers. +inline int64_t TableMessageBatchBytes( int64_t body_bits ) +{ + if ( body_bits < 0 ) { return -1; } + return 1 + ( 8 + body_bits + 7 ) / 8; +} + +// The reading half. A batch is opened once and its bodies are then read in +// order into the storage the caller sized for them: which root a batch carries +// is the APPLICATION's and never this wire's. +struct TableMessageBatchReader +{ + TableBitReader r; + const TableVocabulary * vocabulary = NULL; + TableReport * report = NULL; + int64_t remaining = 0; + // THE SINK A CALLER THAT PASSED NO REPORT WRITES INTO IS THE READER'S OWN, + // not a static: a static is shared mutable state, and two threads reading + // two batches without reports would be writing one object. LoadMessages + // already keeps its sink locally, for the same reason. + TableReport ignored; +}; + +// TableMessageBatchOpen answers the batch's body count, or -1 with the refusal +// on the report: a form byte this reader does not carry, or a body from a peer +// that never announced. +inline int64_t TableMessageBatchOpen( TableMessageBatchReader & br, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + br.report = report != NULL ? report : &br.ignored; + br.vocabulary = &vocabulary; + if ( bytes < 1 ) { br.report->malformed = true; return -1; } + if ( buffer[0] != kTableWireMessageForm ) { br.report->refused = true; br.report->reason = newer_form; return -1; } + if ( !vocabulary.announced ) { br.report->refused = true; br.report->reason = no_vocabulary; return -1; } + br.r = TableBitReader( buffer + 1, bytes - 1 ); + uint64_t count = 0; + if ( !br.r.get( count, 8 ) ) { br.report->malformed = true; return -1; } + br.remaining = (int64_t) count + 1; + return br.remaining; +} + +// TableMessageRefuseBatch is the batch's own refusal (§3.3): M above 256 on the +// write side, or above the caller's capacity on the read side. Nothing is +// written or decoded, no counter moves, and the reason names it. +inline void TableMessageRefuseBatch( TableReport * report ) +{ + if ( report == NULL ) { return; } + report->refused = true; + report->reason = batch_too_large; +} + +// TableMessageBatchClose verifies the trailing pad, and that NOTHING FOLLOWS +// IT: the batch ends at the pad to the byte boundary, and a buffer with bytes +// left over describes no batch this reader can name (§3.3). +inline bool TableMessageBatchClose( TableMessageBatchReader & br ) +{ + if ( br.remaining != 0 || !br.r.align() || br.r.offset != br.r.bits ) { br.report->malformed = true; return false; } + return true; +} + + +// An ENUM-KEYED array's storage: E.Max slots, ONE PER NAMED VARIANT, with the +// key k at index k-1 — the storage SHIFTS LEFT and nothing is stored for None. +// +// NOTHING OUTSIDE THE ARRAY NAMES ITS SIZE: the extent is derived from E::Max +// here and nowhere else, so there is no size parameter to spell and no count a +// consumer could put one out of step with. +// +// NONE IS THE NULL KEY: it names no slot, it never rides on the wire, a stored +// key of 0 is malformed, and INDEXING BY IT IS A PROGRAM ERROR IN EVERY +// CONFIGURATION — caught by operator[], which cannot see a runtime key any +// earlier, and REFUSED UNCONDITIONALLY. A KEY PAST Max IS THE SAME ERROR for +// the same reason — it names a variant this enum does not have — so the +// accessor refuses BOTH ENDS. NDEBUG does not remove the compare: +// there is NO UB PATH here in any build. ITERATION is still the surface a +// consumer of the whole array wants: begin()/end() walk every stored slot and +// yield the KEY, 1..E.Max, so a call site writes no bound, no cast, no shift +// and no None question. +template +struct TableKeyed +{ + // the extent is the enum's, derived here and named nowhere else + static constexpr int32_t kSlots = (int32_t) E::Max; + + T slots[kSlots] = {}; + + T & operator[]( E key ) + { + RefuseKey( key ); + return slots[ (int32_t) key - 1 ]; + } + const T & operator[]( E key ) const + { + RefuseKey( key ); + return slots[ (int32_t) key - 1 ]; + } + + // THE REFUSAL, and it stands in EVERY BUILD, AT BOTH ENDS. The storage + // holds one slot per NAMED variant: nothing for None below it and nothing + // above Max, so a build that skipped this compare would index one element + // BEFORE the array or past its end — undefined behavior in the + // configuration a game ships. Either key is a program error, so the + // accessor ends the program rather than reading something. The assert + // carries the message where a debugger can read it and NDEBUG removes + // that; the fatal is what stands after it. BOTH GO THROUGH THE HOOKS — + // define schema_assert and schema_fatal and this refusal lands in your + // own handler. + // + // ONE UNSIGNED COMPARE COVERS BOTH ENDS: the storage index is key - 1, and + // None's is -1, which wraps above kSlots unsigned. The cost is one + // perfectly-predicted compare, on a path that reads config. + static void RefuseKey( E key ) + { + if ( (uint32_t) ( (int32_t) key - 1 ) >= (uint32_t) kSlots ) + { + schema_assert( false && "an enum-keyed array holds one slot per named variant: None keys none, and neither does a key past Max" ); + schema_fatal(); + } + } + + // ---- iteration: keys 1..E.Max over storage 0..E.Max-1, key beside element ---- + // + // The entry is a key and a REFERENCE, handed out BY VALUE the way any + // proxy is: for ( auto [ key, element ] : keyed ) binds element to the + // reference member, so iterating fills the array as well as reads it. + // auto & [ key, element ] does NOT compile, and that is by design — a + // non-const lvalue reference cannot bind to the proxy. Write + // auto [ ... ], or auto && [ ... ] if you prefer the reference form. + // + // THE ITERATORS CARRY NO iterator_traits TYPEDEFS. They bought std::distance + // and the forward-pass algorithms for an audience that does not call them, + // and the they need is the single most expensive include the + // generated corpus had: 536 headers and 986 KB, in a header whose whole + // remaining set is 123. begin(), end() and size() need none of it. + + struct Entry { E key; T & element; }; + struct ConstEntry { E key; const T & element; }; + + struct Iterator + { + T * slots; + int32_t index; // the STORAGE index; the key it holds is index + 1 + Entry operator*() const { return Entry{ (E) ( index + 1 ), slots[index] }; } + Iterator & operator++() { index++; return *this; } + bool operator==( const Iterator & other ) const { return index == other.index; } + bool operator!=( const Iterator & other ) const { return index != other.index; } + }; + + struct ConstIterator + { + const T * slots; + int32_t index; // the STORAGE index; the key it holds is index + 1 + ConstEntry operator*() const { return ConstEntry{ (E) ( index + 1 ), slots[index] }; } + ConstIterator & operator++() { index++; return *this; } + bool operator==( const ConstIterator & other ) const { return index == other.index; } + bool operator!=( const ConstIterator & other ) const { return index != other.index; } + }; + + Iterator begin() { return Iterator{ slots, 0 }; } + Iterator end() { return Iterator{ slots, kSlots }; } + ConstIterator begin() const { return ConstIterator{ slots, 0 }; } + ConstIterator end() const { return ConstIterator{ slots, kSlots }; } +}; + +inline float table_bits_to_float( uint32_t bits ) { float f; memcpy( &f, &bits, 4 ); return f; } +inline uint32_t table_float_to_bits( float f ) { uint32_t b; memcpy( &b, &f, 4 ); return b; } +inline double table_bits_to_double( uint64_t bits ) { double d; memcpy( &d, &bits, 8 ); return d; } +inline uint64_t table_double_to_bits( double d ) { uint64_t b; memcpy( &b, &d, 8 ); return b; } + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_PRIMITIVES + +#ifndef MAPDEMO_SCHEMA_TABLE_ARENA +#define MAPDEMO_SCHEMA_TABLE_ARENA + +namespace mapdemo { + +// ---- variable-length tables: tuning constants (docs/SPEC-TABLES.md) ---- +// +// The segment size and the count multiply to exactly 2^32: the u32 reference +// is the arena's hard ceiling, and these constants saturate it rather than +// leaving address space unreachable. Slab handout costs one atomic per slab, +// so per-node allocation costs no synchronization at all. + +static const uint32_t kTableSegmentBits = 22; // 4 MiB segments +static const uint32_t kTableSegmentSize = 1u << kTableSegmentBits; +static const uint32_t kTableSegmentMask = kTableSegmentSize - 1u; +static const uint32_t kTableMaxSegments = 1u << ( 32 - kTableSegmentBits ); // 1024 -> 4 GiB +static const uint32_t kTableSlabBytes = 64u * 1024u; // one atomic per slab +static const uint32_t kTableAlign = 8; // every node starts 8-aligned +static const uint32_t kTableAllocFailed = 0xFFFFFFFFu; + +// ---- THE CALLER'S ALLOCATOR (docs/SPEC-TABLES.md §6.5) ---- +// +// Every allocation the variable-length runtime makes goes through one of +// these — the arena's segments, the pack walk's identity map, the numbering's +// entry array, the packed region, and the tool path's node directory. There is +// no other call to the C library on this path, so a counting allocator sees +// every byte and a game's own heap can own all of it. +// +// It is the shape TableBlockAllocator already has (§19.1): two function +// pointers and a context the caller carries. What it adds is a CONTRACT ON +// alloc — the bytes come back ZEROED. Lock copies whole nodes, PADDING +// INCLUDED, so anything left uninitialized reaches a packed region; the default +// pair reaches that through calloc, which costs nothing measurable because a +// fresh segment is untouched pages either way. +struct TableAllocator +{ + void * ( *alloc )( void * context, int64_t bytes ); // ZEROED bytes, NULL on failure + void ( *free )( void * context, void * pointer ); + void * context; +}; + +// The default pair, and it is the one every entry point takes when the caller +// names none. It calls schema_allocate / schema_release, so a program with its +// own C-library replacement can move the floor without writing a struct at all. +inline void * table_default_alloc( void * context, int64_t bytes ) { (void) context; return schema_allocate( bytes ); } +inline void table_default_free( void * context, void * pointer ) { (void) context; schema_release( pointer ); } + +inline TableAllocator TableDefaultAllocator() +{ + TableAllocator allocator; + allocator.alloc = table_default_alloc; + allocator.free = table_default_free; + allocator.context = NULL; + return allocator; +} + +// ---- TableRef: a relocatable reference (never a machine pointer) ---- +// +// Two encodings, one slot, and the FORM says which is in force: +// +// in the arena — the node's arena offset (segment index in the high bits) +// in a region — the SELF-RELATIVE byte delta from this slot's own address, +// so a deref is one add, needs no base pointer, and a whole +// region relocates by memcpy with zero fix-up +// +// 0 is null in both, and a slot can never name the node that contains it, so +// zero names nothing real in either form. +// +// A REGION DELTA HAS NO REQUIRED SIGN (§6.3). A region is packed depth-first, +// so a node's FIRST reference points forward; every LATER reference to that +// same node points BACK at the one body it already has, which is exactly what +// makes one node one node in a region. Sharing and a back-reference are the +// same fact, and nothing validates a reference by its sign. +// +// IT IS EIGHT BYTES, SIGNED, so ONE REGION REACHES EVERYTHING (§6.3, §7): a +// four-byte slot bounded a region at 2 GiB, and the scale a cook exists for is +// *"100mbs or many gigabytes of data in Assets.bin"*. +struct TableRef +{ + int64_t value = 0; + bool null() const { return value == 0; } +}; + +// TableSlot is what Alloc hands back: usable as the node pointer (write +// fields through it) AND as the reference to store in a pointer field. +template struct TableSlot +{ + T * ptr = NULL; + TableRef ref; + T * operator->() const { return ptr; } + T & operator*() const { return *ptr; } + operator T *() const { return ptr; } + operator TableRef() const { return ref; } + bool null() const { return ptr == NULL; } +}; + +inline uint32_t TableAlignUp( uint32_t bytes ) { return ( bytes + kTableAlign - 1 ) & ~( kTableAlign - 1 ); } +inline int64_t TableAlignUp64( int64_t bytes ) { return ( bytes + kTableAlign - 1 ) & ~( int64_t( kTableAlign ) - 1 ); } + +// ---- a BYTE BUFFER's node (docs/SPEC-TABLES.md §2.5, §6.3) ---- +// +// A *bytes or *string slot is a TableRef like every pointer slot, and it names +// a BLOB NODE: this eight-byte header and then the bytes, at offset eight so +// the data is eight-aligned. A *string blob carries one more zero byte after +// its data, so a region hands back a C string with no copy. The node's extent +// is the header plus its bytes, rounded to the arena's alignment like every +// node's; on the wire it is a record whose body is the bytes (§3.1). +struct TableBlob +{ + uint32_t length; + uint32_t zero; +}; + +static const int64_t kTableBlobHeader = 8; // length (u32), then four zero bytes +static const int64_t kTableBlobMaxLength = 0xFFFFFFFF; // a record's length is a u32 (§3.1) + +// the node's storage: the header, the bytes, a string's terminator, rounded +// to the arena's alignment like every node +inline int64_t TableBlobStorage( int64_t length, bool terminated ) +{ + return TableAlignUp64( kTableBlobHeader + length + ( terminated ? 1 : 0 ) ); +} + +// What a read answers: a pointer INTO the region and the length, NULL and +// zero for a null slot. Off a locked region, a loaded one or an opened cook +// the pointer is one add from the slot, and nothing is copied. +struct TableBytesView +{ + const uint8_t * data; + int64_t length; +}; + +struct TableStringView +{ + const char * data; // zero-terminated + int64_t length; +}; + +// What AllocBytes and AllocString hand back: the bytes to write through, the +// length asked for, and the reference to store in the slot — the three +// answers TableSlot gives for a table node. +struct TableBytesSlot +{ + uint8_t * data = NULL; + int64_t length = 0; + TableRef ref; + bool null() const { return data == NULL; } + operator TableRef() const { return ref; } +}; + +struct TableStringSlot +{ + char * data = NULL; // room for length bytes and the terminator, already zero + int64_t length = 0; + TableRef ref; + bool null() const { return data == NULL; } + operator TableRef() const { return ref; } +}; + +// ---- the arena: segmented, slab-handed, lock-free by ownership ---- +// +// Allocation is thread-local inside a worker's slab — no atomics on the node +// path. A worker takes its next slab with ONE compare-exchange, and a new +// segment is published with one more. Nothing ever moves: a segment, once +// allocated, lives untouched until the arena is torn down, so a T* obtained +// from Alloc stays valid while other workers allocate, and an offset stays +// correct while the arena grows. +// +// The model this DELIBERATELY refuses: one buffer under a lock, grown by +// realloc. A realloc moves the buffer under workers mid-write; offsets fix +// identity but not the raw references already resolved from them, and the +// resulting corruption is invisible until much later. Segments never move, so +// that bug class cannot be written here. +// +// Slack: at most one slab tail per worker plus one slab per segment (a slab +// that will not fit is skipped rather than split), i.e. under 2% of a segment +// plus threads x 64 KiB. That is the price of never synchronizing per node. +struct TableArena +{ + std::atomic segments[ kTableMaxSegments ]; + std::atomic cursor; // (segment << kTableSegmentBits) | bytes handed out + bool locked = false; // MONOTONIC: Lock() is one-way, there is no unlock + // THE ARENA CARRIES ITS OWN, so everything downstream of a builder — + // segments, pack map, numbering, region, node directory — allocates through + // the one pair the caller named, with nothing to thread by hand. + TableAllocator allocator; +}; + +inline void TableArenaInit( TableArena & arena, TableAllocator allocator ) +{ + for ( uint32_t i = 0; i < kTableMaxSegments; i++ ) + { + arena.segments[i].store( NULL, std::memory_order_relaxed ); + } + arena.cursor.store( 0, std::memory_order_relaxed ); + arena.locked = false; + arena.allocator = allocator; +} + +inline void TableArenaShutdown( TableArena & arena ) +{ + for ( uint32_t i = 0; i < kTableMaxSegments; i++ ) + { + uint8_t * segment = arena.segments[i].exchange( NULL, std::memory_order_acq_rel ); + if ( segment != NULL ) { arena.allocator.free( arena.allocator.context, segment ); } + } + arena.cursor.store( 0, std::memory_order_relaxed ); +} + +// one L1 load plus an add: the segment table is 8 KiB and stays hot +inline uint8_t * TableArenaAt( const TableArena & arena, uint32_t offset ) +{ + return arena.segments[ offset >> kTableSegmentBits ].load( std::memory_order_relaxed ) + ( offset & kTableSegmentMask ); +} + +// TableArenaGrabSlab hands one worker its next private slab. Returns +// kTableAllocFailed when the arena's address space or the allocator is +// exhausted — a loud refusal, never a silent smaller slab. +inline uint32_t TableArenaGrabSlab( TableArena & arena ) +{ + for ( ;; ) + { + uint32_t cursor = arena.cursor.load( std::memory_order_acquire ); + uint32_t segment = cursor >> kTableSegmentBits; + uint32_t used = cursor & kTableSegmentMask; + // strictly less: a slab is never split across segments, and the tail + // is the documented slack + if ( used + kTableSlabBytes < kTableSegmentSize ) + { + if ( arena.segments[segment].load( std::memory_order_acquire ) == NULL ) + { + // THE SEGMENT COMES BACK ZEROED, which is the allocator's + // contract and not an extra pass here: Lock copies whole nodes, + // PADDING INCLUDED, so anything uninitialized reaches a packed + // region. Value-initializing a node with placement new zeroes + // its MEMBERS and not its padding, so the zeroing has to happen + // at the segment or not at all. It costs nothing measurable: a + // fresh segment is untouched pages either way, and the default + // pair's calloc has the kernel hand them over zeroed. + uint8_t * memory = (uint8_t *) arena.allocator.alloc( arena.allocator.context, (int64_t) kTableSegmentSize ); + if ( memory == NULL ) { return kTableAllocFailed; } + uint8_t * expected = NULL; + if ( !arena.segments[segment].compare_exchange_strong( expected, memory, std::memory_order_acq_rel ) ) + { + // another worker published this segment first + arena.allocator.free( arena.allocator.context, memory ); + } + } + if ( arena.cursor.compare_exchange_weak( cursor, cursor + kTableSlabBytes, std::memory_order_acq_rel ) ) + { + return ( segment << kTableSegmentBits ) | used; + } + continue; + } + uint32_t next_segment = segment + 1; + if ( next_segment >= kTableMaxSegments ) { return kTableAllocFailed; } // 4 GiB: the u32 reference's ceiling + arena.cursor.compare_exchange_weak( cursor, next_segment << kTableSegmentBits, std::memory_order_acq_rel ); + } +} + +// TableArenaGrabSpan reserves a SPAN of the arena's address space for one node +// larger than a slab — a BYTE BUFFER of any size (docs/SPEC-TABLES.md §2.5) — +// and allocates it as one contiguous block. It takes whole segment indices +// from the cursor, starting at the index after the cursor's so nothing else +// is ever handed out inside the span, and publishes the block under the first +// of them; the indices the span covers past that one stay NULL, which is +// enough, because only a node's START is ever resolved through the segment +// table and a blob's bytes follow its header inside the one allocation. The +// unused tail of the segment the cursor was in is slack, like a slab tail. +// Returns kTableAllocFailed when the address space or the allocator is +// exhausted — a loud refusal, never a smaller blob. +inline uint32_t TableArenaGrabSpan( TableArena & arena, int64_t bytes ) +{ + if ( bytes <= 0 || bytes > ( (int64_t) kTableMaxSegments - 2 ) * (int64_t) kTableSegmentSize ) { return kTableAllocFailed; } + const uint32_t spanned = (uint32_t) ( ( bytes + kTableSegmentSize - 1 ) >> kTableSegmentBits ); + for ( ;; ) + { + uint32_t cursor = arena.cursor.load( std::memory_order_acquire ); + uint32_t start = ( cursor >> kTableSegmentBits ) + 1; + if ( start + spanned >= kTableMaxSegments ) { return kTableAllocFailed; } // 4 GiB: the u32 reference's ceiling + uint32_t next = ( start + spanned ) << kTableSegmentBits; + if ( !arena.cursor.compare_exchange_weak( cursor, next, std::memory_order_acq_rel ) ) { continue; } + // the span is this worker's now: nothing else can publish under its + // first index, so a plain store suffices, and the block comes back + // ZEROED like every segment — the blob's bytes and its tail are zeros + // until written + uint8_t * memory = (uint8_t *) arena.allocator.alloc( arena.allocator.context, bytes ); + if ( memory == NULL ) { return kTableAllocFailed; } + arena.segments[start].store( memory, std::memory_order_release ); + return start << kTableSegmentBits; + } +} + +// ---- TableWorker: one thread's allocation front ---- +// +// The threading contract, stated plainly: +// * Alloc on YOUR OWN worker is safe concurrently with any other worker's. +// No locks, no atomics per node. +// * Writing fields of a node ANOTHER worker allocated is your own +// synchronization problem — this runtime does not arbitrate it. +// * Lock and Save are single-threaded: call them after the workers have +// joined. +struct TableWorker +{ + TableArena * arena = NULL; + uint32_t next = 0; + uint32_t end = 0; + + template TableSlot Alloc() + { + static_assert( alignof( T ) <= kTableAlign, "a table node's alignment must fit the arena's" ); + TableSlot slot; + if ( arena == NULL || arena->locked ) { return slot; } + uint32_t bytes = TableAlignUp( (uint32_t) sizeof( T ) ); + if ( bytes > kTableSlabBytes ) { return slot; } // a node larger than a slab: refused, never split + if ( end == 0 || next + bytes > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return slot; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + uint32_t at = next; + next += bytes; + // A NODE IS BORN IN TWO HALVES: start its lifetime in the raw + // storage, then write the declared defaults ONE MEMBER AT A TIME. + // + // It is "T", not "T{}". Value-initialising the whole aggregate says + // the same thing and costs cl O(BYTES) TO COMPILE — it expands element + // by element in its front end — while both halves here cost + // O(declarations). The slab cap below refuses a large node at RUN + // TIME and bounds nothing at compile time: the cost is paid by + // whatever T a caller instantiates this with. + // Padding is not the difference: value-initialisation zeroes MEMBERS + // and not padding either way, which is why the segment is calloc'd. + // + // TableReset is an OVERLOAD SET, one per closure member, reached from + // this template by argument-dependent lookup on T's own namespace — + // Alloc is a template and cannot spell Reset. + // + // The reset is here because ONE DEFINITION SAYS WHAT THE DECLARED + // DEFAULTS ARE, and it is Reset. Default-initialisation lands on + // the same values today, because a member with a non-zero default + // carries a member initializer that says so — but that is the class + // definition agreeing with Reset, not the arena reading it, and #320's + // fix was itself a pass that MOVED initialisation between the two. + // The arena reads the definition. + slot.ptr = new ( TableArenaAt( *arena, at ) ) T; + TableReset( *slot.ptr ); + slot.ref.value = at; + return slot; + } + + // Alloc a BYTE BUFFER's node of exactly length bytes (docs/SPEC-TABLES.md + // §2.5): the blob header and its bytes, zeroed, in this thread's slab when + // it fits and in a span of the arena's own when it does not. NULL is the + // arena locked, a length below zero or past a record's u32, or the + // allocator refusing. The offset comes back for the reference. + TableBlob * AllocBlob( int64_t length, bool terminated, uint32_t & at ) + { + at = 0; + if ( arena == NULL || arena->locked ) { return NULL; } + if ( length < 0 || length > kTableBlobMaxLength ) { return NULL; } + const int64_t bytes = TableBlobStorage( length, terminated ); + if ( bytes > (int64_t) kTableSlabBytes ) + { + at = TableArenaGrabSpan( *arena, bytes ); + if ( at == kTableAllocFailed ) { at = 0; return NULL; } + } + else + { + if ( end == 0 || next + (uint32_t) bytes > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return NULL; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + at = next; + next += (uint32_t) bytes; + } + TableBlob * blob = (TableBlob *) TableArenaAt( *arena, at ); + blob->length = (uint32_t) length; // the bytes after it are the segment's zeros + blob->zero = 0; + return blob; + } + + // RAW, ZEROED storage of the bytes asked for, at the alignment asked for: a MAP's or a LIST's builder + // head and its segments (docs/SPEC-TABLES.md §2.8, §2.9). It is not a node: it carries + // no type id, takes no index and has no Reset, so it goes through the same + // slab and span the blob path uses rather than through Alloc. + uint8_t * AllocRaw( int64_t bytes, int64_t align, uint32_t & at ) + { + at = 0; + if ( arena == NULL || arena->locked ) { return NULL; } + if ( bytes <= 0 || align > (int64_t) kTableAlign ) { return NULL; } + const int64_t rounded = TableAlignUp64( bytes ); + if ( rounded > (int64_t) kTableSlabBytes ) + { + at = TableArenaGrabSpan( *arena, rounded ); + if ( at == kTableAllocFailed ) { at = 0; return NULL; } + return TableArenaAt( *arena, at ); + } + if ( end == 0 || next + (uint32_t) rounded > end ) + { + uint32_t offset = TableArenaGrabSlab( *arena ); + if ( offset == kTableAllocFailed ) { return NULL; } + next = offset; + end = offset + kTableSlabBytes; + if ( next == 0 ) { next = kTableAlign; } // offset 0 is null: the arena's head stays reserved + } + at = next; + next += (uint32_t) rounded; + return TableArenaAt( *arena, at ); // the segment came back zeroed + } + // a *bytes node: the bytes to write through, and the reference to store + TableBytesSlot AllocBytes( int64_t length ) + { + TableBytesSlot slot; + uint32_t at = 0; + TableBlob * blob = AllocBlob( length, false, at ); + if ( blob == NULL ) { return slot; } + slot.data = (uint8_t *) ( blob + 1 ); + slot.length = length; + slot.ref.value = at; + return slot; + } + + // a *string node: room for length bytes and the zero byte after them + TableStringSlot AllocString( int64_t length ) + { + TableStringSlot slot; + uint32_t at = 0; + TableBlob * blob = AllocBlob( length, true, at ); + if ( blob == NULL ) { return slot; } + slot.data = (char *) ( blob + 1 ); + slot.length = length; + slot.ref.value = at; + return slot; + } +}; + +// ---- TablePackMap: the pack walk's identity map (docs/SPEC-TABLES.md §3.1, §6.2) ---- +// +// ONE ENTRY PER REACHABLE NODE, and that map IS identity: a node must know +// where it landed to be named a second time, so Lock packs a shared node ONCE +// and every later reference resolves to the one body it already has. That is +// the same first-visit numbering the wire uses, so the pack order and the node +// order are one order. +// +// COLOURING AN ENTRY WHILE ITS DESCENT IS OPEN COSTS ONE BIT, and it is what +// makes a data cycle free to refuse: a reference to an entry still open is a +// cycle, and Lock returns failure rather than recursing away. The ROOT's entry +// is open for the whole walk. +// +// The map is proportional to NODES, never to bytes, and it lives on the +// AUTHORING side, where §6.5 licenses allocation. Nothing on the reading path +// ever builds one. +struct TablePackEntry +{ + const void * key; // the node's address in the graph being packed + int64_t offset; // where that node landed in the region + uint8_t open; // its descent is still open: a reference here is a cycle +}; + +struct TablePackMap +{ + TablePackEntry * entries = NULL; + int64_t capacity = 0; // a power of two, or zero while empty + int64_t count = 0; + TableAllocator allocator; // the caller's, carried from the walk that built it +}; + +inline void TablePackMapInit( TablePackMap & map, TableAllocator allocator ) +{ + map.entries = NULL; + map.capacity = 0; + map.count = 0; + map.allocator = allocator; +} + +inline void TablePackMapShutdown( TablePackMap & map ) +{ + map.allocator.free( map.allocator.context, map.entries ); + TablePackMapInit( map, map.allocator ); +} + +// The two walks behind Lock re-derive the SAME map from the same graph — the +// numbering is never carried between them (§3.1) — so the second starts from +// an empty map and keeps the capacity the first paid for. +inline void TablePackMapReset( TablePackMap & map ) +{ + if ( map.entries != NULL ) { memset( map.entries, 0, (size_t) map.capacity * sizeof( TablePackEntry ) ); } + map.count = 0; +} + +// open addressing, linear probing, a multiply-shift hash over the address: a +// node key is a pointer and its low bits are alignment, so the low bits alone +// would collide on every node of one type +inline int64_t TablePackMapSlot( const TablePackMap & map, const void * key ) +{ + uint64_t hash = (uint64_t) (uintptr_t) key; + hash *= 0x9E3779B97F4A7C15ull; + hash ^= hash >> 29; + int64_t mask = map.capacity - 1; + int64_t at = (int64_t) ( hash & (uint64_t) mask ); + while ( map.entries[at].key != NULL && map.entries[at].key != key ) + { + at = ( at + 1 ) & mask; + } + return at; +} + +inline TablePackEntry * TablePackMapFind( TablePackMap & map, const void * key ) +{ + if ( map.capacity == 0 ) { return NULL; } + TablePackEntry * entry = &map.entries[ TablePackMapSlot( map, key ) ]; + return entry->key == key ? entry : NULL; +} + +// QUADRUPLING, not doubling, and the reason is measured: growth rehashes every +// entry, and on a graph of 131,071 nodes the doubling schedule spent 45% of +// Lock in rehashing alone. Quadrupling from 1024 buys 1.35x on that graph and +// keeps the map NODE-proportional (§6.2) — under 128 bytes a node at its +// worst, right after a grow, and about 64 on average. +inline bool TablePackMapGrow( TablePackMap & map ) +{ + TablePackMap grown; + grown.allocator = map.allocator; + grown.capacity = map.capacity != 0 ? map.capacity * 4 : 1024; + grown.entries = (TablePackEntry *) map.allocator.alloc( map.allocator.context, grown.capacity * (int64_t) sizeof( TablePackEntry ) ); + if ( grown.entries == NULL ) { return false; } + for ( int64_t i = 0; i < map.capacity; i++ ) + { + if ( map.entries[i].key == NULL ) { continue; } + grown.entries[ TablePackMapSlot( grown, map.entries[i].key ) ] = map.entries[i]; + grown.count++; + } + map.allocator.free( map.allocator.context, map.entries ); + map = grown; + return true; +} + +// REACH a node: one probe answers both questions the walk has. A true "taken" +// says this is a FIRST visit, and the entry is now the node's, coloured open +// at "offset"; otherwise the entry is the one the node already has, and its +// open bit says cycle or sharing. NULL is an allocation failure, and it is a +// refusal like any other: Lock fails rather than packing a graph it cannot +// track. +// +// It is one call and not a find followed by an insert because the walk asks +// this question twice per node — once to measure, once to pack — and every +// probe is a miss into a table larger than L2. +inline TablePackEntry * TablePackMapReach( TablePackMap & map, const void * key, int64_t offset, bool & taken, int64_t & slot ) +{ + if ( ( map.count + 1 ) * 4 >= map.capacity * 3 ) // keep the load factor under three quarters + { + if ( !TablePackMapGrow( map ) ) { return NULL; } + } + slot = TablePackMapSlot( map, key ); + TablePackEntry * entry = &map.entries[slot]; + taken = entry->key != key; // an empty slot is a first visit; the key is never NULL + if ( taken ) + { + entry->key = key; + entry->offset = offset; + entry->open = 1; + map.count++; + } + return entry; +} + +// The descent finished: the node keeps its entry — identity outlives the +// descent — and stops being a cycle. The "hint" is the slot Reach returned, and it +// is checked against the key rather than trusted, so a rehash between the two +// costs a second probe instead of correctness. +inline void TablePackMapClose( TablePackMap & map, const void * key, int64_t hint ) +{ + if ( hint >= 0 && hint < map.capacity && map.entries[hint].key == key ) + { + map.entries[hint].open = 0; + return; + } + TablePackEntry * entry = TablePackMapFind( map, key ); + if ( entry != NULL ) { entry->open = 0; } +} + +// ---- resolution contexts: which encoding a walk is reading ---- + +struct TableArenaCtx { const TableArena * arena; }; +struct TableRegionCtx {}; + +// ---- a BYTE BUFFER's resolution (docs/SPEC-TABLES.md §2.5, §6.3) ---- +// +// The same two encodings a table pointer has, resolved the same way: a +// self-relative delta in a region — one add, no base — and an arena offset +// while the builder is mutable. The blob is reached through its header, and a +// view is the header plus eight and the header's first word. Nothing here +// allocates and nothing copies: off a locked region, a loaded one or an +// opened cook the view points INTO the region. +inline const TableBlob * TableBlobAt( const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) ( (const uint8_t *) &ref + ref.value ) : NULL; +} +inline const TableBlob * TableBlobAt( const TableRegionCtx &, const TableRef & ref ) { return TableBlobAt( ref ); } +inline const TableBlob * TableBlobAt( const TableArenaCtx & ctx, const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) TableArenaAt( *ctx.arena, (uint32_t) ref.value ) : NULL; +} +inline const TableBlob * TableBlobAt( const TableArena & arena, const TableRef & ref ) +{ + return ref.value != 0 ? (const TableBlob *) TableArenaAt( arena, (uint32_t) ref.value ) : NULL; +} + +inline TableBytesView TableBytesViewOf( const TableBlob * blob ) +{ + TableBytesView view = { NULL, 0 }; + if ( blob != NULL ) { view.data = (const uint8_t *) ( blob + 1 ); view.length = (int64_t) blob->length; } + return view; +} +inline TableStringView TableStringViewOf( const TableBlob * blob ) +{ + TableStringView view = { NULL, 0 }; + if ( blob != NULL ) { view.data = (const char *) ( blob + 1 ); view.length = (int64_t) blob->length; } + return view; +} + +// the const form's hot path: one add, no base +inline TableBytesView TableBytesAt( const TableRef & ref ) { return TableBytesViewOf( TableBlobAt( ref ) ); } +inline TableStringView TableStringAt( const TableRef & ref ) { return TableStringViewOf( TableBlobAt( ref ) ); } +// and the context forms a walk uses: a region context, an arena context, or +// the arena itself while the builder is mutable +template inline TableBytesView TableBytesAt( const Ctx & ctx, const TableRef & ref ) { return TableBytesViewOf( TableBlobAt( ctx, ref ) ); } +template inline TableStringView TableStringAt( const Ctx & ctx, const TableRef & ref ) { return TableStringViewOf( TableBlobAt( ctx, ref ) ); } + +// allocate a blob in the arena and point the slot at it; the slot holds the +// arena offset, as every slot does while the builder is mutable +inline uint8_t * TableBytesEmplace( TableWorker & worker, TableRef & slot, int64_t length ) +{ + TableBytesSlot allocated = worker.AllocBytes( length ); + slot = allocated.ref; + return allocated.data; +} +// the text is copied in when one is given; a NULL text leaves the zeros for +// the caller to fill +inline char * TableStringEmplace( TableWorker & worker, TableRef & slot, const char * text, int64_t length ) +{ + TableStringSlot allocated = worker.AllocString( length ); + slot = allocated.ref; + if ( allocated.data != NULL && text != NULL && length > 0 ) { memcpy( allocated.data, text, (size_t) length ); } + return allocated.data; +} + +// ---- the FLAT NODE TABLE (docs/SPEC-TABLES.md §3.1) ---- +// +// A pointered save writes every reachable node ONCE, into a node table, and a +// pointer field rides as an INDEX into it under kind 17. The encoding is +// flat: no pointer edge is a nesting level, so a chain's length is not a depth, +// and two references to one node are one node. +// +// THE FIELD RIDES ONCE: an L with sixty-four bits of capability frames a +// numbering of any size, so the whole numbering is one contiguous payload and a +// save's node bodies have no aggregate ceiling. + +static const uint64_t kTableNodeIndexNull = 0; // absence and null are one value +static const uint64_t kTableNodeIndexRoot = 1; // the body that hosts the table + +// The not-materialized sentinel (§6.3): a record whose type id this build could +// not name. Distinct from every real offset including the root's 0, so an index +// resolving through it yields NULL and can never fabricate the root. +static const uint64_t kTableNodeAbsent = 0xFFFFFFFFFFFFFFFFull; + +// What a node's storage answers when the FRAMING ITSELF is refused rather than +// merely unnameable: a count its L cannot carry, one above the int32 cap, or a +// blob past the size cap (docs/SPEC-TABLES.md §3.1, §6.5). An unnameable type +// id commands no storage and keeps its index. This one makes the whole measure +// answer -1 with its reason. +static const int64_t kTableNodeRefused = -2; + +// ---- the numbering, on the SAVE side ---- +// +// One entry per reachable node in FIRST-VISIT order, so entry k is node index +// k + 2. The two thunks are what let one loop write a table of mixed types: the +// numbering walk knows each target's type STATICALLY at the site it numbers it, +// so it stores the instantiation there and the loop never asks what a node is. +struct TableNumbering; + +struct TableNodeEntry +{ + const void * node; + uint64_t type_id; + // the type id's MESSAGE-FORM SLOT (docs/SPEC-TABLES.md §3.3), stored where + // the numbering walk stores the id itself and for the same reason: the + // target's type is known STATICALLY at the site that numbers it, so a + // form 2 save reads the slot out of the entry instead of looking an id up. + // Every pointer target's type id is an entry of the announcement, which is + // what makes the slot a compile-time fact of a POINTERED message too. + uint64_t type_slot; + int64_t ( * measure )( const void * ctx, const TableNumbering & numbering, TableIds & ids, const void * node ); + bool ( * save )( const void * ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const void * node ); + // the same two over the MESSAGE FORM (docs/SPEC-TABLES.md §3.3): a bitpacked + // body at a bit position, its pointer indices at the width the node count + // settled + int64_t ( * message_measure )( const void * ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const void * node ); + bool ( * message_save )( const void * ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const void * node ); +}; + +struct TableNumbering +{ + TablePackMap seen; // node -> index; the ROOT is index 1, open for the whole walk + TableNodeEntry * entries = NULL; + int64_t count = 0; + int64_t capacity = 0; +}; + +// The numbering allocates through the map's pair rather than carrying a second +// copy of it: one numbering is one walk, and a walk has one allocator. +inline void TableNumberingInit( TableNumbering & n, TableAllocator allocator ) +{ + TablePackMapInit( n.seen, allocator ); + n.entries = NULL; + n.count = 0; + n.capacity = 0; +} + +inline void TableNumberingShutdown( TableNumbering & n ) +{ + TableAllocator allocator = n.seen.allocator; + TablePackMapShutdown( n.seen ); + allocator.free( allocator.context, n.entries ); + n.entries = NULL; + n.count = 0; + n.capacity = 0; +} + +// The index a numbered node was given, for the save that writes it into a +// pointer slot. False means the two walks disagree about the graph, which is a +// refusal and never a guess. +inline bool TableNumberingIndex( const TableNumbering & n, const void * node, uint64_t & index ) +{ + if ( n.seen.capacity == 0 ) { return false; } + const TablePackEntry & entry = n.seen.entries[ TablePackMapSlot( n.seen, node ) ]; + if ( entry.key != node ) { return false; } + index = (uint64_t) entry.offset; + return true; +} + +inline bool TableNumberingAppend( TableNumbering & n, const TableNodeEntry & entry ) +{ + if ( n.count == n.capacity ) + { + // GROW BY COPY, never by realloc: the allocator hook is a PAIR, and a + // game's heap is not required to have a resize primitive at all. The + // schedule quadruples, so the copying is amortized to a constant per + // entry and the growth is the same growth it always was. + int64_t capacity = n.capacity != 0 ? n.capacity * 4 : 256; + TableAllocator allocator = n.seen.allocator; + TableNodeEntry * grown = (TableNodeEntry *) allocator.alloc( allocator.context, capacity * (int64_t) sizeof( TableNodeEntry ) ); + if ( grown == NULL ) { return false; } + if ( n.entries != NULL ) + { + memcpy( grown, n.entries, (size_t) n.count * sizeof( TableNodeEntry ) ); + allocator.free( allocator.context, n.entries ); + } + n.entries = grown; + n.capacity = capacity; + } + n.entries[n.count++] = entry; + return true; +} + +// The thunks the numbering stores. Each resolves to the closure member's own +// MeasureBody / SaveBodyFields through an overload set in the member's DECLARING +// file, reached by argument-dependent lookup at instantiation — the same bridge +// the arena's TableReset uses, and the reason a numbering may span the files of +// one unit without any file naming another's members. +template +inline int64_t TableNodeMeasureThunk( const void * ctx, const TableNumbering & numbering, TableIds & ids, const void * node ) +{ + return TableNodeMeasure( *(const Ctx *) ctx, numbering, ids, *(const T *) node ); +} + +template +inline bool TableNodeSaveThunk( const void * ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const void * node ) +{ + return TableNodeSave( *(const Ctx *) ctx, numbering, w, ids, *(const T *) node ); +} + +template +inline int64_t TableNodeMessageMeasureThunk( const void * ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const void * node ) +{ + return TableNodeMessageMeasure( *(const Ctx *) ctx, numbering, index_bits, at, *(const T *) node ); +} + +template +inline bool TableNodeMessageSaveThunk( const void * ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const void * node ) +{ + return TableNodeMessageSave( *(const Ctx *) ctx, numbering, index_bits, w, *(const T *) node ); +} +// ---- a BYTE BUFFER's record (docs/SPEC-TABLES.md §2.5, §3.1) ---- +// +// A blob rides as a node record under one of two RESERVED type ids — the fold +// a table's name takes, over the keywords "bytes" and "string", which no table +// can be named — with the bytes as its body and nothing framed inside. These +// two thunks are what the numbering stores for a blob, as it stores a +// member's codec for a table: the length, and the bytes verbatim. +static const uint64_t kTableBytesTypeId = 0x2f2ec0474f1c4fe4ull; // fnv1a64( "bytes" ) +static const uint64_t kTableStringTypeId = 0x704be0d8faaffc58ull; // fnv1a64( "string" ) + +template +inline int64_t TableBlobMeasureThunk( const void *, const TableNumbering &, TableIds &, const void * node ) +{ + return (int64_t) ( (const TableBlob *) node )->length; +} + +template +inline bool TableBlobSaveThunk( const void *, const TableNumbering &, TableWriter & w, TableIds &, const void * node ) +{ + const TableBlob * blob = (const TableBlob *) node; + w.raw( (const void *) ( blob + 1 ), (int64_t) blob->length ); + return true; +} + +// and the same two on the MESSAGE FORM (§3.3): a blob record is its length at +// thirty-two raw bits, an ALIGN, then the bytes verbatim +template +inline int64_t TableBlobMessageMeasureThunk( const void *, const TableNumbering &, int64_t, int64_t at, const void * node ) +{ + const int64_t length = (int64_t) ( (const TableBlob *) node )->length; + return 32 + TableAlignBits( at + 32 ) + length * 8; +} + +template +inline bool TableBlobMessageSaveThunk( const void *, const TableNumbering &, int64_t, TableBitWriter & w, const void * node ) +{ + const TableBlob * blob = (const TableBlob *) node; + w.put( (uint64_t) blob->length, 32 ); + w.align(); + w.putbytes( (const uint8_t *) ( blob + 1 ), (int64_t) blob->length ); + return !w.overflow; +} +// TableNodeTableMeasure and TableNodeTableSave are the framing, and they are +// ONE fill rule written twice — measure derives it from the graph and save +// derives the same one, which is what makes measure == save hold across a +// pointer graph (§3.1). +// +// The field rides ONCE, under the reserved id, kind 12: the payload opens with +// the count and then carries the records back to back, each a type id +// REFERENCE, a length and a body. The reserved id is interned BEFORE the +// records, and a record's type id before its body, which is the first-use order +// the trailer is written in (§3). +template +inline int64_t TableNodeTablePayload( const Ctx & ctx, TableIds & ids, const TableNumbering & n ) +{ + int64_t payload = TableLebBytes( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + payload += TableLebBytes( ids.ref( n.entries[k].type_id ) ); + const int64_t body = n.entries[k].measure( (const void *) &ctx, n, ids, n.entries[k].node ); + if ( body < 0 ) { return -1; } + payload += TableLebBytes( (uint64_t) body ) + body; + } + return payload; +} + +template +inline int64_t TableNodeTableMeasure( const Ctx & ctx, TableIds & ids, const TableNumbering & n ) +{ + if ( n.count == 0 ) { return 0; } // a root that reaches no nodes writes none of them + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayload( ctx, ids, n ); + if ( payload < 0 ) { return -1; } + return TableLebBytes( ref ) + 1 + TableLebBytes( (uint64_t) payload ) + payload; +} + +template +inline bool TableNodeTableSave( const Ctx & ctx, TableWriter & w, TableIds & ids, const TableNumbering & n ) +{ + if ( n.count == 0 ) { return true; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayload( ctx, ids, n ); + if ( payload < 0 ) { return false; } + w.putleb( ref ); + w.put8( 12 ); // kind 12 is the opaque byte payload: a reader that cannot name the id skips by L + w.putleb( (uint64_t) payload ); + w.putleb( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.putleb( ids.ref( n.entries[k].type_id ) ); + const int64_t body = n.entries[k].measure( (const void *) &ctx, n, ids, n.entries[k].node ); + if ( body < 0 ) { return false; } + w.putleb( (uint64_t) body ); + if ( !n.entries[k].save( (const void *) &ctx, n, w, ids, n.entries[k].node ) ) { return false; } + } + return true; +} + +// ---- the numbering, on the LOAD side: a region's NODE DIRECTORY (§6.3) ---- +// +// The wire's numbering made resident: one entry per numbered node, in index +// order, position i describing node index i + 1 — so position 0 is the ROOT at +// offset 0. It is ATTRIBUTION, and attribution is separable: nothing that reads +// a structure touches it, a deref is one add on a self-relative offset, and a +// caller may release it once Load returns. +struct TableNodeDirEntry +{ + uint64_t offset; + uint64_t type_id; +}; + +// the node's extent cursor, defined with the extent runtime (docs/SPEC-TABLES.md +// §2.8, §2.9); the node map names it only through a pointer. +struct TableExtentCarve; + +// TableNodeMap is what a pointer slot resolves through while a body decodes. +struct TableNodeMap +{ + uint8_t * base = NULL; + const TableNodeDirEntry * entries = NULL; + int64_t count = 0; // the ROOT's entry included, so it is records + 1 + bool good = false; // the node table read whole; a numbering that failed resolves nothing + // WHERE THE NODES LIVE, and therefore what a resolved slot holds: a region + // takes the SELF-RELATIVE delta so a deref is one add, and the tool's + // builder path takes the node's ARENA OFFSET (§6.3). + bool arena = false; + // WHERE A MAP'S ENTRIES AND A LIST'S ELEMENTS LAND while this node's body + // decodes (docs/SPEC-TABLES.md §2.8, §2.9): the node's own extent on the + // region path and the builder's arena on the tool's. It is MUTABLE + // because the cursor belongs to ONE node's decode and the dispatch that + // owns that node holds the map by const reference, exactly as it did + // before either construct existed. The decoder's signature does not + // move for a construct it may not carry. + mutable TableExtentCarve * carve = NULL; + // and the TOOL's path's allocation front, set once: there the arrays + // are the builder's arena's rather than a node's extent. + TableWorker * worker = NULL; + // THE TOOL PATH'S REFUSAL (docs/SPEC-TABLES.md §2.9): a count above the + // int32 cap met while a body decoded. LoadBuilder answers NULL for it + // and moves no counter; mutable for the reason the cursor is. + mutable bool refused = false; +}; + +// TableNodeResolve places one node index in a pointer slot, and every failure +// is one of §4's events with the pointer left null. The declared TARGET type id +// is checked at every index, the root's included: the root carries no record +// and therefore no wire type id, so the READER'S OWN root type is what the +// claim is checked against. +inline void TableNodeResolve( const TableNodeMap & map, TableRef & slot, uint64_t index, uint64_t target, TableReport * report ) +{ + slot.value = 0; + if ( index == kTableNodeIndexNull || !map.good ) { return; } + if ( index - 1 >= (uint64_t) map.count ) + { + report->malformed = true; // an index above node_count + 1 + return; + } + const TableNodeDirEntry & entry = map.entries[index - 1]; + if ( entry.offset == kTableNodeAbsent ) + { + // a node whose type id this build could not name KEEPS ITS INDEX, and + // every pointer naming it reads null. The unknown was counted once, at + // the node, not once per pointer. + return; + } + if ( entry.type_id != target ) + { + report->kind_mismatch++; + return; + } + slot.value = map.arena ? (int64_t) entry.offset + : (int64_t) ( ( map.base + entry.offset ) - (const uint8_t *) &slot ); +} + +// ---- the record SCAN, and it is the whole of load's bound (§3.1) ---- +// +// Reading follows no reference. The scan walks the root body's top-level fields, +// finds the ONE under the reserved id, and reads records out of its payload in +// order — the field rides once, so nothing is copied to make a body contiguous +// and the generated body decoder never learns the transport exists. +struct TableNodeScan +{ + TableReader fields; // over the ROOT body, skipping past everything else + const uint8_t * payload; // the node-table field's payload + int64_t payload_size; + int64_t payload_offset; + bool opened; // the root body has been walked for the field + uint64_t declared; + int64_t records; + bool present; // the root body carries a node table at all + bool malformed; + const TableIdTable * ids; +}; + +inline TableNodeScan TableNodeScanBegin( const uint8_t * body, int64_t size, TableReport * report, const TableIdTable * ids ) +{ + TableNodeScan s = { TableReader( body, size, report, ids ), NULL, 0, 0, false, 0, 0, false, false, ids }; + return s; +} + +// find the node-table field, or answer false when the root body has none. A +// body carrying an id more than once is legal input and THE LAST OCCURRENCE +// WINS (docs/SPEC-TABLES.md §3), so the walk runs to the terminator and keeps +// the last rather than stopping at the first. +inline bool TableNodeScanOpen( TableNodeScan & s ) +{ + if ( s.opened ) { return false; } + s.opened = true; + for ( ;; ) + { + uint64_t ref = 0; + if ( !s.fields.getleb( ref ) ) { break; } + if ( ref == 0 ) { break; } // the terminator + if ( s.ids == NULL || ref > (uint64_t) s.ids->count ) { break; } + const uint64_t id = s.ids->at( ref ); + if ( !s.fields.has( 1 ) ) { break; } + const uint8_t kind = s.fields.get8(); + if ( id == kTableNodeTableFieldId ) + { + s.present = true; + if ( kind != 12 ) { s.malformed = true; return false; } + uint64_t length = 0; + if ( !s.fields.getleb( length ) || !s.fields.room( length ) ) { s.malformed = true; return false; } + s.payload = s.fields.buffer + s.fields.offset; + s.payload_size = (int64_t) length; + s.fields.offset += (int64_t) length; + continue; + } + if ( !s.fields.skip( kind ) ) { break; } + } + if ( s.payload == NULL ) { return false; } + TableReader head( s.payload, s.payload_size, s.fields.report, s.ids ); + if ( !head.getleb( s.declared ) ) { s.malformed = true; return false; } + s.payload_offset = head.offset; + return true; +} + +// the next record, or false at the end of the table — s.malformed says whether +// the end was the end or the framing giving out +inline bool TableNodeScanNext( TableNodeScan & s, uint64_t & type_id, const uint8_t * & body, int64_t & length ) +{ + if ( !s.opened && !TableNodeScanOpen( s ) ) { return false; } + if ( s.payload == NULL || s.payload_offset >= s.payload_size ) { return false; } + TableReader rec( s.payload, s.payload_size, s.fields.report, s.ids ); + rec.offset = s.payload_offset; + uint64_t ref = 0; + if ( !rec.getleb( ref ) || ref == 0 || s.ids == NULL || ref > (uint64_t) s.ids->count ) + { + s.malformed = true; // a type id reference of 0, or one past the table + return false; + } + type_id = s.ids->at( ref ); + uint64_t declared_length = 0; + if ( !rec.getleb( declared_length ) ) + { + s.malformed = true; // a record whose length is damaged + return false; + } + if ( declared_length > (uint64_t) ( s.payload_size - rec.offset ) ) + { + s.malformed = true; // a record whose length runs past its field + return false; + } + body = s.payload + rec.offset; + length = (int64_t) declared_length; + s.payload_offset = rec.offset + length; + s.records++; + return true; +} + +// The record scan is AUTHORITATIVE: node_count is data from the wire, and a +// count that disagrees with the scan is malformed. Nothing is sized from it +// before the scan has confirmed it. +inline bool TableNodeScanWhole( TableNodeScan & s ) +{ + if ( s.malformed ) { return false; } + if ( !s.present ) { return true; } // no node table at all is not a broken one + return s.declared == (uint64_t) s.records; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_ARENA + +#ifndef MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES +#define MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES + +namespace mapdemo { + +// ---- the NODE TABLE on the message wire (docs/SPEC-TABLES.md §3.1, §3.3) ---- +// +// THE NODE TABLE, WHEN A BODY HAS ONE, IS THE FIRST FIELD OF THE ROOT BODY: the +// reserved id as a reference, the node count at THIRTY-TWO RAW BITS, then the +// records back to back, each a type id reference and a body: a table's fields +// end at their own zero reference, and a blob's body is a length, an align and +// its bytes. A root +// that reaches no node elides the field, like every other empty thing. +// +// Measure derives the numbering from the graph and save derives the same one, +// and the two thunks stored at numbering time are what let one loop write a +// table of mixed types. +template +inline int64_t TableMessageNodeTableMeasure( const Ctx & ctx, const TableNumbering & n, int64_t index_bits, int64_t at ) +{ + if ( n.count == 0 ) { return 0; } // a root that reaches no nodes writes none of them + int64_t bits = kTableMessageRefBitsHere + 32; + for ( int64_t k = 0; k < n.count; k++ ) + { + bits += kTableMessageRefBitsHere; + const int64_t body = n.entries[k].message_measure( (const void *) &ctx, n, index_bits, at + bits, n.entries[k].node ); + if ( body < 0 ) { return -1; } + bits += body; + } + return bits; +} + +template +inline bool TableMessageNodeTableSave( const Ctx & ctx, const TableNumbering & n, int64_t index_bits, TableBitWriter & w ) +{ + if ( n.count == 0 ) { return true; } + w.put( kTableNodeTableFieldSlot, kTableMessageRefBitsHere ); + w.put( (uint64_t) n.count, 32 ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.put( n.entries[k].type_slot, kTableMessageRefBitsHere ); + if ( !n.entries[k].message_save( (const void *) &ctx, n, index_bits, w, n.entries[k].node ) ) { return false; } + } + return !w.overflow; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_MESSAGE_NODES + +#ifndef MAPDEMO_SCHEMA_TABLE_RETAIN +#define MAPDEMO_SCHEMA_TABLE_RETAIN + +namespace mapdemo { + +// ---- RETAIN-UNKNOWN (docs/SPEC-TABLES.md §6.6) ---- +// +// A REGION ROUND TRIP AND ONLY THAT: LoadRetain is Load's path into a region +// and SaveRetain saves from that same region. The builder path carries no +// retention, because a builder has no node directory to anchor a record on and +// re-derives its numbering from the reader's declaration order. +// +// Nothing here allocates. The record bytes and the retained-id list are the +// caller's storage, declared with their capacities, and a record that does not +// fit whole is dropped with one retain_lost. + +// THE PATH NAMES THE BODY, and it is the REGION's own address (§6.6). Step one +// is the node's index in the region's node directory, 1 for the root body and +// k for the node at directory position k - 1. Every further step is the PAIR: +// the field ordinal in the body the step descends from, in the READER's own +// declaration order, and the element index inside that field: zero for a +// scalar body, the element's index for an array of any of the four kinds, the +// ARM's OWN ORDINAL for a union, and the key's slot for a map. +static const int32_t kTableRetainDepthMax = 5; + +struct TableRetainStep +{ + uint32_t ordinal; + uint32_t index; +}; + +// at is the node's own address, which is what the SAVE side matches on: a +// record carries the directory INDEX and the directory answers the address in +// one add, so neither side ever searches a numbering. +struct TableRetainPath +{ + const void * at; + uint32_t node; + int32_t depth; + TableRetainStep steps[ kTableRetainDepthMax ]; +}; + +inline TableRetainPath TableRetainPathRoot( const void * at, uint32_t node ) +{ + TableRetainPath path; + path.at = at; + path.node = node; + path.depth = 0; + return path; +} + +// A STEP IS COMPUTED LOCALLY, at the moment the walk descends (§6.6), and it +// is taken by VALUE so that a descent is an expression: both sides walk the +// same declaration order, so neither numbers a tree and neither pops. +inline TableRetainPath TableRetainStepInto( const TableRetainPath & path, uint32_t ordinal, uint32_t index ) +{ + TableRetainPath out = path; + if ( out.depth < kTableRetainDepthMax ) + { + out.steps[ out.depth ].ordinal = ordinal; + out.steps[ out.depth ].index = index; + } + out.depth++; + return out; +} + +// THE CALLER'S TWO STORES (§6.6): the record bytes and the retained ids, each +// a pointer, a capacity and what has been used of it. A retention buffer +// belongs to ONE loaded region, and the next LoadRetain into it resets both. +struct TableRetain +{ + // AN ENTRY IS THE ID AND ITS SLOT IN THE TRAILER BEING WRITTEN. The two + // stores are numbered into ONE trailer in merged first-use order, so an + // index into this list is not the number a second reference wants and the + // slot rides beside the id. The layout is this port's own. + struct Id + { + uint64_t id; + int32_t slot; + }; + + uint8_t * bytes = NULL; + int64_t capacity = 0; + int64_t used = 0; + Id * ids = NULL; + int32_t id_capacity = 0; + int32_t id_used = 0; + int32_t count = 0; // records held + + // the REGION this buffer belongs to: a record carries a directory index + // and the save resolves it here, so nothing searches and nothing allocates + const uint8_t * base = NULL; + const TableNodeDirEntry * directory = NULL; + int64_t directory_count = 0; +}; + +// A RETAINED RECORD IS READER-PRIVATE (§6.6). It is not a wire form: no form +// byte, no version, no declared byte order, and nothing ever writes one to +// disk or hands one to another process. What it must CARRY is the body it +// belongs to, and the field's identity and bytes with every reference +// resolved. This layout is one sound way to carry them and nothing compares +// two ports' buffers. +// +// u32 record bytes, this header included +// u32 node the path's first step +// u32 depth the step pairs that follow +// u32 payload bytes +// u64 field id +// u8 kind +// u8 placed the save's own mark, cleared before every save +// depth x { u32 ordinal, u32 index } +// payload the field's payload with every reference resolved +static const int64_t kTableRetainRecordHeader = 26; + +inline uint32_t TableRetainRead32( const uint8_t * p ) +{ + return uint32_t( p[0] ) | uint32_t( p[1] ) << 8 | uint32_t( p[2] ) << 16 | uint32_t( p[3] ) << 24; +} + +inline uint64_t TableRetainRead64( const uint8_t * p ) +{ + return uint64_t( TableRetainRead32( p ) ) | ( uint64_t( TableRetainRead32( p + 4 ) ) << 32 ); +} + +inline void TableRetainWrite32( uint8_t * p, uint32_t v ) +{ + p[0] = uint8_t( v ); p[1] = uint8_t( v >> 8 ); p[2] = uint8_t( v >> 16 ); p[3] = uint8_t( v >> 24 ); +} + +inline void TableRetainWrite64( uint8_t * p, uint64_t v ) +{ + TableRetainWrite32( p, uint32_t( v ) ); + TableRetainWrite32( p + 4, uint32_t( v >> 32 ) ); +} + +inline int64_t TableRetainRecordBytes( const uint8_t * record ) { return (int64_t) TableRetainRead32( record ); } +inline uint32_t TableRetainRecordNode( const uint8_t * record ) { return TableRetainRead32( record + 4 ); } +inline int32_t TableRetainRecordDepth( const uint8_t * record ) { return (int32_t) TableRetainRead32( record + 8 ); } +inline int64_t TableRetainRecordPayloadBytes( const uint8_t * record ) { return (int64_t) TableRetainRead32( record + 12 ); } +inline uint64_t TableRetainRecordId( const uint8_t * record ) { return TableRetainRead64( record + 16 ); } +inline uint8_t TableRetainRecordKind( const uint8_t * record ) { return record[24]; } +inline bool TableRetainRecordPlaced( const uint8_t * record ) { return record[25] != 0; } +inline const uint8_t * TableRetainRecordSteps( const uint8_t * record ) { return record + kTableRetainRecordHeader; } +inline const uint8_t * TableRetainRecordPayload( const uint8_t * record ) +{ + return record + kTableRetainRecordHeader + 8 * (int64_t) TableRetainRecordDepth( record ); +} +inline uint8_t * TableRetainRecordPayload( uint8_t * record ) +{ + return record + kTableRetainRecordHeader + 8 * (int64_t) TableRetainRecordDepth( record ); +} + +// THE RECORD'S OWN BODY, resolved through the directory the buffer holds. A +// node index names one node for the life of the region, so this is one add. +inline const void * TableRetainRecordAt( const TableRetain & retain, const uint8_t * record ) +{ + const uint32_t node = TableRetainRecordNode( record ); + if ( retain.directory == NULL || node == 0 || (int64_t) node > retain.directory_count ) { return NULL; } + return (const void *) ( retain.base + retain.directory[ node - 1 ].offset ); +} + +// Does this record belong to the body the walk is standing in? The node first, +// which rejects almost everything in one compare, then the step pairs. +inline bool TableRetainRecordHere( const TableRetain & retain, const uint8_t * record, const TableRetainPath & path ) +{ + if ( TableRetainRecordDepth( record ) != path.depth ) { return false; } + if ( TableRetainRecordAt( retain, record ) != path.at ) { return false; } + const uint8_t * steps = TableRetainRecordSteps( record ); + for ( int32_t i = 0; i < path.depth; i++ ) + { + if ( TableRetainRead32( steps + 8 * i ) != path.steps[i].ordinal ) { return false; } + if ( TableRetainRead32( steps + 8 * i + 4 ) != path.steps[i].index ) { return false; } + } + return true; +} + +// EVERY ID THIS BUILD CAN NAME, ascending: the set TableIds's capacity is +// derived from. An id inside a retained record takes its trailer entry from +// the GENERATED table when it is here and from the CALLER's list otherwise, so +// no retained id ever enters the generated table and no id is written twice. +static const int32_t kTableRetainKnownIds = 76; +static const uint64_t kTableRetainKnown[ kTableRetainKnownIds ] = { + 0x033de3f1246bba76ull, 0x036ffe7360826852ull, 0x099d588c4981296dull, 0x0a53e00afba279afull, + 0x0c2643993e3ece2eull, 0x11e7ec757c03c70aull, 0x124250ad5a5b6d14ull, 0x1404200dab337086ull, + 0x14e2eaab9cde925bull, 0x18691a70a0e3fe31ull, 0x1a08aa1921ca5cafull, 0x1c84390d304f4f42ull, + 0x1f6459a2cea1fc02ull, 0x1f781dc01a2b5152ull, 0x2492f5fb1b05b45eull, 0x294a5c4913e1ad44ull, + 0x294fa1b3f0f5f070ull, 0x29cf72329075c5aaull, 0x2b7dea192bb7be29ull, 0x2f2ec0474f1c4fe4ull, + 0x3231d0dc6fe30d4aull, 0x3dc94a19365b10ecull, 0x3e8426f7e349c9dcull, 0x437dfc8ab2566816ull, + 0x4fda04cbc245e18bull, 0x52cfa1d198476806ull, 0x5de00b6a76064442ull, 0x610dcbb318a2e4faull, + 0x63dfa0c4a4b3815dull, 0x6dd8dc6c5fdae3ceull, 0x704be0d8faaffc58ull, 0x70551ff29550f15dull, + 0x755332609c470fbdull, 0x758252d2d1b14f0dull, 0x79d594675e391090ull, 0x7b024c46e98d3404ull, + 0x7ce4fd9430e80ceaull, 0x7d015e53d7cb2c7cull, 0x7f6548303072b061ull, 0x7f69d4b5288ba9cfull, + 0x8119d921e2250c6aull, 0x816c207ba6213983ull, 0x8dc5f55c70e0f637ull, 0x8ec370bd37dc5e06ull, + 0x9b18b54fbe8e2161ull, 0xa013e119fec906fbull, 0xa0b562a796b69487ull, 0xa3a7061ff10a8138ull, + 0xa633f1f655715ccaull, 0xab01daa76a48769full, 0xaf05dfb30c5ca3deull, 0xafb728fff268814full, + 0xb1e5e28e4479a274ull, 0xb413964e3571a316ull, 0xb4578774a78fb150ull, 0xbc08b7f228c93506ull, + 0xbf82010f6f71eae9ull, 0xc4bcadba8e631b86ull, 0xc5b2a72c0845a253ull, 0xc85b940060088651ull, + 0xd5c2bb95d63e6331ull, 0xda73c178dfcf57b7ull, 0xdcdbddf89c9310a1ull, 0xde1c38d5bac1485dull, + 0xe1185043515c812bull, 0xe68c2e6bb1ee5646ull, 0xe756c0190570ccb5ull, 0xe8130af045a036f8ull, + 0xea7bdd2b70c8c2bbull, 0xec07a2f760550a91ull, 0xf03923dbb2943618ull, 0xf96b15cd3921d4a6ull, + 0xfa903574575fc678ull, 0xfd4d194e1652b207ull, 0xfedcb40b5d600538ull, 0xffffffffffffffffull, +}; + +inline bool TableRetainNameable( uint64_t id ) +{ + int32_t low = 0, high = kTableRetainKnownIds - 1; + while ( low <= high ) + { + const int32_t mid = low + ( high - low ) / 2; + if ( kTableRetainKnown[mid] == id ) { return true; } + if ( kTableRetainKnown[mid] < id ) { low = mid + 1; } else { high = mid - 1; } + } + return false; +} + +// THE TWO STORES, NUMBERED INTO ONE TRAILER in merged first-use order (§6.6). +// It answers the surface TableIds answers, ref, count, truncate and +// overflow, so the retain family's codec is the plain one with its names +// changed, and +// the GENERATED TABLE IS UNTOUCHED: its capacity, its overflow rule and its +// -1 stand exactly as they are for every save. +struct TableRetainIds +{ + TableIds known; + int32_t known_slot[ TableIds::kCapacity ]; + TableRetain * retain; + int32_t count; + bool overflow; + bool lost; // a retained id past the caller's capacity: the record is dropped + + TableRetainIds( TableRetain * to_retain ) : retain( to_retain ), count( 0 ), overflow( false ), lost( false ) {} + + // an id this build CAN name, which is every id the generated codec writes + uint64_t ref( uint64_t id ) + { + const int32_t before = known.count; + const uint64_t k = known.ref( id ); + if ( known.overflow ) { overflow = true; return 1; } + if ( known.count != before ) { known_slot[ (int32_t) k - 1 ] = ++count; } + return (uint64_t) known_slot[ (int32_t) k - 1 ]; + } + + // an id from INSIDE a retained record. A retained id takes its entry from + // the caller's list, and one past the capacity sets lost: the record is + // dropped, nothing else about the save changes, and the save is never + // refused (§6.6). + uint64_t record_ref( uint64_t id ) + { + if ( TableRetainNameable( id ) ) { return ref( id ); } + if ( retain == NULL ) { lost = true; return 0; } + for ( int32_t i = 0; i < retain->id_used; i++ ) + { + if ( retain->ids[i].id == id ) { return (uint64_t) retain->ids[i].slot; } + } + if ( retain->id_used >= retain->id_capacity ) { lost = true; return 0; } + retain->ids[ retain->id_used ].id = id; + retain->ids[ retain->id_used ].slot = ++count; + retain->id_used++; + return (uint64_t) count; + } + + // undo every entry taken since mark, in either store. Both are appended in + // slot order, so an entry removed is the last one of its store. + void truncate( int32_t mark ) + { + while ( known.count > 0 && known_slot[ known.count - 1 ] > mark ) { known.truncate( known.count - 1 ); } + while ( retain != NULL && retain->id_used > 0 && retain->ids[ retain->id_used - 1 ].slot > mark ) { retain->id_used--; } + count = mark; + } +}; + +// THE FILE STILL CARRIES ONE ID TABLE (§3): the split is the writer's storage +// rather than the wire's, and the trailer is one merge of two slot-ordered +// stores. +inline int64_t TableRetainIdsBytes( const TableRetainIds & ids ) { return int64_t( ids.count ) * 8 + 8; } + +// A TWO-WAY MERGE over the stores' own slot order, and not a scan for each +// slot. Every entry either store holds took its slot from the same counter, so +// the two runs interleave to exactly the slots 1 to count and the merge has no +// case for a slot neither store took. +inline void TableRetainIdsWrite( TableWriter & w, const TableRetainIds & ids ) +{ + const int32_t retained = ids.retain != NULL ? ids.retain->id_used : 0; + int32_t i = 0, j = 0; + while ( i < ids.known.count || j < retained ) + { + if ( j >= retained || ( i < ids.known.count && ids.known_slot[i] < ids.retain->ids[j].slot ) ) + { + w.put64( ids.known.ids[i] ); + i++; + continue; + } + w.put64( ids.retain->ids[j].id ); + j++; + } + w.put64( uint64_t( ids.count ) ); +} + +// ---- THE RESOLVING WALK (§6.6) ---- +// +// A reference names a SLOT of the file's id table, so a verbatim copy +// re-emitted into a file whose table is ordered differently would point at +// other names in silence. A retained record therefore holds the field with +// every reference replaced by the sixty-four-bit id it names, and every length +// that frames a rewritten reference recomputed. +// +// THE WALK IS AN INTERPRETATION, AND ITS VERDICT IS STATED: it reads kind +// bytes, lengths and references and nothing else. No value is decoded, no +// bound is checked, no branch is taken on a payload byte, and anything it +// cannot frame DROPS THE RECORD, counts one retain_lost, and never raises +// malformed on the plain read. +// +// THE WALK IS ONE PASS EACH WAY, and its cost is linear in the record's own +// bytes. Every length that frames a content in the resolved form is a fixed +// slot rather than a canonical LEB128, so the capture reserves it, writes the +// content, and fills the slot in behind it. A spelling that had to know the +// resolved size before writing it would have to walk each content twice, once +// at every level, and the file chooses the nesting. +// +// A retained record's inner nesting is the WRITER's and not this build's, so +// it is the one depth on this path a file can drive. The cap counts NESTED +// BODIES, and a record past it is dropped on the same rule as any other shape +// the walk cannot take. Time no longer rests on it: it is a small stated +// constant and nothing more. +static const int32_t kTableRetainWalkDepthMax = 64; + +// the three RESERVED ids (§3.1, §3.3). One inside a retained record's payload +// would be re-emitted into a nested body, where it is malformed, so meeting +// one drops the record. +inline bool TableRetainReservedId( uint64_t id ) +{ + return id == kTableNodeTableFieldId || id == kTableBuildVersionFieldId || id == kTableMessageVocabularyFieldId; +} + +struct TableRetainIn +{ + const uint8_t * in; + int64_t size; + int64_t at; + const TableIdTable * ids; + uint8_t * out; // NULL: measuring, and nothing is written + int64_t out_at; +}; + +inline void TableRetainInRaw( TableRetainIn & s, const uint8_t * from, int64_t bytes ) +{ + if ( s.out != NULL ) { memcpy( s.out + s.out_at, from, (size_t) bytes ); } + s.out_at += bytes; +} + +inline void TableRetainInLeb( TableRetainIn & s, uint64_t v ) +{ + uint8_t b[10]; + int64_t n = 0; + while ( v >= 0x80 ) { b[n++] = uint8_t( v ) | 0x80; v >>= 7; } + b[n++] = uint8_t( v ); + TableRetainInRaw( s, b, n ); +} + +inline void TableRetainInId( TableRetainIn & s, uint64_t id ) +{ + uint8_t b[8]; + TableRetainWrite64( b, id ); + TableRetainInRaw( s, b, 8 ); +} + +inline bool TableRetainInLebRead( TableRetainIn & s, uint64_t & value ) +{ + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( s.at >= s.size ) { return false; } + const uint8_t b = s.in[ s.at++ ]; + if ( i == 9 && b > 1 ) { return false; } + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) { return i == 0 || b != 0; } + shift += 7; + } + return false; +} + +// one REFERENCE resolved to the id it names. A zero reference is the wire's +// own "no id", the enum's None and the union's empty arm, and rides as the +// id zero. A reference above the entry count, a reference at an id-table entry +// of zero, and a reference at a reserved id are each damage the plain read +// never looked at, and each drops the record. +inline bool TableRetainInRef( TableRetainIn & s, bool zero_allowed ) +{ + uint64_t ref = 0; + if ( !TableRetainInLebRead( s, ref ) ) { return false; } + if ( ref == 0 ) + { + if ( !zero_allowed ) { return false; } + TableRetainInId( s, 0 ); + return true; + } + if ( s.ids == NULL || ref > (uint64_t) s.ids->count ) { return false; } + const uint64_t id = s.ids->at( ref ); + if ( id == 0 || TableRetainReservedId( id ) ) { return false; } + TableRetainInId( s, id ); + return true; +} + +inline int64_t TableRetainInPayload( TableRetainIn & s, uint8_t kind, int32_t depth ); +inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ); + +// ONE FRAMED LENGTH in the resolved form: a pair of fixed u32. The first is +// the RESOLVED byte count of the content it frames, reserved here and written +// once the content is out. The second is the SAVE's scratch, left zero by the +// capture and filled by the walk that emits. +// +// The record is the reader's own storage and nothing outside this family ever +// reads it, so a length may be written after the bytes it measures. That is +// the whole of what makes the walk one pass. +static const int64_t kTableRetainSlotBytes = 8; + +inline int64_t TableRetainInSlot( TableRetainIn & s ) +{ + const int64_t at = s.out_at; + const uint8_t zero[ kTableRetainSlotBytes ] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + TableRetainInRaw( s, zero, kTableRetainSlotBytes ); + return at; +} + +inline void TableRetainInPatch( TableRetainIn & s, int64_t slot, int64_t resolved ) +{ + if ( s.out != NULL ) { TableRetainWrite32( s.out + slot, (uint32_t) resolved ); } +} + +// one framed CONTENT: the slot, the content, and the slot filled in behind it. +// The measuring pass takes the same path and reserves the same fixed width, so +// the size it answers is the size the writing pass lays down. +inline int64_t TableRetainInFramed( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ) +{ + const int64_t slot = TableRetainInSlot( s ); + const int64_t resolved = TableRetainInContent( s, kind, length, depth ); + if ( resolved < 0 || resolved > 0xFFFFFFFFll ) { return -1; } + TableRetainInPatch( s, slot, resolved ); + return resolved; +} + +inline int64_t TableRetainInContent( TableRetainIn & s, uint8_t kind, int64_t length, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return -1; } + if ( length < 0 || s.at + length > s.size ) { return -1; } + const int64_t end = s.at + length; + const int64_t began = s.out_at; + switch ( kind ) + { + case 13: // a table BODY: fields, then the zero reference + { + for ( ;; ) + { + uint64_t ref = 0; + const int64_t mark = s.at; + if ( !TableRetainInLebRead( s, ref ) ) { return -1; } + // THE TERMINATOR IS A REFERENCE, and a reference in the + // resolved form is a fixed eight-byte id: the zero that ends a + // body rides at the width every other one does. + if ( ref == 0 ) { TableRetainInId( s, 0 ); break; } + s.at = mark; + if ( !TableRetainInRef( s, false ) ) { return -1; } + if ( s.at >= end ) { return -1; } + const uint8_t field_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &field_kind, 1 ); + if ( TableRetainInPayload( s, field_kind, depth ) < 0 ) { return -1; } + if ( s.at > end ) { return -1; } + } + break; + } + case 14: // an ARRAY body: the element kind, the count, then the elements + { + if ( s.at >= end ) { return -1; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainInLebRead( s, n ) ) { return -1; } + TableRetainInLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( TableRetainInPayload( s, elem_kind, depth ) < 0 ) { return -1; } + if ( s.at > end ) { return -1; } + } + break; + } + case 16: // an ENUM-KEYED body: N triples of a KEY REFERENCE, an L and the element + { + if ( s.at >= end ) { return -1; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainInLebRead( s, n ) ) { return -1; } + TableRetainInLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + // A KEYED BODY'S KEYS RESOLVE AT EVERY ELEMENT KIND (§6.6, §3.2) + if ( !TableRetainInRef( s, false ) ) { return -1; } + uint64_t slot_bytes = 0; + if ( !TableRetainInLebRead( s, slot_bytes ) ) { return -1; } + if ( slot_bytes > (uint64_t) ( end - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, elem_kind, (int64_t) slot_bytes, depth + 1 ) < 0 ) { return -1; } + } + break; + } + case 15: case 30: + // A UNION ARM AND AN ENUM'S VARIANT REFERENCE RESOLVE AS A FRAMED + // CONTENT TOO (§6.6): a kind 15 arm whose own payload is a union, + // and a kind 16 slot whose element kind is 15 or 30, both arrive + // here, and both carry a reference. Copying them as bytes would + // re-emit a reference into a permuted trailer, where it names + // another id, and would let a kind 17 UNDER A KIND 15 ARM through + // a walk whose whole job is to catch it. + if ( TableRetainInPayload( s, kind, depth ) < 0 ) { return -1; } + break; + case 17: return -1; // A NODE INDEX ANYWHERE DROPS THE WHOLE RECORD (§6.6) + default: + // every other content is bytes: a string, wide text, an escape, a + // payload-free kind, a scalar under a keyed slot's own length + TableRetainInRaw( s, s.in + s.at, length ); + s.at += length; + break; + } + if ( s.at != end ) { return -1; } + return s.out_at - began; +} + +// the depth a payload carries is its enclosing body's: only a framed CONTENT +// is a level, and TableRetainInContent is the one place the cap is read. +inline int64_t TableRetainInPayload( TableRetainIn & s, uint8_t kind, int32_t depth ) +{ + const int64_t began = s.out_at; + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: // the fixed-width kinds, by width + case 3: case 7: case 21: case 26: + case 4: case 8: case 10: case 22: case 27: + case 5: case 9: case 11: case 23: case 28: + case 18: case 19: case 24: case 29: + { + int64_t width = 1; + switch ( kind ) + { + case 3: case 7: case 21: case 26: width = 2; break; + case 4: case 8: case 10: case 22: case 27: width = 4; break; + case 5: case 9: case 11: case 23: case 28: width = 8; break; + case 18: case 19: case 24: case 29: width = 16; break; + default: width = 1; break; + } + if ( s.at + width > s.size ) { return -1; } + TableRetainInRaw( s, s.in + s.at, width ); + s.at += width; + break; + } + case 12: case 31: case 32: case 33: // L, then L bytes, nothing framed inside + { + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + TableRetainInLeb( s, length ); + TableRetainInRaw( s, s.in + s.at, (int64_t) length ); + s.at += (int64_t) length; + break; + } + case 13: case 14: case 16: // L, then a body the walk resolves + { + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, kind, (int64_t) length, depth + 1 ) < 0 ) { return -1; } + break; + } + case 15: // a UNION: the arm id reference, and when it is not zero its kind, L and payload + { + const int64_t mark = s.at; + uint64_t arm = 0; + if ( !TableRetainInLebRead( s, arm ) ) { return -1; } + s.at = mark; + if ( !TableRetainInRef( s, true ) ) { return -1; } + if ( arm == 0 ) { break; } + if ( s.at >= s.size ) { return -1; } + const uint8_t arm_kind = s.in[ s.at++ ]; + TableRetainInRaw( s, &arm_kind, 1 ); + uint64_t length = 0; + if ( !TableRetainInLebRead( s, length ) ) { return -1; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return -1; } + if ( TableRetainInFramed( s, arm_kind, (int64_t) length, depth + 1 ) < 0 ) { return -1; } + break; + } + case 30: // an ENUM's variant reference, zero for None + { + if ( !TableRetainInRef( s, true ) ) { return -1; } + break; + } + case 17: return -1; // A NODE INDEX (§3.1): the whole record goes with it + default: return -1; // a kind this walk cannot frame + } + return s.out_at - began; +} + +// ---- CAPTURE: the load side (§6.6) ---- +// +// The field is skipped by its framing exactly as it always was and counted +// unknown exactly as it always was, so a full buffer degrades to the default +// behavior one field at a time. False is what r.skip( kind ) answers false +// for, and nothing else: retention can lose a field, it can never turn a good +// read into a bad one. +inline bool TableRetainCapture( TableRetain * retain, TableReader & r, const TableRetainPath & path, + uint64_t field_id, uint8_t kind ) +{ + const int64_t start = r.offset; + if ( !r.skip( kind ) ) { return false; } + if ( retain == NULL ) { return true; } + const int64_t wire_bytes = r.offset - start; + + TableRetainIn probe; + probe.in = r.buffer + start; + probe.size = wire_bytes; + probe.at = 0; + probe.ids = r.ids; + probe.out = NULL; + probe.out_at = 0; + const int64_t payload = TableRetainInPayload( probe, kind, 0 ); + if ( payload < 0 || probe.at != wire_bytes ) { r.report->retain_lost++; return true; } + + const int64_t need = kTableRetainRecordHeader + 8 * (int64_t) path.depth + payload; + if ( need > 0xFFFFFFFFll || retain->used + need > retain->capacity ) + { + // REFUSAL IS PER RECORD AND NEVER PARTIAL: the buffer never holds a + // truncated field, and the read continues (§6.6) + r.report->retain_lost++; + return true; + } + uint8_t * record = retain->bytes + retain->used; + TableRetainWrite32( record, (uint32_t) need ); + TableRetainWrite32( record + 4, path.node ); + TableRetainWrite32( record + 8, (uint32_t) path.depth ); + TableRetainWrite32( record + 12, (uint32_t) payload ); + TableRetainWrite64( record + 16, field_id ); + record[24] = kind; + record[25] = 0; + for ( int32_t i = 0; i < path.depth; i++ ) + { + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i, path.steps[i].ordinal ); + TableRetainWrite32( record + kTableRetainRecordHeader + 8 * i + 4, path.steps[i].index ); + } + TableRetainIn write; + write.in = r.buffer + start; + write.size = wire_bytes; + write.at = 0; + write.ids = r.ids; + write.out = record + kTableRetainRecordHeader + 8 * (int64_t) path.depth; + write.out_at = 0; + if ( TableRetainInPayload( write, kind, 0 ) < 0 ) { r.report->retain_lost++; return true; } + retain->used += need; + retain->count++; + r.report->retained++; + return true; +} + +// LoadRetain RESETS BOTH STORES and writes into neither list (§6.6): a +// retained record carries its field's identity in the record itself, with +// every reference resolved. +inline void TableRetainReset( TableRetain * retain, const TableNodeMap & nodes, const uint8_t * region ) +{ + if ( retain == NULL ) { return; } + retain->used = 0; + retain->id_used = 0; + retain->count = 0; + retain->base = region; + retain->directory = nodes.entries; + retain->directory_count = nodes.count; +} + +// ---- RECORD LIFETIME (docs/SPEC-TABLES.md §6.6) ---- +// +// A RETAINED RECORD BELONGS TO THE BODY OCCURRENCE THAT CARRIED IT, AND DIES +// WITH IT. Legal input can carry a known child body twice, and the later +// occurrence resets the child and wins whole (§3, §4): the records retained +// under the earlier occurrence go with the values it held. The discard moves +// NEITHER counter. The writer superseded the data, so nothing was lost that +// the load could have kept, and retained counted the record when its bytes +// were kept and does not fall when they are let go. +// +// The occurrences are four, and each is a body the wire lets a writer put down +// again: a repeated TABLE field, by value or under ?, a UNION whose arm is +// written again, a MAP's duplicate key, and a KEYED-ARRAY slot written again. +// The FIELD form covers the three where the field itself is read again, arm +// switches and shrinking arrays included; the BODY form covers a duplicate key +// inside one occurrence of a map, where the field is read once and the entry +// twice. +inline bool TableRetainUnder( const uint8_t * record, const void * at, const TableRetain & retain, + const TableRetainPath & path, bool field, uint32_t ordinal ) +{ + if ( TableRetainRecordAt( retain, record ) != at ) { return false; } + const int32_t depth = TableRetainRecordDepth( record ); + if ( field ) + { + if ( depth <= path.depth ) { return false; } + } + else if ( depth < path.depth ) { return false; } + const uint8_t * steps = TableRetainRecordSteps( record ); + for ( int32_t i = 0; i < path.depth; i++ ) + { + if ( TableRetainRead32( steps + 8 * i ) != path.steps[i].ordinal ) { return false; } + if ( TableRetainRead32( steps + 8 * i + 4 ) != path.steps[i].index ) { return false; } + } + if ( field && TableRetainRead32( steps + 8 * path.depth ) != ordinal ) { return false; } + return true; +} + +inline void TableRetainDiscard( TableRetain * retain, const TableRetainPath & path, bool field, uint32_t ordinal ) +{ + if ( retain == NULL || retain->count == 0 ) { return; } + int64_t read = 0, write = 0; + int32_t kept = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + const int64_t bytes = TableRetainRecordBytes( retain->bytes + read ); + if ( !TableRetainUnder( retain->bytes + read, path.at, *retain, path, field, ordinal ) ) + { + if ( write != read ) { memmove( retain->bytes + write, retain->bytes + read, (size_t) bytes ); } + write += bytes; + kept++; + } + read += bytes; + } + retain->used = write; + retain->count = kept; +} + +inline void TableRetainDiscardBody( TableRetain * retain, const TableRetainPath & path ) +{ + TableRetainDiscard( retain, path, false, 0 ); +} + +inline void TableRetainDiscardField( TableRetain * retain, const TableRetainPath & path, uint32_t ordinal ) +{ + TableRetainDiscard( retain, path, true, ordinal ); +} + +// ---- EMIT: the save side (§6.6) ---- +// +// The record read back the other way: every resolved id becomes the reference +// the trailer being written gives it, and every length is recomputed against +// the references' new widths. The walk is the capture's mirror and the same +// damage rules apply, except that damage cannot be met: these bytes are the +// reader's own. +// +// A WIRE LENGTH IS CANONICAL LEB128 AND RIDES BEFORE ITS CONTENT, so this side +// cannot fill a slot in behind the bytes the way the capture does. It takes +// one POST-ORDER pass instead: measuring computes each content's wire size and +// leaves it in that content's own scratch slot, and the emit reads the size +// there rather than walking for it. Measuring runs immediately before the +// emit, on the same record and the same id table, which is what makes the two +// readings one walk. +struct TableRetainOut +{ + uint8_t * in; // the record: only a framed length's scratch half is written + int64_t size; + int64_t at; + TableRetainIds * ids; + TableWriter * w; // NULL: measuring, and the scratch slots are being filled + int64_t bytes; +}; + +inline void TableRetainOutRaw( TableRetainOut & s, const uint8_t * from, int64_t bytes ) +{ + if ( s.w != NULL ) { s.w->raw( from, bytes ); } + s.bytes += bytes; +} + +inline void TableRetainOutLeb( TableRetainOut & s, uint64_t v ) +{ + if ( s.w != NULL ) { s.w->putleb( v ); } + s.bytes += TableLebBytes( v ); +} + +inline bool TableRetainOutLebRead( TableRetainOut & s, uint64_t & value ) +{ + value = 0; + uint32_t shift = 0; + for ( int32_t i = 0; i < 10; i++ ) + { + if ( s.at >= s.size ) { return false; } + const uint8_t b = s.in[ s.at++ ]; + value |= uint64_t( b & 0x7F ) << shift; + if ( ( b & 0x80 ) == 0 ) { return true; } + shift += 7; + } + return false; +} + +inline bool TableRetainOutRef( TableRetainOut & s ) +{ + if ( s.at + 8 > s.size ) { return false; } + const uint64_t id = TableRetainRead64( s.in + s.at ); + s.at += 8; + if ( id == 0 ) { TableRetainOutLeb( s, 0 ); return true; } // the wire's own no-id + const uint64_t ref = s.ids->record_ref( id ); + if ( s.ids->lost || s.ids->overflow ) { return false; } + TableRetainOutLeb( s, ref ); + return true; +} + +inline bool TableRetainOutPayload( TableRetainOut & s, uint8_t kind, int32_t depth ); +inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t length, int32_t depth ); + +// ONE FRAMED CONTENT, read out of the record's fixed slot and written with the +// canonical LEB128 length this wire wants. The ids it names are interned on +// the way past, which is what makes measure and save one walk in two readings, +// exactly as every other body on this wire is. +// +// MEASURING walks the content, then leaves the wire size it found in the +// scratch half of the slot. EMITTING reads that size, writes it, and CHECKS +// the content against it: a size no measure of this save left there is a +// record refused rather than a length that does not frame what follows. +inline bool TableRetainOutFramed( TableRetainOut & s, uint8_t kind, int32_t depth ) +{ + if ( s.at + kTableRetainSlotBytes > s.size ) { return false; } + uint8_t * const slot = s.in + s.at; + const int64_t resolved = (int64_t) TableRetainRead32( slot ); + s.at += kTableRetainSlotBytes; + if ( s.w == NULL ) + { + const int64_t began = s.bytes; + if ( !TableRetainOutContent( s, kind, resolved, depth ) ) { return false; } + const int64_t wire = s.bytes - began; + if ( wire > 0xFFFFFFFFll ) { return false; } + TableRetainWrite32( slot + 4, (uint32_t) wire ); + s.bytes += TableLebBytes( (uint64_t) wire ); + return true; + } + const int64_t wire = (int64_t) TableRetainRead32( slot + 4 ); + TableRetainOutLeb( s, (uint64_t) wire ); + const int64_t began = s.bytes; + if ( !TableRetainOutContent( s, kind, resolved, depth ) ) { return false; } + return s.bytes - began == wire; +} + +inline bool TableRetainOutContent( TableRetainOut & s, uint8_t kind, int64_t length, int32_t depth ) +{ + if ( depth > kTableRetainWalkDepthMax ) { return false; } + if ( length < 0 || s.at + length > s.size ) { return false; } + const int64_t end = s.at + length; + switch ( kind ) + { + case 13: + { + for ( ;; ) + { + if ( s.at + 8 > end ) { return false; } + const uint64_t id = TableRetainRead64( s.in + s.at ); + if ( id == 0 ) { s.at += 8; TableRetainOutLeb( s, 0 ); break; } + if ( !TableRetainOutRef( s ) ) { return false; } + if ( s.at >= end ) { return false; } + const uint8_t field_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &field_kind, 1 ); + if ( !TableRetainOutPayload( s, field_kind, depth ) ) { return false; } + } + break; + } + case 14: + { + if ( s.at >= end ) { return false; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainOutLebRead( s, n ) ) { return false; } + TableRetainOutLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( !TableRetainOutPayload( s, elem_kind, depth ) ) { return false; } + } + break; + } + case 16: + { + if ( s.at >= end ) { return false; } + const uint8_t elem_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &elem_kind, 1 ); + uint64_t n = 0; + if ( !TableRetainOutLebRead( s, n ) ) { return false; } + TableRetainOutLeb( s, n ); + for ( uint64_t i = 0; i < n; i++ ) + { + if ( !TableRetainOutRef( s ) ) { return false; } + if ( !TableRetainOutFramed( s, elem_kind, depth + 1 ) ) { return false; } + } + break; + } + case 15: case 30: + // the emit side of the capture's own rule (§6.6): an arm and a + // variant reference resolve as a framed content too + if ( !TableRetainOutPayload( s, kind, depth ) ) { return false; } + break; + default: + TableRetainOutRaw( s, s.in + s.at, length ); + s.at += length; + break; + } + return s.at == end; +} + +// the depth a payload carries is its enclosing body's, exactly as on the +// capture side: only a framed CONTENT is a level. +inline bool TableRetainOutPayload( TableRetainOut & s, uint8_t kind, int32_t depth ) +{ + switch ( kind ) + { + case 1: case 2: case 6: case 20: case 25: + case 3: case 7: case 21: case 26: + case 4: case 8: case 10: case 22: case 27: + case 5: case 9: case 11: case 23: case 28: + case 18: case 19: case 24: case 29: + { + int64_t width = 1; + switch ( kind ) + { + case 3: case 7: case 21: case 26: width = 2; break; + case 4: case 8: case 10: case 22: case 27: width = 4; break; + case 5: case 9: case 11: case 23: case 28: width = 8; break; + case 18: case 19: case 24: case 29: width = 16; break; + default: width = 1; break; + } + if ( s.at + width > s.size ) { return false; } + TableRetainOutRaw( s, s.in + s.at, width ); + s.at += width; + break; + } + case 12: case 31: case 32: case 33: + { + uint64_t length = 0; + if ( !TableRetainOutLebRead( s, length ) ) { return false; } + if ( length > (uint64_t) ( s.size - s.at ) ) { return false; } + TableRetainOutLeb( s, length ); + TableRetainOutRaw( s, s.in + s.at, (int64_t) length ); + s.at += (int64_t) length; + break; + } + case 13: case 14: case 16: + { + if ( !TableRetainOutFramed( s, kind, depth + 1 ) ) { return false; } + break; + } + case 15: + { + if ( s.at + 8 > s.size ) { return false; } + const uint64_t arm = TableRetainRead64( s.in + s.at ); + if ( !TableRetainOutRef( s ) ) { return false; } + if ( arm == 0 ) { break; } + if ( s.at >= s.size ) { return false; } + const uint8_t arm_kind = s.in[ s.at++ ]; + TableRetainOutRaw( s, &arm_kind, 1 ); + if ( !TableRetainOutFramed( s, arm_kind, depth + 1 ) ) { return false; } + break; + } + case 30: + { + if ( !TableRetainOutRef( s ) ) { return false; } + break; + } + default: return false; + } + return true; +} + +// ---- THE RETAINED TAIL: where the records go back (§6.6) ---- +// +// AT THE END OF THEIR OWN BODY, IN THE ORDER RETAINED. Position carries +// nothing on this wire, so appending is chosen for three properties: it is a +// write with no splice, the retained order is preserved, and the result is +// IDEMPOTENT after the first save. +// +// A RETAINED ID PAST THE CAPACITY COUNTS ONE retain_lost AND ITS RECORD IS +// DROPPED, and the save is never refused. MeasureRetain and SaveRetain drop +// the same records under the same walk, so the measure sees the same overflow +// and its answer is the size the save writes. + +// one record's WIRE bytes under the trailer being written, and -1 for a record +// this save cannot place: an id the caller's list had no room for, or a +// resolved form the walk cannot read back. The ids it names are interned on +// the way past, which is what makes measure and save one rule read twice. +// +// THIS IS THE MEASURING PASS, and it leaves every framed content's wire size +// in that content's own scratch slot. The record is the caller's buffer and +// the pass writes nothing else into it. +inline int64_t TableRetainRecordWire( uint8_t * record, TableRetainIds & ids, uint64_t & ref ) +{ + const int32_t mark = ids.count; + ids.lost = false; + ref = ids.record_ref( TableRetainRecordId( record ) ); + if ( !ids.lost && !ids.overflow ) + { + TableRetainOut s; + s.in = TableRetainRecordPayload( record ); + s.size = TableRetainRecordPayloadBytes( record ); + s.at = 0; + s.ids = &ids; + s.w = NULL; + s.bytes = 0; + if ( TableRetainOutPayload( s, TableRetainRecordKind( record ), 0 ) && s.at == s.size ) + { + return TableLebBytes( ref ) + 1 + s.bytes; + } + } + // the record is not written at all, and nothing else about the save + // changes: a full id list degrades to the default behavior one record at a + // time, and the entries this attempt took are given back + ids.truncate( mark ); + ids.lost = false; + return -1; +} + +inline int64_t TableRetainTailMeasure( TableRetain * retain, TableRetainIds & ids, const TableRetainPath & path ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return 0; } + int64_t bytes = 0; + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordHere( *retain, record, path ) ) { continue; } + uint64_t ref = 0; + const int64_t wire = TableRetainRecordWire( record, ids, ref ); + if ( wire < 0 ) { continue; } + bytes += wire; + } + return bytes; +} + +inline bool TableRetainTailSave( TableRetain * retain, TableRetainIds & ids, TableWriter & w, const TableRetainPath & path ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return true; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordHere( *retain, record, path ) ) { continue; } + uint64_t ref = 0; + if ( TableRetainRecordWire( record, ids, ref ) < 0 ) { continue; } + w.putleb( ref ); + w.put8( TableRetainRecordKind( record ) ); + TableRetainOut s; + s.in = TableRetainRecordPayload( record ); + s.size = TableRetainRecordPayloadBytes( record ); + s.at = 0; + s.ids = &ids; + s.w = &w; + s.bytes = 0; + if ( !TableRetainOutPayload( s, TableRetainRecordKind( record ), 0 ) ) { return false; } + record[25] = 1; // PLACED: the one mark the save leaves on the buffer + } + return !w.overflow; +} + +// THE SAVE'S OWN SHARE OF retain_lost, counted ONCE and read after the save +// (§6.6): every record the walk did not place. A record whose path no longer +// names a body, one the caller's id list had no room for, and one the walk +// could not read back are one number here, because the check a caller reads is +// one number. A record is marked as it is written, so this cannot double-count +// a body measured twice. +inline void TableRetainClearPlaced( TableRetain * retain ) +{ + if ( retain == NULL || retain->bytes == NULL ) { return; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + retain->bytes[ at + 25 ] = 0; + at += TableRetainRecordBytes( retain->bytes + at ); + } +} + +inline void TableRetainCountLost( const TableRetain * retain, TableReport * report ) +{ + if ( retain == NULL || retain->bytes == NULL || report == NULL ) { return; } + int64_t at = 0; + for ( int32_t k = 0; k < retain->count; k++ ) + { + const uint8_t * record = retain->bytes + at; + at += TableRetainRecordBytes( record ); + if ( !TableRetainRecordPlaced( record ) ) { report->retain_lost++; } + } +} + +// THE NODE TABLE under retention (§3.1, §6.6): the same fill rule the plain +// pair derives, with the retain family's ids and each record's own body +// reached through a dispatch the CALL supplies rather than a second pair of +// thunks on the numbering. A store per node on the PLAIN save path would be a +// cost this feature is not allowed to have. +template +inline int64_t TableNodeTablePayloadRetain( const Ctx & ctx, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure ) +{ + int64_t payload = TableLebBytes( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + payload += TableLebBytes( ids.ref( n.entries[k].type_id ) ); + const int64_t body = measure( ctx, n, ids, n.entries[k].type_id, n.entries[k].node, retain ); + if ( body < 0 ) { return -1; } + payload += TableLebBytes( (uint64_t) body ) + body; + } + return payload; +} + +template +inline int64_t TableNodeTableMeasureRetain( const Ctx & ctx, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure ) +{ + if ( n.count == 0 ) { return 0; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayloadRetain( ctx, ids, n, retain, measure ); + if ( payload < 0 ) { return -1; } + return TableLebBytes( ref ) + 1 + TableLebBytes( (uint64_t) payload ) + payload; +} + +template +inline bool TableNodeTableSaveRetain( const Ctx & ctx, TableWriter & w, TableRetainIds & ids, const TableNumbering & n, + TableRetain * retain, Measure measure, Save save ) +{ + if ( n.count == 0 ) { return true; } + const uint64_t ref = ids.ref( kTableNodeTableFieldId ); + const int64_t payload = TableNodeTablePayloadRetain( ctx, ids, n, retain, measure ); + if ( payload < 0 ) { return false; } + w.putleb( ref ); + w.put8( 12 ); // kind 12 is the opaque byte payload, exactly as the plain save writes it + w.putleb( (uint64_t) payload ); + w.putleb( (uint64_t) n.count ); + for ( int64_t k = 0; k < n.count; k++ ) + { + w.putleb( ids.ref( n.entries[k].type_id ) ); + const int64_t body = measure( ctx, n, ids, n.entries[k].type_id, n.entries[k].node, retain ); + if ( body < 0 ) { return false; } + w.putleb( (uint64_t) body ); + if ( !save( ctx, n, w, ids, n.entries[k].type_id, n.entries[k].node, retain ) ) { return false; } + } + return true; +} +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_RETAIN + +#ifndef MAPDEMO_SCHEMA_TABLE_EXTENT +#define MAPDEMO_SCHEMA_TABLE_EXTENT + +namespace mapdemo { + +// ---- the NODE EXTENT: where a map's entries and a list's elements live (§2.8, §2.9) ---- + +// TableExtentCarve is a node's extent cursor, PRE-ORDER: a container's whole +// array first, then, element by element in the container's own order, the +// arrays of any list or map an element holds by value. The cursor is the node +// map's, because the generated decoder is threaded with that and not with a +// region. +struct TableExtentCarve +{ + uint8_t * at = NULL; // the region path: the node's extent, unspent + int64_t left = 0; + TableWorker * worker = NULL; // the TOOL's path: the arrays come from the arena +}; + +// AN UNREACHED SLOT MUST HOLD NO LIST OR MAP WITH ELEMENTS IN IT (§2.8, §2.9, +// §7.6). An empty one takes no bytes, so a record whose extent measures ZERO is +// a record whose every by-value list and map is empty. A measure that REFUSED +// answers non-zero here too, and refusing on it is the same answer one level up. +inline bool TableExtentUnreachedEmpty( int64_t extent ) { return extent == 0; } + +// ---- LoadMeasure's framing walk (§6.5) ---- +// +// The measure reads no field value: it walks each record's field headers, +// skipping every payload by its framing, to reach each N at every depth. A +// false is a REFUSAL, and it carries its reason (§6.5). +typedef bool ( * TableWireExtentFn )( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ); + +// the framing walk over an ARRAY OF TABLES held by value: its elements' own +// lists and maps are part of this node's extent too +inline bool TableWireExtentElements( const uint8_t * body, int64_t length, int64_t & at, TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } + if ( r.get8() != 13 ) { return true; } + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +// and over an ENUM-KEYED array, whose triples carry a key REFERENCE before each +// length-prefixed element (docs/SPEC-TABLES.md §3.2) +inline bool TableWireExtentKeyed( const uint8_t * body, int64_t length, int64_t & at, TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } + if ( r.get8() != 13 ) { return true; } + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t key = 0; + if ( !r.getleb( key ) ) { return true; } + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_EXTENT + +#ifndef MAPDEMO_SCHEMA_TABLE_MAP +#define MAPDEMO_SCHEMA_TABLE_MAP + +namespace mapdemo { + +// ---- a MAP: a sorted entry array, and the lookup over it (§2.8) ---- +// +// On the wire, in a region and in a cook a map is an array of one generated +// ENTRY table held in ascending key order. What this adds is Find — a binary +// search over that array where it lies — and a builder that inserts, replaces +// and erases by key. Nothing here is stored: a region and a cook carry the +// array and the count, and not one byte about a hash or a probe. + +// entries carved from ONE call to the allocator pair; a new segment is +// appended when the current one fills, and nothing ever moves (§6.4) +static const int32_t kTableMapSegmentEntries = 32; + +// TableDeclRef names a type in an unevaluated context and is never defined — +// what 's declval is for, without the include the generated corpus +// refuses to pay for (the iterator_traits note, §13.9). +template T & TableDeclRef(); + +// THE ORDER IS TOTAL, AND IT IS THE SAME IN NINE LANGUAGES (§2.8). Integers +// compare by VALUE, signed for the signed kinds and unsigned for the unsigned. +// Strings compare by BYTES, unsigned, a shorter string that is a prefix of a +// longer one first: memcmp over the common length, then the lengths. Never a +// locale, never a code point, never a case fold. +inline int TableKeyOrder( uint64_t a, uint64_t b ) { return a < b ? -1 : ( a > b ? 1 : 0 ); } +inline int TableKeyOrder( int64_t a, int64_t b ) { return a < b ? -1 : ( a > b ? 1 : 0 ); } +inline int TableKeyOrder( const char * a, int32_t a_length, const char * b, int32_t b_length ) +{ + const int32_t common = a_length < b_length ? a_length : b_length; + if ( common > 0 ) + { + const int order = memcmp( (const void *) a, (const void *) b, (size_t) common ); + if ( order != 0 ) { return order < 0 ? -1 : 1; } + } + return a_length < b_length ? -1 : ( a_length > b_length ? 1 : 0 ); +} + +// the length of a NUL-terminated key at a call site, bounded by the storage it +// has to fit: a key one byte longer than the bound is refused, never truncated +inline int32_t TableKeyLength( const char * key, int32_t bound ) +{ + if ( key == NULL ) { return 0; } + for ( int32_t i = 0; i <= bound; i++ ) { if ( key[i] == 0 ) { return i; } } + return bound + 1; // longer than the bound: the caller refuses it +} + +// A KEY IS DATA AND A LENGTH, and the length is CARRIED, never recomputed +// (§2.8, §3). A string(N) key holds any byte a wire or a text can spell, +// U+0000 included, so a lookup that measures to the first NUL answers that "a" +// and "a", 0, "b" are the same key: the first entry is found, RESET, and +// relabeled with the second key, which deletes an entry the report never +// mentions. Every internal lookup and every insertion takes this pair, and the +// public const char * surface builds one and is a wrapper over it. +struct TableMapKeyRef +{ + const char * data; + int32_t length; +}; + +// ---- the storage: SIXTEEN BYTES in the holder's record (§2.8, §7.2) ---- +// +// An int64 self-relative reference to the entry array and an int32 count, then +// padding to eight. The reference is a TableRef like a pointer's: in the arena +// it names the builder's HEAD, in a region it is the delta from the slot to +// the first entry, and 0 is the empty map in both. +template struct TableMap +{ + TableRef entries; + int32_t count = 0; // the LIVE count, in both forms + int32_t padding = 0; // named, so the record has no unwritten byte in it + + // ---- the CONST form: a locked region, a loaded one, an opened cook ---- + // + // One surface over one encoding (§6.3). A region reference resolves from + // the slot's own address, so every one of these is a member and needs no + // base and no context. + const Entry * Entries() const + { + return entries.value != 0 ? (const Entry *) ( (const uint8_t *) &entries + entries.value ) : NULL; + } + int32_t size() const { return count; } + + // FIND: floor( log2 n ) + 1 key compares, in place, no allocation. NULL + // when absent, and on a map[K]*T the RESOLVED pointer, which is what a + // pointer field's accessor answers. + template const Entry * FindEntry( Key key ) const + { + const Entry * base = Entries(); + int32_t low = 0, high = count; + while ( low < high ) + { + const int32_t mid = low + ( high - low ) / 2; + const int order = TableEntryOrder( base[mid], key ); + if ( order == 0 ) { return base + mid; } + if ( order < 0 ) { low = mid + 1; } else { high = mid; } + } + return NULL; + } + // the return type is DEDUCED, so it is worked out when a call site + // instantiates Find and not when the holder's record declares the slot — + // which is what lets the entry's own overloads be declared after it + template auto Find( Key key ) const + { + return TableEntryFound( FindEntry( key ) ); + } + + // ---- iteration: ASCENDING key order, the key beside the value ---- + // + // A proxy BY VALUE, the keyed array's shape (§2.4): for ( auto [ key, + // value ] : map ). It carries no iterator_traits, for the reason + // TableKeyed's does not (§13.9). + struct ConstEntry + { + decltype( TableEntryKey( TableDeclRef() ) ) key; + decltype( TableEntryFound( (const Entry *) NULL ) ) value; + }; + + struct ConstIterator + { + const Entry * at; + ConstEntry operator*() const { return ConstEntry{ TableEntryKey( *at ), TableEntryFound( at ) }; } + ConstIterator & operator++() { at++; return *this; } + bool operator==( const ConstIterator & other ) const { return at == other.at; } + bool operator!=( const ConstIterator & other ) const { return at != other.at; } + }; + + ConstIterator begin() const { return ConstIterator{ Entries() }; } + ConstIterator end() const { return ConstIterator{ Entries() + count }; } +}; + +// ---- the BUILDER's side: a head, and segments that never move (§2.8, §6.4) ---- +// +// The head is a small node in the arena holding the segment chain, the live +// count and the dead count, allocated when the first entry is inserted. Each +// segment is a fixed number of entries carved from one call to the allocator +// pair. An entry's address is stable for the arena's life, so a value handed +// back by an insert stays valid while other entries arrive. +struct TableMapHead +{ + TableRef first; // the arena offset of the first segment + TableRef last; // and of the one an insert appends into + int32_t live; + int32_t dead; +}; + +template struct TableMapSegment +{ + TableRef next; + int32_t used; // entries carved from this segment + int32_t padding; + uint32_t dead[ ( kTableMapSegmentEntries + 31 ) / 32 ]; // Erase marks one bit, never the entry + Entry entries[ kTableMapSegmentEntries ]; +}; + +inline bool TableMapSegmentDead( const uint32_t * dead, int32_t index ) +{ + return ( dead[ index / 32 ] & ( 1u << ( index % 32 ) ) ) != 0; +} + +// ---- the ORDERED CURSOR the four writing walks read (§2.8) ---- +// +// Measure, Save, Lock and Cook each write a map's entries in ascending key +// order with no key twice, deriving the order from the builder's entries as +// each walk derives the numbering (§3.1). Nothing passes between them, so +// measure == save over a map is a real check on two sorts agreeing. +// +// A REGION is already sorted, so its cursor is the array in place and +// allocates nothing. The BUILDER's is the sort: an array of entry pointers +// allocated through the pair and released before the walk returns, because +// sorting the segments themselves would move entries whose addresses a caller +// holds. +template struct TableMapCursor +{ + const Entry * const * order = NULL; // the builder's form: sorted pointers + const Entry * entries = NULL; // the region's form: the array in place + int32_t count = 0; + TableAllocator allocator; + bool ok = false; + const Entry * operator[]( int32_t index ) const + { + return order != NULL ? order[index] : entries + index; + } +}; + +// heapsort: O( n log n ) once per map, no recursion, no allocation past the +// pointer array the caller already paid for +template inline void TableMapSort( const Entry ** order, int32_t count ) +{ + for ( int32_t start = count / 2 - 1; start >= 0; start-- ) + { + int32_t root = start; + for ( ;; ) + { + int32_t child = 2 * root + 1; + if ( child >= count ) { break; } + if ( child + 1 < count && TableEntryOrder( *order[child], *order[child + 1] ) < 0 ) { child++; } + if ( TableEntryOrder( *order[root], *order[child] ) >= 0 ) { break; } + const Entry * swap = order[root]; order[root] = order[child]; order[child] = swap; + root = child; + } + } + for ( int32_t end = count - 1; end > 0; end-- ) + { + const Entry * swap = order[0]; order[0] = order[end]; order[end] = swap; + int32_t root = 0; + for ( ;; ) + { + int32_t child = 2 * root + 1; + if ( child >= end ) { break; } + if ( child + 1 < end && TableEntryOrder( *order[child], *order[child + 1] ) < 0 ) { child++; } + if ( TableEntryOrder( *order[root], *order[child] ) >= 0 ) { break; } + const Entry * hold = order[root]; order[root] = order[child]; order[child] = hold; + root = child; + } + } +} + +// the REGION form: the array is already sorted, so the cursor is the array +template +inline TableMapCursor TableMapOrder( const TableRegionCtx &, const TableMap & map ) +{ + TableMapCursor cursor; + cursor.entries = map.Entries(); + cursor.count = map.count; + cursor.ok = true; + return cursor; +} + +// the BUILDER's form: gather the LIVE entries out of the segment chain in +// insertion order, then sort. A dead entry costs nothing on any wire (§2.8). +template +inline TableMapCursor TableMapOrder( const TableArena & arena, const TableMap & map ) +{ + TableMapCursor cursor; + cursor.allocator = arena.allocator; + cursor.count = map.count; + if ( map.entries.value == 0 || map.count <= 0 ) { cursor.ok = map.count == 0; cursor.count = 0; return cursor; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + if ( head->live != map.count ) { return cursor; } // the slot and the head disagree: refused, never guessed + const Entry ** order = (const Entry **) arena.allocator.alloc( arena.allocator.context, (int64_t) map.count * (int64_t) sizeof( const Entry * ) ); + if ( order == NULL ) { return cursor; } + int32_t at = 0; + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 && at < map.count ) + { + const TableMapSegment * segment = (const TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used && at < map.count; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + order[at++] = segment->entries + i; + } + segment_ref = segment->next; + } + if ( at != map.count ) + { + arena.allocator.free( arena.allocator.context, order ); + return cursor; + } + TableMapSort( order, map.count ); + cursor.order = order; + cursor.ok = true; + return cursor; +} + +template +inline TableMapCursor TableMapOrder( const TableArenaCtx & ctx, const TableMap & map ) +{ + return TableMapOrder( *ctx.arena, map ); +} + +template inline void TableMapRelease( TableMapCursor & cursor ) +{ + if ( cursor.order != NULL ) { cursor.allocator.free( cursor.allocator.context, (void *) cursor.order ); } + cursor.order = NULL; +} + +// ---- the builder's five (§2.8) ---- +// +// Insert APPENDS after one LINEAR SCAN of the live entries for the key it may +// replace, Find is that same scan, and Erase is the scan and one bit. The +// builder builds NO INDEX, and that is a rule: the sort happens once, at Lock, +// Save or Cook, and every lookup that matters runs over the sorted region. + +// the head, allocated when the first entry is inserted +template +inline TableMapHead * TableMapReach( TableWorker & worker, TableMap & map ) +{ + if ( worker.arena == NULL || worker.arena->locked ) { return NULL; } + if ( map.entries.value != 0 ) { return (TableMapHead *) TableArenaAt( *worker.arena, (uint32_t) map.entries.value ); } + uint32_t at = 0; + TableMapHead * head = (TableMapHead *) worker.AllocRaw( (int64_t) sizeof( TableMapHead ), (int64_t) alignof( TableMapHead ), at ); + if ( head == NULL ) { return NULL; } + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + map.entries.value = (int64_t) at; + return head; +} + +// one entry's storage, appended: the current segment when it has room, a new +// one carved from one call to the pair when it does not +template +inline Entry * TableMapAppend( TableWorker & worker, TableMapHead * head, TableMap & map ) +{ + TableMapSegment * segment = NULL; + if ( head->last.value != 0 ) + { + segment = (TableMapSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + if ( segment->used >= kTableMapSegmentEntries ) { segment = NULL; } + } + if ( segment == NULL ) + { + uint32_t at = 0; + segment = (TableMapSegment *) worker.AllocRaw( (int64_t) sizeof( TableMapSegment ), (int64_t) alignof( TableMapSegment ), at ); + if ( segment == NULL ) { return NULL; } // the arena could not carve another segment + segment->next.value = 0; + segment->used = 0; + segment->padding = 0; + for ( int32_t i = 0; i < (int32_t) ( sizeof( segment->dead ) / sizeof( segment->dead[0] ) ); i++ ) { segment->dead[i] = 0; } + if ( head->last.value != 0 ) + { + TableMapSegment * previous = (TableMapSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + previous->next.value = (int64_t) at; + } + else + { + head->first.value = (int64_t) at; + } + head->last.value = (int64_t) at; + } + Entry * entry = segment->entries + segment->used; + segment->used++; + head->live++; + map.count++; + return entry; +} + +// the LINEAR SCAN: the live entries in insertion order, O( n ) key compares +template +inline Entry * TableMapScan( const TableArena & arena, const TableMap & map, Key key ) +{ + if ( map.entries.value == 0 ) { return NULL; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( TableEntryOrder( segment->entries[i], key ) == 0 ) { return segment->entries + i; } + } + segment_ref = segment->next; + } + return NULL; +} + +// ERASE marks the entry DEAD, one bit in the segment's slot and not in the +// entry table, and decrements the live count. Its storage is reclaimed at +// RESET and never reused mid-build, because reusing a slot would make "an +// entry's address is stable" false for exactly one case. +template +inline bool TableMapErase( TableArena & arena, TableMap & map, Key key ) +{ + if ( map.entries.value == 0 ) { return false; } + TableMapHead * head = (TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( TableEntryOrder( segment->entries[i], key ) != 0 ) { continue; } + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + map.count--; + return true; + } + segment_ref = segment->next; + } + return false; +} + +// ---- iterate on the BUILDER: INSERTION order, live entries only (§2.8) ---- +template struct TableMapEach +{ + const TableArena * arena; + TableRef first; + + struct Iterator + { + const TableArena * arena; + TableMapSegment * segment; + int32_t index; + + void Skip() + { + for ( ;; ) + { + if ( segment == NULL ) { return; } + if ( index >= segment->used ) + { + segment = segment->next.value != 0 ? (TableMapSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + index = 0; + continue; + } + if ( TableMapSegmentDead( segment->dead, index ) ) { index++; continue; } + return; + } + } + auto operator*() const { return TableEntryEach( segment->entries + index ); } + Iterator & operator++() { index++; Skip(); return *this; } + bool operator==( const Iterator & other ) const { return segment == other.segment && index == other.index; } + bool operator!=( const Iterator & other ) const { return !( *this == other ); } + }; + + Iterator begin() const + { + Iterator it = { arena, first.value != 0 ? (TableMapSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL, 0 }; + it.Skip(); + return it; + } + Iterator end() const { Iterator it = { arena, NULL, 0 }; return it; } +}; + +template +inline TableMapEach TableMapEachOf( const TableArena & arena, const TableMap & map ) +{ + TableMapEach each = { &arena, TableRef() }; + if ( map.entries.value != 0 ) + { + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + each.first = head->first; + } + return each; +} + +// ---- the LOAD side: where a decoded entry lands (§2.8) ---- +// +// THE READER TRUSTS NOTHING and spends one compare per entry. Every load path +// applies the same rules and produces one report (§4), so the region load of +// §6.5 and LoadBuilder never disagree about a wire. These two shapes are what +// makes that true with one generated decoder: a REGION carves the entry array +// out of the holder node's own extent, and the TOOL's path appends into the +// builder's arena, and the decoder above them cannot tell which it has. + +// The node's extent cursor is TableExtentCarve, the extent runtime's (§2.8, +// §2.9): a map's whole entry array is carved first, then, entry by entry in +// key order, the arrays of any list or map an entry's value holds by value. + +// TableMapFill is one map field being decoded: where the next entry lands, and +// the entry that last LANDED, which is what the ascending check compares +// against. +template struct TableMapFill +{ + TableMap * map = NULL; + Entry * array = NULL; // the region path: the carved array + int32_t capacity = 0; + TableWorker * worker = NULL; // the TOOL's path + bool ok = false; +}; + +template +inline TableMapFill TableMapFillBegin( const TableNodeMap & nodes, TableMap & map, uint32_t n ) +{ + TableMapFill fill; + fill.map = ↦ + map.entries.value = 0; + map.count = 0; + if ( nodes.carve == NULL ) { return fill; } + if ( nodes.carve->worker != NULL ) + { + fill.worker = nodes.carve->worker; // the tool's path: the arena carves + fill.ok = true; + return fill; + } + const int64_t align = (int64_t) alignof( Entry ); + uint8_t * base = (uint8_t *) ( ( (uintptr_t) nodes.carve->at + (uintptr_t) ( align - 1 ) ) & ~( (uintptr_t) ( align - 1 ) ) ); + const int64_t bytes = (int64_t) n * (int64_t) sizeof( Entry ); + const int64_t pad = (int64_t) ( base - nodes.carve->at ); + if ( pad + bytes > nodes.carve->left ) { return fill; } // the measure and the load disagree: refused + nodes.carve->at = base + bytes; + nodes.carve->left -= pad + bytes; + fill.array = (Entry *) base; + fill.capacity = (int32_t) n; + map.entries.value = (int64_t) ( base - (const uint8_t *) &map.entries ); + fill.ok = true; + return fill; +} + +// the entry that last LANDED — NULL before the first +template inline Entry * TableMapFillLast( TableMapFill & fill ) +{ + if ( fill.map->count <= 0 ) { return NULL; } + if ( fill.array != NULL ) { return fill.array + ( fill.map->count - 1 ); } + return TableMapLive( *fill.worker->arena, *fill.map, fill.map->count - 1 ); +} + +// the next slot, at the entry type's declared defaults +template inline Entry * TableMapFillNext( TableMapFill & fill ) +{ + if ( fill.array != NULL ) + { + if ( fill.map->count >= fill.capacity ) { return NULL; } + Entry * entry = fill.array + fill.map->count; + TableReset( *entry ); + fill.map->count++; + return entry; + } + TableMapHead * head = TableMapReach( *fill.worker, *fill.map ); + if ( head == NULL ) { return NULL; } + Entry * entry = TableMapAppend( *fill.worker, head, *fill.map ); + if ( entry != NULL ) { TableReset( *entry ); } + return entry; +} + +// A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): at the first entry whose key +// kind disagrees with the reader's declaration the map resets to EMPTY, one +// kind_mismatch is counted for the map, and its remaining bytes are skipped. +template inline void TableMapFillReset( TableMapFill & fill ) +{ + if ( fill.array != NULL ) + { + fill.map->entries.value = 0; + fill.map->count = 0; + return; + } + if ( fill.map->entries.value != 0 ) + { + TableMapHead * head = (TableMapHead *) TableArenaAt( *fill.worker->arena, (uint32_t) fill.map->entries.value ); + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + } + fill.map->count = 0; +} + +// an EMPTY map's reference is null in both encodings, so a load that placed +// nothing leaves the slot exactly as a Reset does +template inline void TableMapFillEnd( TableMapFill & fill ) +{ + if ( fill.array != NULL && fill.map->count == 0 ) { fill.map->entries.value = 0; } +} + +// the k-th LIVE entry of a builder map, in insertion order — what the tool +// path's ascending check compares against +template +inline Entry * TableMapLive( const TableArena & arena, const TableMap & map, int32_t index ) +{ + if ( map.entries.value == 0 ) { return NULL; } + const TableMapHead * head = (const TableMapHead *) TableArenaAt( arena, (uint32_t) map.entries.value ); + TableRef segment_ref = head->first; + int32_t at = 0; + while ( segment_ref.value != 0 ) + { + TableMapSegment * segment = (TableMapSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + for ( int32_t i = 0; i < segment->used; i++ ) + { + if ( TableMapSegmentDead( segment->dead, i ) ) { continue; } + if ( at == index ) { return segment->entries + i; } + at++; + } + segment_ref = segment->next; + } + return NULL; +} + +// ---- LoadMeasure's term, from the FRAMING alone (§2.8, §6.5) ---- +// +// LoadMeasure's term for a map is N x sizeof( Entry ) rounded to +// alignof( Entry ), AT EVERY DEPTH. N is framing and not a value, so this +// reads no field: it walks the map's own header and, where an entry's value +// holds a map or a list of its own, the entries' headers under it. The caller +// owns the allocation precisely so it can refuse a number it did not expect. +// Every -1 carries its REASON (§6.5): the int32 cap first, because a count +// past it cannot fit any body, and then the body's own L, the one rule a +// list's term answers by. +// A MAP ENTRY'S SMALLEST WIRE FOOTPRINT that commands one storage unit is its +// own L and the body's terminator, and under this form's variable lengths that +// footprint is TWO BYTES (docs/SPEC-TABLES.md §4.2). It is what bounds the N a +// map's L can carry, and therefore what a LoadMeasure may be asked for. +static const int64_t kTableMapEntryFloor = 2; + +inline bool TableMapWireExtent( const uint8_t * body, int64_t length, int64_t & at, + int64_t entry_size, int64_t entry_align, TableWireExtentFn inner, + const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } // no array header: nothing rides + if ( r.get8() != 13 ) { return true; } // not an array of tables: §4's ordinary kind mismatch + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + if ( n > (uint64_t) INT32_MAX ) { reason = count_over_extent_cap; return false; } + const int64_t rest = length - r.offset; + if ( n > (uint64_t) ( rest / kTableMapEntryFloor ) ) { reason = count_over_length; return false; } // an N the map's L cannot carry + at = ( at + entry_align - 1 ) & ~( entry_align - 1 ); + at += (int64_t) n * entry_size; + if ( inner == NULL ) { return true; } // no map below an entry: one depth is the whole term + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } // framing damage: the load reports it + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +// ---- the TEXT form's placement (docs/SPEC-TABLES.md §2.8, §16) ---- +// +// The text is a plain JSON object keyed by the KEY, and the generic walk fills +// it through the ENTRY'S OWN descriptor — so all it needs from here is one +// entry at one key, handed back at its defaults. It is the builder's Insert +// with the ENTRY returned rather than its value, because the walk writes the +// value through a field row and not through a typed pointer. +// +// THE ONE INSERTION PRIMITIVE. Lookup, reset, allocation and the KEY COPY are +// all here, so no caller mutates an entry this did not create and no caller +// relabels one it found. A key is copied only when an entry is created, which +// is what makes a duplicate key leave the identity it matched untouched. NULL +// is one thing and one thing only: the arena refused. +template +inline Entry * TableMapPlace( TableWorker & worker, TableMap & map, Key key ) +{ + if ( worker.arena == NULL ) { return NULL; } + Entry * found = TableMapScan( *worker.arena, map, key ); + if ( found != NULL ) + { + TableResetMapValue( *found ); // a repeated key is LAST-WINS, whole + return found; + } + TableMapHead * head = TableMapReach( worker, map ); + if ( head == NULL ) { return NULL; } + Entry * entry = TableMapAppend( worker, head, map ); + if ( entry == NULL ) { return NULL; } + TableReset( *entry ); + TableEntrySetKey( *entry, key ); + return entry; +} + +// ---- the OPTIONAL RUNTIME INDEX (§2.8) ---- +// +// Open addressing with LINEAR PROBING over the sorted array, built AT LOAD for +// a map large enough that log n compares over a cold array cost more than one +// hash and a probe. IT IS NEVER STORED: the caller measures it, owns its +// storage, builds it in one pass and releases it whenever. +// +// ITS HASH AND ITS LOAD FACTOR ARE NOT A CROSS-PORT CONTRACT, and that is a +// rule. What a port is held to is the CONTRACT of the lookup: the same value +// the sorted array's Find returns for the same key, and no allocation past the +// storage the caller handed in. +struct TableMapIndex +{ + int32_t * slots = NULL; // entry indices, +1; 0 is an empty slot + int32_t capacity = 0; + bool good = false; +}; + +// this runtime's own, and no port reproduces it: fnv1a64 over the key's bytes +inline uint64_t TableMapHash( const void * bytes, int32_t length ) +{ + uint64_t hash = 0xCBF29CE484222325ull; + const uint8_t * at = (const uint8_t *) bytes; + for ( int32_t i = 0; i < length; i++ ) { hash ^= (uint64_t) at[i]; hash *= 0x100000001B3ull; } + return hash; +} +inline uint64_t TableMapHash( uint64_t key ) { return TableMapHash( (const void *) &key, (int32_t) sizeof( key ) ); } + +// this runtime's own load factor, and no port reproduces it either: the next +// power of two at or above twice the count, so a probe run stays short +inline int32_t TableMapIndexSlots( int32_t count ) +{ + int32_t slots = 8; + while ( slots < count * 2 ) { slots *= 2; } + return slots; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_MAP + +#ifndef MAPDEMO_SCHEMA_TABLE_LIST +#define MAPDEMO_SCHEMA_TABLE_LIST + +namespace mapdemo { + +// ---- an UNBOUNDED ARRAY: a counted array whose count the data decides (§2.9) ---- +// +// On the wire, in a region and in a cook a list is the kind 14 body a [..N]T +// writes, its elements by-value records inside the holder's node extent. What +// this adds is the slot, a builder that appends into segments that never +// move, and a const surface that indexes and iterates in place. There is no +// sort, no key and no lookup: the order is INSERTION order, and it is +// identity the way position is identity in a fixed array. + +// elements carved from ONE call to the allocator pair. A new segment is +// appended when the current one fills, and nothing ever moves (§6.4) +static const int32_t kTableListSegmentElements = 32; + +// THE ELEMENT STORAGE: T itself, and a TableRef slot for a []*T, whose +// elements are references exactly as a pointer field's slot is (§2.1) +template struct TableListStorage { typedef T Element; }; +template struct TableListStorage { typedef TableRef Element; }; + +// WHAT THE CONST FORM ANSWERS: the element by reference, and on a []*T the +// RESOLVED pointer, one add on the self-relative delta, NULL for a null slot, +// exactly as At answers it (§6.2, §6.3) +template struct TableListConst +{ + typedef const T & Result; + static Result At( const T * element ) { return *element; } +}; +template struct TableListConst +{ + typedef const T * Result; + static Result At( const TableRef * element ) + { + return element->value != 0 ? (const T *) ( (const uint8_t *) element + element->value ) : NULL; + } +}; + +// ---- the storage: SIXTEEN BYTES in the holder's record (§2.9, §7.2) ---- +// +// An int64 self-relative reference to the element array and an int32 count, +// then padding to eight. The reference is a TableRef like a pointer's: in the +// arena it names the builder's HEAD, in a region it is the delta from the slot +// to the first element, and 0 is the empty list in both. It is the map's slot +// exactly, because it is the same two facts. +template struct TableList +{ + typedef typename TableListStorage::Element Element; + + TableRef elements; + int32_t count = 0; // the LIVE count, in both forms + int32_t padding = 0; // named, so the record has no unwritten byte in it + + // ---- the CONST form: a locked region, a loaded one, an opened cook ---- + // + // One surface over one encoding (§6.3). A region reference resolves from + // the slot's own address, so every one of these is a member and needs no + // base and no context. + const Element * Elements() const + { + return elements.value != 0 ? (const Element *) ( (const uint8_t *) &elements + elements.value ) : NULL; + } + int32_t size() const { return count; } + + // INDEXING IS BOUNDS-CHECKED IN EVERY BUILD (§2.4, §2.9): the extent is a + // number that CAME FROM A FILE, so an index past it is not a mistake a + // release build gets to make cheaply. There is no undefined-behavior path + // here in any configuration. The assert carries the message where a + // debugger can read it and NDEBUG removes that. The fatal is what stands + // after it. Both go through the hooks: define schema_assert and + // schema_fatal and this refusal lands in your own handler. + void RefuseIndex( int32_t index ) const + { + if ( (uint32_t) index >= (uint32_t) count ) + { + schema_assert( false && "an unbounded array is indexed inside its count, which came from a file" ); + schema_fatal(); + } + } + typename TableListConst::Result operator[]( int32_t index ) const + { + RefuseIndex( index ); + return TableListConst::At( Elements() + index ); + } + + // ---- iteration: INDEX order, the element and no key ---- + // + // It carries no iterator_traits, for the reason TableKeyed's does not + // (§13.9). + struct ConstIterator + { + const Element * at; + typename TableListConst::Result operator*() const { return TableListConst::At( at ); } + ConstIterator & operator++() { at++; return *this; } + bool operator==( const ConstIterator & other ) const { return at == other.at; } + bool operator!=( const ConstIterator & other ) const { return at != other.at; } + }; + + ConstIterator begin() const { return ConstIterator{ Elements() }; } + ConstIterator end() const { return ConstIterator{ Elements() + count }; } +}; + +// ---- the BUILDER's side: a head, and segments that never move (§2.9, §6.4) ---- +// +// The head is a small node in the arena holding the segment chain, the live +// count and the dead count, allocated when the first element is added. Each +// segment is a fixed number of elements carved from one call to the allocator +// pair. An element's address is stable for the arena's life, so a T * handed +// back by Add stays valid while other elements arrive. +struct TableListHead +{ + TableRef first; // the arena offset of the first segment + TableRef last; // and of the one an Add appends into + int32_t live; + int32_t dead; +}; + +template struct TableListSegment +{ + TableRef next; + int32_t used; // elements carved from this segment + int32_t padding; + uint32_t dead[ ( kTableListSegmentElements + 31 ) / 32 ]; // Erase marks one bit, never the element + Element elements[ kTableListSegmentElements ]; +}; + +inline bool TableListSegmentDead( const uint32_t * dead, int32_t index ) +{ + return ( dead[ index / 32 ] & ( 1u << ( index % 32 ) ) ) != 0; +} + +// the head, allocated when the first element is added +template +inline TableListHead * TableListReach( TableWorker & worker, TableList & list ) +{ + if ( worker.arena == NULL || worker.arena->locked ) { return NULL; } + if ( list.elements.value != 0 ) { return (TableListHead *) TableArenaAt( *worker.arena, (uint32_t) list.elements.value ); } + uint32_t at = 0; + TableListHead * head = (TableListHead *) worker.AllocRaw( (int64_t) sizeof( TableListHead ), (int64_t) alignof( TableListHead ), at ); + if ( head == NULL ) { return NULL; } + head->first.value = 0; + head->last.value = 0; + head->live = 0; + head->dead = 0; + list.elements.value = (int64_t) at; + return head; +} + +// one element's storage, appended: the current segment when it has room, a +// new one carved from one call to the pair when it does not. NULL means NOT +// ADDED: an arena that cannot carve another segment, or a count at the int32 +// cap (§2.2, §2.9). +template +inline typename TableList::Element * TableListAppend( TableWorker & worker, TableListHead * head, TableList & list ) +{ + typedef typename TableList::Element Element; + if ( list.count >= INT32_MAX ) { return NULL; } // the int32 storage cap + TableListSegment * segment = NULL; + if ( head->last.value != 0 ) + { + segment = (TableListSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + if ( segment->used >= kTableListSegmentElements ) { segment = NULL; } + } + if ( segment == NULL ) + { + uint32_t at = 0; + segment = (TableListSegment *) worker.AllocRaw( (int64_t) sizeof( TableListSegment ), (int64_t) alignof( TableListSegment ), at ); + if ( segment == NULL ) { return NULL; } // the arena could not carve another segment + segment->next.value = 0; + segment->used = 0; + segment->padding = 0; + for ( int32_t i = 0; i < (int32_t) ( sizeof( segment->dead ) / sizeof( segment->dead[0] ) ); i++ ) { segment->dead[i] = 0; } + if ( head->last.value != 0 ) + { + TableListSegment * previous = (TableListSegment *) TableArenaAt( *worker.arena, (uint32_t) head->last.value ); + previous->next.value = (int64_t) at; + } + else + { + head->first.value = (int64_t) at; + } + head->last.value = (int64_t) at; + } + Element * element = segment->elements + segment->used; + segment->used++; + head->live++; + list.count++; + return element; +} + +// ADD, whole: the head, the append, and the element at its declared defaults +// (§2.9). The text form's placement is this same call, because a list has no +// key to place under (§16). +template +inline typename TableList::Element * TableListPlace( TableWorker & worker, TableList & list ) +{ + typedef typename TableList::Element Element; + TableListHead * head = TableListReach( worker, list ); + if ( head == NULL ) { return NULL; } + Element * element = TableListAppend( worker, head, list ); + if ( element == NULL ) { return NULL; } + new ( element ) Element(); // value-init: the declared defaults, and null for a slot + return element; +} + +// ERASE, ADDRESSED BY THE POINTER (§2.9): the element Add handed back is the +// handle, because a list has no key and the address is the one thing the +// builder promises never moves (§6.4). It marks the element DEAD, one bit in +// the segment's slot and not in the element storage, and decrements the live +// count. False when the pointer is not this list's. Its storage is reclaimed +// at RESET and never reused mid-build, the map's rule for the map's reason. +template +inline bool TableListErase( TableArena & arena, TableList & list, const typename TableList::Element * element ) +{ + typedef typename TableList::Element Element; + if ( list.elements.value == 0 || element == NULL ) { return false; } + TableListHead * head = (TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + TableRef segment_ref = head->first; + while ( segment_ref.value != 0 ) + { + TableListSegment * segment = (TableListSegment *) TableArenaAt( arena, (uint32_t) segment_ref.value ); + if ( element >= segment->elements && element < segment->elements + segment->used ) + { + const int32_t i = (int32_t) ( element - segment->elements ); + if ( TableListSegmentDead( segment->dead, i ) ) { return false; } // already erased + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + list.count--; + return true; + } + segment_ref = segment->next; + } + return false; +} + +// ---- iterate on the BUILDER: INDEX order, live elements only (§2.9) ---- +template struct TableListEach +{ + typedef typename TableList::Element Element; + const TableArena * arena; + TableRef first; + + struct Iterator + { + const TableArena * arena; + TableListSegment * segment; + int32_t index; + + void Skip() + { + for ( ;; ) + { + if ( segment == NULL ) { return; } + if ( index >= segment->used ) + { + segment = segment->next.value != 0 ? (TableListSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + index = 0; + continue; + } + if ( TableListSegmentDead( segment->dead, index ) ) { index++; continue; } + return; + } + } + Element * operator*() const { return segment->elements + index; } + Iterator & operator++() { index++; Skip(); return *this; } + bool operator==( const Iterator & other ) const { return segment == other.segment && index == other.index; } + bool operator!=( const Iterator & other ) const { return !( *this == other ); } + }; + + Iterator begin() const + { + Iterator it = { arena, first.value != 0 ? (TableListSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL, 0 }; + it.Skip(); + return it; + } + Iterator end() const { Iterator it = { arena, NULL, 0 }; return it; } +}; + +template +inline TableListEach TableListEachOf( const TableArena & arena, const TableList & list ) +{ + TableListEach each = { &arena, TableRef() }; + if ( list.elements.value != 0 ) + { + const TableListHead * head = (const TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + each.first = head->first; + } + return each; +} + +// ---- the INDEX-ORDER CURSOR the four writing walks read (§2.9) ---- +// +// Measure, Save, Lock and Cook each visit a list's live elements in the order +// they were added, and they allocate nothing to do it: a region's cursor is +// the array in place, and the builder's walks the segment chain. Indexing the +// builder's form is SEQUENTIAL by construction, every walk steps i, i + 1, +// i + 2, so the cursor remembers where the last access landed and moves one +// live slot per step. An access behind the memo restarts from the first +// segment, which no walk here does. +template struct TableListCursor +{ + const Element * elements = NULL; // the region's form: the array in place + const TableArena * arena = NULL; // the builder's form: the segments + TableRef first; + int32_t count = 0; + bool ok = false; + // the memo: the segment and slot the last access landed on, and the live + // index that slot holds + mutable const TableListSegment * segment = NULL; + mutable int32_t within = -1; + mutable int32_t logical = -1; + + const Element * At( int32_t index ) const + { + if ( elements != NULL ) { return elements + index; } + if ( segment == NULL || index < logical ) + { + segment = first.value != 0 ? (const TableListSegment *) TableArenaAt( *arena, (uint32_t) first.value ) : NULL; + within = -1; + logical = -1; + } + while ( logical < index ) + { + for ( ;; ) + { + within++; + while ( segment != NULL && within >= segment->used ) + { + segment = segment->next.value != 0 ? (const TableListSegment *) TableArenaAt( *arena, (uint32_t) segment->next.value ) : NULL; + within = 0; + } + if ( segment == NULL ) { return NULL; } // the slot and the head disagree + if ( !TableListSegmentDead( segment->dead, within ) ) { break; } + } + logical++; + } + return segment->elements + within; + } + const Element & operator[]( int32_t index ) const { return *At( index ); } +}; + +// the REGION form: the array is the cursor +template +inline TableListCursor::Element> TableListElements( const TableRegionCtx &, const TableList & list ) +{ + TableListCursor::Element> cursor; + cursor.elements = list.Elements(); + cursor.count = list.count; + cursor.ok = true; + return cursor; +} + +// the BUILDER's form: the live elements out of the segment chain, in the +// order they were added. A dead element costs nothing on any wire (§2.9). +template +inline TableListCursor::Element> TableListElements( const TableArena & arena, const TableList & list ) +{ + TableListCursor::Element> cursor; + cursor.arena = &arena; + cursor.count = list.count; + if ( list.elements.value == 0 || list.count <= 0 ) { cursor.ok = list.count == 0; cursor.count = 0; return cursor; } + const TableListHead * head = (const TableListHead *) TableArenaAt( arena, (uint32_t) list.elements.value ); + if ( head->live != list.count ) { return cursor; } // the slot and the head disagree: refused, never guessed + cursor.first = head->first; + cursor.ok = true; + return cursor; +} + +template +inline TableListCursor::Element> TableListElements( const TableArenaCtx & ctx, const TableList & list ) +{ + return TableListElements( *ctx.arena, list ); +} + +// ---- the LOAD side: where a decoded element lands (§2.9) ---- +// +// The same two shapes the map's fill takes, because the decoder above them +// cannot tell which it has: a REGION carves the element array out of the +// holder node's own extent, PRE-ORDER, and the TOOL's path appends into the +// builder's arena. +template struct TableListFill +{ + typedef typename TableList::Element Element; + TableList * list = NULL; + Element * array = NULL; // the region path: the carved array + int32_t capacity = 0; + TableWorker * worker = NULL; // the TOOL's path + bool ok = false; + bool refused = false; // a count above the int32 cap on the tool's path: LoadBuilder answers NULL +}; + +template +inline TableListFill TableListFillBegin( const TableNodeMap & nodes, TableList & list, uint64_t n ) +{ + typedef typename TableList::Element Element; + TableListFill fill; + fill.list = &list; + list.elements.value = 0; + list.count = 0; + if ( nodes.carve == NULL ) { return fill; } + if ( n > (uint64_t) INT32_MAX ) + { + // A COUNT ABOVE THE int32 STORAGE CAP (§2.2, §2.9): into a region it was + // refused by LoadMeasure before this ran, and into a builder it is the + // refusal LoadBuilder answers NULL for, moving no counter + fill.refused = nodes.carve->worker != NULL; + return fill; + } + if ( nodes.carve->worker != NULL ) + { + fill.worker = nodes.carve->worker; // the tool's path: the arena carves + fill.ok = true; + return fill; + } + const int64_t align = (int64_t) alignof( Element ); + uint8_t * base = (uint8_t *) ( ( (uintptr_t) nodes.carve->at + (uintptr_t) ( align - 1 ) ) & ~( (uintptr_t) ( align - 1 ) ) ); + const int64_t bytes = (int64_t) n * (int64_t) sizeof( Element ); + const int64_t pad = (int64_t) ( base - nodes.carve->at ); + if ( pad + bytes > nodes.carve->left ) { return fill; } // the measure and the load disagree: refused + nodes.carve->at = base + bytes; + nodes.carve->left -= pad + bytes; + fill.array = (Element *) base; + fill.capacity = (int32_t) n; + list.elements.value = (int64_t) ( base - (const uint8_t *) &list.elements ); + fill.ok = true; + return fill; +} + +// the next slot, at the element's declared defaults. NULL when the arena +// could not carve, which the decoder reports as framing damage +template inline typename TableList::Element * TableListFillNext( TableListFill & fill ) +{ + typedef typename TableList::Element Element; + if ( fill.array != NULL ) + { + if ( fill.list->count >= fill.capacity ) { return NULL; } + Element * element = fill.array + fill.list->count; + new ( element ) Element(); + fill.list->count++; + return element; + } + return TableListPlace( *fill.worker, *fill.list ); +} + +// A SLOT WHOSE ELEMENT NEVER LANDED is given back (§2.9, §4): the array keeps +// what it decoded, and an element whose own framing gave out before one byte +// of it decoded was not decoded. The region's form uncounts it, and the builder's +// marks it dead, which is what the storage rule allows mid-build. +template inline void TableListFillDrop( TableListFill & fill ) +{ + typedef typename TableList::Element Element; + if ( fill.array != NULL ) + { + if ( fill.list->count > 0 ) { fill.list->count--; } + return; + } + if ( fill.list->elements.value == 0 ) { return; } + TableListHead * head = (TableListHead *) TableArenaAt( *fill.worker->arena, (uint32_t) fill.list->elements.value ); + if ( head->last.value == 0 ) { return; } + TableListSegment * segment = (TableListSegment *) TableArenaAt( *fill.worker->arena, (uint32_t) head->last.value ); + if ( segment->used <= 0 ) { return; } + const int32_t i = segment->used - 1; + if ( TableListSegmentDead( segment->dead, i ) ) { return; } + segment->dead[ i / 32 ] |= 1u << ( i % 32 ); + head->live--; + head->dead++; + fill.list->count--; +} + +// an EMPTY list's reference is null in both encodings, so a load that placed +// nothing leaves the slot exactly as a Reset does +template inline void TableListFillEnd( TableListFill & fill ) +{ + if ( fill.array != NULL && fill.list->count == 0 ) { fill.list->elements.value = 0; } +} + +// ---- LoadMeasure's term, from the FRAMING alone (§2.9, §6.5) ---- +// +// N x sizeof( T ) rounded to alignof( T ), AT EVERY DEPTH. N is framing and +// not a value, so this reads no field: it walks the list's own header and, +// where a table element holds a list or a map of its own, the elements' +// headers under it. Every -1 carries its REASON (§6.5): the int32 cap first, +// because a count past it cannot fit any body, and then the body's own L. +inline bool TableListWireExtent( const uint8_t * body, int64_t length, int64_t & at, + int64_t elem_size, int64_t elem_align, uint8_t elem_kind, int64_t elem_floor, + TableWireExtentFn inner, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; + TableReader r( body, length, &scratch, ids ); + if ( length < 2 ) { return true; } // no array header: nothing rides + if ( r.get8() != elem_kind ) { return true; } // another element kind: §4's ordinary kind mismatch, the field reads empty + uint64_t n = 0; + if ( !r.getleb( n ) ) { return true; } + if ( n > (uint64_t) INT32_MAX ) { reason = count_over_extent_cap; return false; } + const int64_t rest = length - r.offset; + if ( n > (uint64_t) ( rest / elem_floor ) ) { reason = count_over_length; return false; } // an N the list's L cannot carry + at = ( at + elem_align - 1 ) & ~( elem_align - 1 ); + at += (int64_t) n * elem_size; + if ( inner == NULL ) { return true; } // nothing below an element: one depth is the whole term + for ( uint64_t i = 0; i < n; i++ ) + { + uint64_t elem = 0; + if ( !r.getleb( elem ) || !r.room( elem ) ) { return true; } // framing damage: the load reports it + if ( !inner( r.buffer + r.offset, (int64_t) elem, at, ids, reason ) ) { return false; } + r.offset += (int64_t) elem; + } + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_LIST + +#ifndef MAPDEMO_SCHEMA_BUILD_VERSION +#define MAPDEMO_SCHEMA_BUILD_VERSION + +namespace mapdemo { + +// THE BUILD VERSION (docs/SPEC-TABLES.md §20): one digest over every fact the bytes +// this build produces depend on — the type wire's protocol id, every record's +// layout as the compiler's own C ABI model computes it, and the facts that +// decide what a load PUTS in those slots. It is the number a cook's header +// carries and the number Open compares, and the number a block's prologue +// carries and BlockOpen compares: a build version answers "which build?" and +// not "which form?", and what separates the two forms is their MAGIC. +// +// There are TWO ids in the design and they are not interchangeable: the +// PROTOCOL ID is the type wire's and nothing else, and the BUILD VERSION is +// what everything cooked or blocked is keyed by. A table edit moves this and +// never the protocol id; a type edit moves both. +static const uint64_t BuildVersion = 0x194c2068d2c970ddull; + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_BUILD_VERSION + +#ifndef MAPDEMO_SCHEMA_TABLE_COOK +#define MAPDEMO_SCHEMA_TABLE_COOK + +namespace mapdemo { + +// ---- the cooked form (docs/SPEC-TABLES.md §7) ---- +// +// A cooked file is a HEADER, a DATA part and an ATTRIBUTION part, in that +// order. Every word of the header is a u64 written in the byte order the cook +// was produced in, and the header is 64 bytes: +// +// 0 magic 0x4b4f4f434d484353, read BYTEWISE before anything else +// 8 build_version the unit's id (docs/SPEC-TABLES.md §20) +// 16 byte_order 1 little, 2 big — the order that WROTE the file +// 24 data_length the region's bytes, rounded up to alignment +// 32 attribution_length the directory's bytes, or 0 +// 40 alignment the region's alignment, never below eight +// 48 reserved zero +// 56 reserved zero +// +// The DATA part is Lock's region written verbatim (§7.2) — the root at its +// base — and it is what a runtime points at. The ATTRIBUTION part is the node +// directory (§6.3), and NOTHING THAT READS THE STRUCTURE TOUCHES IT: it is +// written beside the data for schema cook-check, so a build that ships no +// tooling need not carry it at all. +static const int64_t kTableCookHeaderBytes = 64; + +// THE MAGIC'S VALUE, and a consumer written from the page needs the constant +// rather than a description of one. It is "SCHMCOOK" read as ASCII in the byte +// order a little-endian store produces — the same shape the block form's +// SCHMABLK takes, so a hex dump of a little-endian cook is legible and the two +// accelerators sit in one vocabulary. +// +// IT IS STORED IN THE PRODUCER'S ORDER, which is what makes it the byte-order +// check as well as the form check: a consumer reads back this build's +// constant, or that constant byte-reversed — which identifies a cook of the +// OTHER order — or something that is not a cook. All three answers but the +// first refuse, and a cook and a BLOCK are separated here too, because a +// form's identity belongs in its magic rather than in a second digest. +static const uint64_t TableCookMagic = 0x4b4f4f434d484353ull; + +// THIS BUILD's byte order, as the header's own word carries it. The magic is +// what REFUSES a foreign order; this word is what RECORDS which order wrote +// the file, so a refusal names the order rather than inferring it and a tool +// dumping a cook reads the fact. A file whose magic matched and whose order +// word did not is corrupt, and there is no reading that recovers it. +// +// The BUILD VERSION cannot do either job: §20.1 digests byteorder as a +// GENERATION input, little for every target schema generates for today, so +// two builds of one schema for two orders emit the same id. +#if defined( __BYTE_ORDER__ ) && defined( __ORDER_BIG_ENDIAN__ ) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +static const uint64_t TableCookByteOrder = 2; // big +#else +static const uint64_t TableCookByteOrder = 1; // little +#endif + +// The greatest region alignment a cooked file may name. The DATA part begins +// at align_up( 64, alignment ), which is 64 for every unit this language can +// declare — the largest alignment it has is sixteen — so a word past this cap +// describes a file no build of this schema wrote (docs/SPEC-TABLES.md §7.1). +static const uint64_t TableCookMaxAlign = 64; + +// The header read, BYTEWISE. memcpy is the portable spelling of "these eight +// bytes, in this machine's order"; every compiler this repo builds under folds +// it to one load, and it is the only read in the whole of Open that is not a +// comparison. +inline uint64_t table_cook_read64( const uint8_t * p ) +{ + uint64_t v; + memcpy( &v, p, sizeof( v ) ); + return v; +} + +// TableCookOpen: THE WHOLE CHECK, in one place, because §7 states the +// enumeration once and every generated Open is that one enumeration plus +// its own root's two layout facts. +// +// THE CHECK, in order: the magic read bytewise, the byte order it establishes, +// the build version against this build's own, both RESERVED words zero, the +// region alignment the header names, the two part lengths against the length +// the caller passed — a truncated file and a file with trailing bytes are the +// same refusal — the root's own storage inside the data part, and the +// alignment of the base. +// +// AND THAT IS ALL OF IT. On a match the bytes ARE what this build wrote, in +// this build's layout and this build's byte order, so there is nothing to +// validate and nothing to fix up: the caller gets the root. Nothing per node +// happens here, which is what makes open O(1) in the file's size; a walk of +// any shape would forfeit that, and validating an untrusted file is schema +// cook-check's job and a person's decision (§7.4). +// +// EVERY NUMBER BELOW COMES OUT OF THE FILE, so the arithmetic is unsigned and +// each term is BOUNDED BEFORE IT IS ADDED: a forged length near 2^64 must +// refuse, and an addition that wrapped would be the defect the comparison +// after it was supposed to catch. Nothing past length is read on any path, +// including every refusing one. +// A REFUSAL NAMES ITSELF, beside the null (docs/SPEC-TABLES.md §7): the reason +// is written on the refusal path only, so a match costs nothing and a caller +// that passed no out-parameter pays nothing. +inline const uint8_t * TableCookRefuse( TableRefuseReason * reason, TableRefuseReason why ) +{ + if ( reason != NULL ) { *reason = why; } + return NULL; +} + +inline uint64_t table_cook_byteswap64( uint64_t v ) +{ + return ( v >> 56 ) | ( ( v >> 40 ) & 0xff00ull ) | ( ( v >> 24 ) & 0xff0000ull ) | ( ( v >> 8 ) & 0xff000000ull ) + | ( ( v << 8 ) & 0xff00000000ull ) | ( ( v << 24 ) & 0xff0000000000ull ) | ( ( v << 40 ) & 0xff000000000000ull ) + | ( v << 56 ); +} + +inline const uint8_t * TableCookOpen( const void * bytes, uint64_t length, uint64_t root_size, uint64_t root_align, TableRefuseReason * reason ) +{ + // a null buffer is the CALLER's defect, as an unaligned base is; a buffer + // shorter than the header has no header to read and is truncated + if ( bytes == NULL ) { return TableCookRefuse( reason, unaligned_base ); } + if ( length < (uint64_t) kTableCookHeaderBytes ) { return TableCookRefuse( reason, truncated ); } + const uint8_t * raw = (const uint8_t *) bytes; + // the MAGIC, bytewise and first: it is what establishes the byte order + // every other header word is read in, so nothing else may be read before + // it. A byte-reversed constant is a cook of the other order and refuses + // here, which is why the order never reaches a fix-up pass; anything else + // is not a cook at all, a BLOCK's magic included. + const uint64_t magic = table_cook_read64( raw ); + if ( magic != TableCookMagic ) + { + return TableCookRefuse( reason, magic == table_cook_byteswap64( TableCookMagic ) ? foreign_order : not_a_cook ); + } + // a byte-order word that contradicts its own magic describes no cook in + // EITHER order, so it shares the magic's own value (§7.1) + if ( table_cook_read64( raw + 16 ) != TableCookByteOrder ) { return TableCookRefuse( reason, not_a_cook ); } + if ( table_cook_read64( raw + 8 ) != BuildVersion ) { return TableCookRefuse( reason, wrong_build_version ); } + // the RESERVED words: a non-zero one means a writer used a form this build + // does not understand, and Open refuses rather than ignoring it. + if ( table_cook_read64( raw + 48 ) != 0 ) { return TableCookRefuse( reason, reserved_not_zero ); } + if ( table_cook_read64( raw + 56 ) != 0 ) { return TableCookRefuse( reason, reserved_not_zero ); } + const uint64_t data_length = table_cook_read64( raw + 24 ); + const uint64_t attribution_length = table_cook_read64( raw + 32 ); + const uint64_t alignment = table_cook_read64( raw + 40 ); + // THE ALIGNMENT WORD IS DATA, and it is the one header field the rest of + // the check does arithmetic WITH rather than only comparison against. A + // region's alignment is a power of two, never below eight (the floor that + // puts the attribution part on an eight-byte boundary without a second + // padding rule) and never past the cap above; a word that is none of those + // rounds nothing and aligns nothing, so it is refused before it is used, + // which is why bad_alignment precedes both truncated clauses (§7). + if ( alignment < 8 || alignment > TableCookMaxAlign ) { return TableCookRefuse( reason, bad_alignment ); } + if ( ( alignment & ( alignment - 1 ) ) != 0 ) { return TableCookRefuse( reason, bad_alignment ); } + // and it must be an alignment THE ROOT CAN SIT AT, since the root is at + // the region's base: both are powers of two, so "at least the root's" + // is one division. + if ( ( alignment % root_align ) != 0 ) { return TableCookRefuse( reason, bad_alignment ); } + // The DATA part begins at align_up( 64, alignment ). It is DERIVED and not + // a header field, because a fact a reader computes is a fact two writers + // cannot disagree about. + const uint64_t data_offset = ( (uint64_t) kTableCookHeaderBytes + alignment - 1 ) & ~( alignment - 1 ); + if ( length < data_offset ) { return TableCookRefuse( reason, truncated ); } + // the two part lengths against the length the caller passed. The whole + // file is data_offset + data_length + attribution_length, and a length + // that is not EXACTLY that refuses — truncation and trailing bytes are one + // refusal, and both terms are subtracted rather than added so no sum can + // carry. + if ( data_length > length - data_offset ) { return TableCookRefuse( reason, truncated ); } + if ( attribution_length != length - data_offset - data_length ) { return TableCookRefuse( reason, truncated ); } + // the ROOT sits at the region's base, so the region has to hold it: a + // shorter data part describes a root partly outside the file, which is the + // one way a match-and-point reader could hand back storage it never + // received. It is the second clause on truncated (§7). + if ( data_length < root_size ) { return TableCookRefuse( reason, truncated ); } + const uint8_t * base = raw + data_offset; + // the alignment of the BASE, LAST, because it is the only clause that reads + // nothing out of the file. The header pads the data part to the region's + // alignment, so a base an allocator or mmap gave you is already aligned — + // mmap gives page alignment for free — and a base that is not is a caller's + // buffer this form cannot be read out of: the caller's defect, not the file's. + if ( ( (uintptr_t) base % (uintptr_t) alignment ) != 0 ) { return TableCookRefuse( reason, unaligned_base ); } + return base; +} + +// ---- the cooked form, the WRITE side (docs/SPEC-TABLES.md §7.6) ---- +// +// THE BYTE ORDER IS THE TARGET'S, NOT THE HOST'S. A cook is produced in the +// byte order of the build that will read it (§7), so the fixing happens here — +// offline, once, on the writing side — and never at Open. Passing +// TableByteOrder::Big on a little-endian machine produces a big-endian build's +// file, and nothing about the writing host reaches the bytes. +enum class TableByteOrder +{ + Little = 1, // the header's byte_order word, and the order every scalar is written in + Big = 2, +}; + +// One store, width as an argument. Every call site passes a literal width, so +// the loop folds to a store (and a byte swap on the foreign order); a name per +// width would claim four §11 names to save nothing. +inline void table_cook_put( uint8_t * at, uint64_t value, int32_t width, TableByteOrder order ) +{ + if ( order == TableByteOrder::Little ) + { + for ( int32_t i = 0; i < width; i++ ) { at[i] = (uint8_t) ( value >> ( 8 * i ) ); } + } + else + { + for ( int32_t i = 0; i < width; i++ ) { at[i] = (uint8_t) ( value >> ( 8 * ( width - 1 - i ) ) ); } + } +} + +// A 128-bit store as two lanes: sixteen bytes, the low lane first in the +// little order and the high lane first — each lane big-endian — in the big +// order, exactly as a u64 is one lane of eight (docs/SPEC-TABLES.md §7.2). +inline void table_cook_put128( uint8_t * at, uint64_t lo, uint64_t hi, TableByteOrder order ) +{ + if ( order == TableByteOrder::Little ) { table_cook_put( at, lo, 8, order ); table_cook_put( at + 8, hi, 8, order ); } + else { table_cook_put( at, hi, 8, order ); table_cook_put( at + 8, lo, 8, order ); } +} + +// A buffer piece: the USED bytes and nothing else. The tail is already zero — +// the whole extent was zeroed before any field was written — so this copies the +// used prefix and leaves the rest, which is what makes a string's unused tail a +// consequence of one memset rather than a rule per buffer. A used length past +// the buffer, or below zero, is a value no reader could have produced and it is +// clamped rather than trusted: this writes inside the caller's buffer on every +// input. +inline void table_cook_bytes( uint8_t * at, const void * source, int64_t used, int64_t capacity ) +{ + if ( used <= 0 ) { return; } + const int64_t n = used < capacity ? used : capacity; + memcpy( at, source, (size_t) n ); +} + +// A WIDE TEXT buffer piece (docs/SPEC-TABLES.md §7.2): the USED code units, +// each a TWO-BYTE SCALAR in the cook's byte order. A record is written piece +// by piece and never memcpy'd, and a swap has to know where every scalar +// begins — a char16_t is one, so the units go one store each rather than as +// bytes. The tail is already zero, as the narrow twin's is, so the terminating +// zero unit at index used costs nothing here. +inline void table_cook_units( uint8_t * at, const char16_t * source, int64_t used, int64_t capacity, TableByteOrder order ) +{ + if ( used <= 0 ) { return; } + const int64_t n = used < capacity ? used : capacity; + for ( int64_t i = 0; i < n; i++ ) { table_cook_put( at + i * 2, (uint64_t) (uint16_t) source[i], 2, order ); } +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_COOK + +#ifndef MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE +#define MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE + +namespace mapdemo { + +// ---- the cooked form's WRITE side for a POINTERED root (docs/SPEC-TABLES.md §7.6) ---- +// +// A pointered root's cook is the region of §7.2: every node the numbering +// reached (§3.1), once, at its own type's alignment, in index order, the root +// at offset zero. This is that region while it is being laid out and written — +// the tool's own Layout and Write, in one struct. +// +// The OFFSETS are one per node, the root's zero at position 0 and node index k +// at position k - 1, which is the directory's own order (§6.3); they are the +// one allocation the write makes beyond the numbering, and they go through the +// same pair. A measure needs no offsets and leaves the pointer NULL. +struct TableCookRegion +{ + const TableNumbering * numbering = NULL; // node -> index, from the walk that placed it + int64_t * offsets = NULL; // index - 1 -> the node's region offset; NULL while measuring + int64_t count = 0; // nodes, the root included + int64_t bytes = 0; // the data part's length, rounded to align + int64_t align = 0; // the region's alignment: the nodes' greatest, never below eight + uint8_t * base = NULL; // where the data part is being written; NULL while measuring +}; + +// A reference slot: the SELF-RELATIVE delta from the slot's own address to the +// node's start (§6.3), and zero for null. The node is found by the address the +// numbering keyed it under, which is the same address the walk resolved through +// the same context — so a reference the numbering does not carry is a slot the +// walk never reached (a counted array's slot past its count, an absent +// optional's value) holding a node the region will not hold, and it is refused +// rather than written as a delta to nowhere. +inline bool table_cook_ref( const TableCookRegion & region, uint8_t * at, const void * pointee, TableByteOrder order ) +{ + if ( pointee == NULL ) { table_cook_put( at, 0, 8, order ); return true; } + uint64_t index = 0; + if ( !TableNumberingIndex( *region.numbering, pointee, index ) ) { return false; } + if ( index == 0 || index > (uint64_t) region.count ) { return false; } + const int64_t delta = region.offsets[index - 1] - (int64_t) ( at - region.base ); + table_cook_put( at, (uint64_t) delta, 8, order ); + return true; +} + +} // namespace mapdemo + +#endif // MAPDEMO_SCHEMA_TABLE_COOK_VARIABLE + +namespace mapdemo { + +// table TrailsStepsEntry — TABLE-wire storage: relocatable, bounded, defaults in the +// member initializers (docs/SPEC-TABLES.md) +struct TrailsStepsEntry { + uint32_t key = 0; + TableList value; // Item: the element array, empty until an Add +}; + +// table Trails — TABLE-wire storage: relocatable, bounded, defaults in the +// member initializers (docs/SPEC-TABLES.md) +struct Trails { + TableMap steps; // map[uint32]*Item — the sorted entry array, empty until an insert + int32_t after = 0; +}; + +// ---- prefill: the declared defaults, in place (docs/SPEC-TABLES.md) ---- + +inline void TrailsStepsEntryReset( TrailsStepsEntry & value ); +inline void TrailsReset( Trails & value ); + +inline void TrailsStepsEntryReset( TrailsStepsEntry & value ) +{ + value.key = 0; + value.value.elements.value = 0; // Item: empty + value.value.count = 0; + value.value.padding = 0; +} + +inline void TrailsReset( Trails & value ) +{ + value.steps.entries.value = 0; // map[uint32]*Item: empty + value.steps.count = 0; + value.steps.padding = 0; + value.after = 0; +} + +template inline int64_t TrailsStepsEntryMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const TrailsStepsEntry & value ); +template inline bool TrailsStepsEntrySaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const TrailsStepsEntry & value ); +inline bool TrailsStepsEntryLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, TrailsStepsEntry & value ); +inline bool TrailsStepsEntryMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ); +template inline int64_t TrailsMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Trails & value ); +template inline bool TrailsSaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Trails & value ); +inline bool TrailsLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Trails & value ); +inline bool TrailsMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ); + +// TrailsStepsEntryMessageKeyRead: the key of one entry on the message wire, before the +// slot is chosen (docs/SPEC-TABLES.md §2.8, §3.3), and the bit the entry's +// body ends at. Field order inside a body is not contractual, so this scans +// the whole body by its announced shapes rather than assuming a position. +struct TrailsStepsEntryMessageKeyRead +{ + uint32_t key; + int64_t end; // the bit after the entry's own zero reference + bool found; // the body carried the key's id + bool kind_bad; // it carried it under another kind: the MAP's event + bool widened; // under a kind the declaration WIDENS (§4): decoded exactly, the MAP counts one + bool over; // longer than this reader's bound: the ENTRY is dropped + bool malformed; // the entry's framing gave out +}; + +inline TrailsStepsEntryMessageKeyRead TrailsStepsEntryMessageReadKey( TableBitReader r, const TableVocabulary & vocabulary, int64_t index_bits ) +{ + TrailsStepsEntryMessageKeyRead out = { 0, 0, false, false, false, false, false }; + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { out.malformed = true; return out; } + if ( ref == 0 ) { out.end = r.offset; return out; } // the terminator: no key field is the key's DEFAULT + if ( ref > (uint64_t) vocabulary.count ) { out.malformed = true; return out; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { out.malformed = true; return out; } + if ( entry.id == 0x3dc94a19365b10ecull ) // `key`, the ordinary hash of an ordinary name + { + const bool kind_bad = entry.kind != 8 && !TableKindWidens( entry.kind, 8 ); // THE KEY KIND IS THE READER'S DECLARATION + out.kind_bad = kind_bad; + out.found = !kind_bad; + out.widened = entry.kind != 8; + if ( kind_bad ) + { + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { out.malformed = true; return out; } + continue; + } + { + uint64_t raw = 0; + const int64_t width = entry.value_bits; + if ( width < 0 || !r.get( raw, width ) ) { out.malformed = true; return out; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + out.key = (uint32_t) decoded_wide; + } + continue; // the LAST occurrence is the one §3 keeps + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { out.malformed = true; return out; } + } +} + +// ---- the arena's reset hook (docs/SPEC-TABLES.md §6) ---- +// +// TableWorker::Alloc is a template and cannot name a member's Reset, so +// the arena reaches it through this overload set by argument-dependent +// lookup. It is how a node born in raw arena storage comes to hold the +// declared defaults without value-initialising the whole aggregate. + +inline void TableReset( TrailsStepsEntry & value ) { TrailsStepsEntryReset( value ); } +inline void TableReset( Trails & value ) { TrailsReset( value ); } + +// ---- pointer targets: allocation and resolution (docs/SPEC-TABLES.md §2) ---- +// +// A reference resolves differently in the two forms, and the CONTEXT says +// which: in the arena it is an offset; in a region it is a self-relative +// delta, so the const deref below is one add and needs no base pointer. + +// ---- codecs: measure/save/load per closure member ---- + +template inline int64_t TrailsStepsEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const TrailsStepsEntry & value ); +template inline bool TrailsStepsEntrySaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const TrailsStepsEntry & value ); +template inline bool TrailsStepsEntrySaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const TrailsStepsEntry & value ); +inline bool TrailsStepsEntryLoadBody( TableReader & r, const TableNodeMap & nodes, TrailsStepsEntry & value ); +template inline int64_t TrailsMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Trails & value ); +template inline bool TrailsSaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Trails & value ); +template inline bool TrailsSaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Trails & value ); +inline bool TrailsLoadBody( TableReader & r, const TableNodeMap & nodes, Trails & value ); + +// ---- pointer-graph walkers: number (measure/save), pack (Lock) ---- + +template inline bool TrailsStepsEntryNumber( const Ctx & ctx, TableNumbering & numbering, const TrailsStepsEntry & value ); +template inline int64_t TrailsStepsEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const TrailsStepsEntry & value ); +template inline bool TrailsStepsEntryPack( const Ctx & ctx, TablePackMap & seen, const TrailsStepsEntry & src, TrailsStepsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); +template inline bool TrailsNumber( const Ctx & ctx, TableNumbering & numbering, const Trails & value ); +template inline int64_t TrailsPackMeasure( const Ctx & ctx, TablePackMap & seen, const Trails & value ); +template inline bool TrailsPack( const Ctx & ctx, TablePackMap & seen, const Trails & src, Trails & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +// ---- the numbering's bridge to each member's codec (docs/SPEC-TABLES.md §3.1) ---- + +template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const TrailsStepsEntry & value ) { return TrailsStepsEntryMeasureBody( ctx, numbering, ids, value ); } +template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const TrailsStepsEntry & value ) { return TrailsStepsEntrySaveBody( ctx, numbering, w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const TrailsStepsEntry & value ) { return TrailsStepsEntryMeasureMessageBody( ctx, numbering, index_bits, at, value ); } +template inline bool TableNodeMessageSave( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const TrailsStepsEntry & value ) { return TrailsStepsEntrySaveMessageBody( ctx, numbering, index_bits, w, value ); } +template inline int64_t TableNodeMeasure( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Trails & value ) { return TrailsMeasureBody( ctx, numbering, ids, value ); } +template inline bool TableNodeSave( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Trails & value ) { return TrailsSaveBody( ctx, numbering, w, ids, value ); } +template inline int64_t TableNodeMessageMeasure( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Trails & value ) { return TrailsMeasureMessageBody( ctx, numbering, index_bits, at, value ); } +template inline bool TableNodeMessageSave( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Trails & value ) { return TrailsSaveMessageBody( ctx, numbering, index_bits, w, value ); } + +// ---- TrailsStepsEntry: the order, the key and the value (docs/SPEC-TABLES.md §2.8) ---- +// +// The four overloads the map runtime's templates reach by argument-dependent +// lookup. Nothing outside this file names them. +static_assert( alignof( TrailsStepsEntry ) <= kTableAlign, "a map entry's alignment must fit the arena's" ); + +inline int TableEntryOrder( const TrailsStepsEntry & a, const TrailsStepsEntry & b ) +{ + return TableKeyOrder( (uint64_t) a.key, (uint64_t) b.key ); // integers compare by VALUE, unsigned here +} +inline int TableEntryOrder( const TrailsStepsEntry & entry, uint32_t key ) +{ + return TableKeyOrder( (uint64_t) entry.key, (uint64_t) key ); +} +inline uint32_t TableEntryKey( const TrailsStepsEntry & entry ) { return entry.key; } +inline void TableEntrySetKey( TrailsStepsEntry & entry, uint32_t key ) { entry.key = key; } +inline const TableList * TableEntryFound( const TrailsStepsEntry * entry ) { return entry != NULL ? &entry->value : NULL; } +inline TableList * TableEntryValue( TrailsStepsEntry * entry ) { return &entry->value; } +struct TrailsStepsEntryEach { uint32_t key; decltype( TableEntryValue( (TrailsStepsEntry *) NULL ) ) value; }; +inline TrailsStepsEntryEach TableEntryEach( TrailsStepsEntry * entry ) { return TrailsStepsEntryEach{ TableEntryKey( *entry ), TableEntryValue( entry ) }; } +inline void TableResetMapValue( TrailsStepsEntry & value ) +{ + value.value.elements.value = 0; // Item: empty + value.value.count = 0; + value.value.padding = 0; +} + +// TrailsStepsEntryReadKey: the key, before the slot is chosen (docs/SPEC-TABLES.md §2.8). +// Field order inside a body is not contractual (§3), so this scans rather +// than assuming a position — and this implementation writes the key first, +// so on any wire it wrote the scan ends at the first header. +struct TrailsStepsEntryKeyRead +{ + uint32_t key; + bool found; // the body carried the key's id + bool kind_bad; // it carried it under another kind: the MAP's event + bool widened; // under a kind the declaration WIDENS (§4): decoded exactly, the MAP counts one + bool over; // longer than this reader's bound: the ENTRY is dropped + bool malformed; // the entry's framing gave out +}; + +inline TrailsStepsEntryKeyRead TrailsStepsEntryReadKey( const uint8_t * body, int64_t length, const TableIdTable * ids ) +{ + TrailsStepsEntryKeyRead out = { 0, false, false, false, false, false }; + TableReport scratch; // the scan's own framing damage is the MAP's, raised by the caller + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { out.malformed = true; return out; } + if ( field_ref == 0 ) { return out; } // the terminator: no key field is the key's DEFAULT + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { out.malformed = true; return out; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { out.malformed = true; return out; } + uint8_t field_kind = r.get8(); + if ( field_id == 0x3dc94a19365b10ecull ) // `key`, the ordinary hash of an ordinary name + { + if ( field_kind != 8 && TableKindWidens( field_kind, 8 ) ) + { + out.widened = true; + out.found = true; + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, field_kind, widened_v ) ) { out.malformed = true; return out; } + out.key = (uint32_t) widened_v; + continue; // the LAST occurrence is the one §3 keeps + } + out.kind_bad = field_kind != 8; // THE KEY KIND IS THE READER'S DECLARATION + out.found = !out.kind_bad; + if ( !out.kind_bad ) + { + if ( !r.has( 4 ) ) { out.malformed = true; return out; } + out.key = (uint32_t) r.get32(); + continue; // the LAST occurrence is the one §3 keeps + } + } + if ( !r.skip( field_kind ) ) { out.malformed = true; return out; } + } +} + +// ---- retain-unknown: the second family (docs/SPEC-TABLES.md §6.6) ---- +// +// The same walks, with the PATH threaded and the unknown arm capturing. The +// three above are untouched and cost nothing for these being here: a caller +// that does not ask instantiates none of them. + +template inline int64_t TrailsStepsEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool TrailsStepsEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool TrailsStepsEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TrailsStepsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ); +template inline int64_t TrailsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool TrailsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ); +template inline bool TrailsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ); +inline bool TrailsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Trails & value, TableRetain * retain, const TableRetainPath & path ); + +template +inline int64_t TrailsStepsEntryMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const TrailsStepsEntry & value ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + { + // value: a kind 14 array of kind 17 elements, INDEX order (§2.9) + TableListCursor cursor_value = TableListElements( ctx, value.value ); + if ( !cursor_value.ok ) { return -1; } // the slot and the head disagree + if ( cursor_value.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( cursor_value.count ) ); // the element kind byte and the count + for ( int32_t elem_i_value = 0; elem_i_value < cursor_value.count; elem_i_value++ ) + { + { + const Item * slot_pointee_value = ItemAt( ctx, cursor_value[elem_i_value] ); + uint64_t slot_index_value = 0; + if ( slot_pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee_value, slot_index_value ) ) { return -1; } + body_value += TableLebBytes( slot_index_value ); + } + } + bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); + } + } + return bytes; +} + +template +inline bool TrailsStepsEntrySaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const TrailsStepsEntry & value ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 8 ); // key + w.put32( uint32_t( value.key ) ); + } + { + TableListCursor cursor_value = TableListElements( ctx, value.value ); // value + if ( !cursor_value.ok ) { return false; } + if ( cursor_value.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( cursor_value.count ) ); // the element kind byte and the count + for ( int32_t elem_i_value = 0; elem_i_value < cursor_value.count; elem_i_value++ ) + { + { + const Item * slot_pointee_value = ItemAt( ctx, cursor_value[elem_i_value] ); + uint64_t slot_index_value = 0; + if ( slot_pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee_value, slot_index_value ) ) { return false; } + body_value += TableLebBytes( slot_index_value ); + } + } + w.putleb( ref_value ); w.put8( 14 ); w.putleb( (uint64_t) body_value ); // value + w.put8( 17 ); w.putleb( (uint64_t) ( cursor_value.count ) ); + for ( int32_t elem_i_value = 0; elem_i_value < cursor_value.count; elem_i_value++ ) + { + { + const Item * slot_pointee_value = ItemAt( ctx, cursor_value[elem_i_value] ); + uint64_t slot_index_value = 0; + if ( slot_pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee_value, slot_index_value ) ) { return false; } + w.putleb( slot_index_value ); + } + } + } + } + return !w.overflow; +} + +template +inline bool TrailsStepsEntrySaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const TrailsStepsEntry & value ) +{ + if ( !TrailsStepsEntrySaveBodyFields( ctx, numbering, w, ids, value ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool TrailsStepsEntryLoadBody( TableReader & r, const TableNodeMap & nodes, TrailsStepsEntry & value ) +{ + TrailsStepsEntryReset( value ); // prefill declared defaults in place, then overlay + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x3dc94a19365b10ecull: // key + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.key = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.key = decoded_v; + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER is INERT (§4): the field keeps + // the value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + if ( !counted_ok ) { r.report->malformed = true; } + // AN ELEMENT KIND THAT DISAGREES with the reader's declaration is §3's + // element-kind rule: the field reads EMPTY and one kind_mismatch counts + else if ( elem_kind != 17 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + // THE COUNT IS THE DATA'S (§2.9): there is no bound, so clamped + // cannot fire on it. A count above the int32 storage cap is the + // fill's refusal, and it moves no counter. + TableListFill fill = TableListFillBegin( nodes, value.value, count ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + // elements are BOUNDED by the field body: a count the length cannot + // cover keeps the decoded prefix, flags malformed, and the parent + // continues at the next field + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + for ( uint64_t i = 0; i < count; i++ ) + { + TableRef * slot = TableListFillNext( fill ); + if ( slot == NULL ) { r.report->malformed = true; break; } // the arena could not carve + bool landed = false; + do + { + { + uint64_t node_index_value = 0; + if ( !sub.getleb( node_index_value ) ) { r.report->malformed = true; break; } + TableNodeResolve( nodes, ( *slot ), node_index_value, 0x52cfa1d198476806ull, r.report ); // *Item + } + landed = true; + } while ( 0 ); + if ( !landed ) { TableListFillDrop( fill ); break; } // the element's own framing gave out before it decoded + } + TableListFillEnd( fill ); + } + } + r.offset = body_end; // excess bytes and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's cost, in BITS (docs/SPEC-TABLES.md §3.3). `at` is the +// body's own bit position in the batch, because a `string(N)` ALIGNS before +// its bytes and an align costs what the position says it costs. +template +inline int64_t TrailsStepsEntryMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const TrailsStepsEntry & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + int64_t bits = 0; + if ( value.key != 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + } + { + TableListCursor cursor_value = TableListElements( ctx, value.value ); // value + if ( !cursor_value.ok ) { return -1; } // the slot and the head disagree + if ( cursor_value.count > 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + bits += (int64_t) ( cursor_value.count ) * index_bits; + } + } + bits += kTableMessageRefBitsHere; // the ZERO REFERENCE that ends the body + (void) at; + return bits; +} + +// The BITPACKED body: the fields, then the ZERO REFERENCE that ends it. No +// kind byte rides at all, and no length frames a nested body, because a +// body is self-delimiting: it is written where the file form put an L. +template +inline bool TrailsStepsEntrySaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const TrailsStepsEntry & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + if ( value.key != 0 ) + { + w.put( 9, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.key ), 32 ); + } + { + TableListCursor cursor_value = TableListElements( ctx, value.value ); // value + if ( !cursor_value.ok ) { return false; } // the slot and the head disagree + if ( cursor_value.count > 0 ) + { + w.put( 49, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( cursor_value.count ) - 0, 32 ); + for ( int32_t i = 0; i < cursor_value.count; i++ ) + { + const Item * pointee_value = ItemAt( ctx, cursor_value[i] ); // *Item + uint64_t index_value = 0; + if ( pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) pointee_value, index_value ) ) { return false; } + w.put( index_value, index_bits ); + } + } + } + w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +// TrailsStepsEntryMessageExtent: the extent TrailsStepsEntry's maps command on the message wire, from +// the FRAMING alone (docs/SPEC-TABLES.md §2.8, §3.3, §6.5). +inline bool TrailsStepsEntryMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { return false; } + if ( entry.id == 0x7ce4fd9430e80ceaull && entry.kind == 14 && entry.elem_kind == 17 ) // value: an unbounded array + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { return false; } + n += (uint64_t) entry.min; + if ( n > (uint64_t) INT32_MAX ) { return false; } // above the int32 storage cap (§2.9) + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( TableRef ) + at += (int64_t) n * (int64_t) sizeof( TableRef ); // the whole array FIRST + const int64_t run = TableMessageElementRunBits( vocabulary, entry ); + if ( run >= 0 ) { if ( !TableMessageSkipRun( r, n, run ) ) { return false; } } + else { for ( uint64_t i = 0; i < n; i++ ) { if ( !TableMessageSkipElement( r, vocabulary, index_bits, entry ) ) { return false; } } } + continue; + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { return false; } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +inline bool TrailsStepsEntryLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, TrailsStepsEntry & value ) +{ + (void) nodes; (void) index_bits; + TrailsStepsEntryReset( value ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x3dc94a19365b10ecull: // key + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 8 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 8 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + if ( (uint64_t) decoded_wide > 4294967295ull ) { decoded_wide = (int64_t) 4294967295ull; report->clamped++; } + uint32_t decoded_v = (uint32_t) decoded_wide; + value.key = decoded_v; + } + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 17 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + n += (uint64_t) entry.min; + TableListFill fill = TableListFillBegin( nodes, value.value, n ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + for ( uint64_t i = 0; i < n; i++ ) + { + TableRef * slot = TableListFillNext( fill ); + if ( slot == NULL ) { report->malformed = true; return false; } // the arena could not carve + { + uint64_t node_index_2 = 0; + if ( !r.get( node_index_2, index_bits ) ) { report->malformed = true; return false; } + TableNodeResolve( nodes, ( *slot ), node_index_2, 0x52cfa1d198476806ull, report ); // *Item + } + } + TableListFillEnd( fill ); + } + break; + } + default: + report->unknown++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + } +} + +template +inline int64_t TrailsMeasureBody( const Ctx & ctx, const TableNumbering & numbering, TableIds & ids, const Trails & value ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // steps: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); + if ( !order_steps.ok ) { return -1; } // the sort could not run + if ( order_steps.count > 0 ) + { + const uint64_t ref_steps = ids.ref( 0x124250ad5a5b6d14ull ); + int64_t body_steps = 1 + TableLebBytes( (uint64_t) order_steps.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_steps = TrailsStepsEntryMeasureBody( ctx, numbering, ids, *order_steps[i] ); + if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return -1; } + body_steps += TableLebBytes( (uint64_t) ( elem_steps ) ) + ( elem_steps ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_steps ) + 1 + TableLebBytes( (uint64_t) ( body_steps ) ) + ( body_steps ); + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + return bytes; +} + +template +inline bool TrailsSaveBodyFields( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Trails & value ) +{ + { + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); // steps + if ( !order_steps.ok ) { return false; } + if ( order_steps.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_steps = ids.ref( 0x124250ad5a5b6d14ull ); + int64_t body_steps = 1 + TableLebBytes( (uint64_t) order_steps.count ); + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_steps = TrailsStepsEntryMeasureBody( ctx, numbering, ids, *order_steps[i] ); + if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return false; } + body_steps += TableLebBytes( (uint64_t) ( elem_steps ) ) + ( elem_steps ); + } + w.putleb( ref_steps ); w.put8( 14 ); w.putleb( (uint64_t) body_steps ); + w.put8( 13 ); w.putleb( (uint64_t) order_steps.count ); + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_len_steps = TrailsStepsEntryMeasureBody( ctx, numbering, ids, *order_steps[i] ); + if ( elem_len_steps < 0 ) { TableMapRelease( order_steps ); return false; } + w.putleb( (uint64_t) elem_len_steps ); + if ( !TrailsStepsEntrySaveBody( ctx, numbering, w, ids, *order_steps[i] ) ) { TableMapRelease( order_steps ); return false; } + } + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + return !w.overflow; +} + +template +inline bool TrailsSaveBody( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableIds & ids, const Trails & value ) +{ + if ( !TrailsSaveBodyFields( ctx, numbering, w, ids, value ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool TrailsLoadBody( TableReader & r, const TableNodeMap & nodes, Trails & value ) +{ + TrailsReset( value ); // prefill declared defaults in place, then overlay + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x124250ad5a5b6d14ull: // steps + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + TableMapFill fill = TableMapFillBegin( nodes, value.steps, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + TrailsStepsEntryKeyRead read = TrailsStepsEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + TrailsStepsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + TrailsStepsEntryLoadBody( elem, nodes, *slot ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// The BITPACKED body's cost, in BITS (docs/SPEC-TABLES.md §3.3). `at` is the +// body's own bit position in the batch, because a `string(N)` ALIGNS before +// its bytes and an align costs what the position says it costs. +template +inline int64_t TrailsMeasureMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, int64_t at, const Trails & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + int64_t bits = 0; + { + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); // steps + if ( !order_steps.ok ) { return -1; } // the sort could not run + if ( order_steps.count > 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; // the count the data decides + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_steps = TrailsStepsEntryMeasureMessageBody( ctx, numbering, index_bits, at + bits, *order_steps[i] ); + if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return -1; } + bits += elem_steps; // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) + { + bits += kTableMessageRefBitsHere; + bits += 32; + } + bits += kTableMessageRefBitsHere; // the ZERO REFERENCE that ends the body + (void) at; + return bits; +} + +// The BITPACKED body: the fields, then the ZERO REFERENCE that ends it. No +// kind byte rides at all, and no length frames a nested body, because a +// body is self-delimiting: it is written where the file form put an L. +template +inline bool TrailsSaveMessageBody( const Ctx & ctx, const TableNumbering & numbering, int64_t index_bits, TableBitWriter & w, const Trails & value ) +{ + (void) ctx; (void) numbering; (void) index_bits; + { + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); // steps + if ( !order_steps.ok ) { return false; } // the sort could not run + if ( order_steps.count > 0 ) + { + w.put( 48, kTableMessageRefBitsHere ); + w.put( (uint64_t) order_steps.count, 32 ); // the count the data decides + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + if ( !TrailsStepsEntrySaveMessageBody( ctx, numbering, index_bits, w, *order_steps[i] ) ) { TableMapRelease( order_steps ); return false; } + } + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) + { + w.put( 18, kTableMessageRefBitsHere ); + w.put( (uint64_t) ( value.after ), 32 ); + } + w.put( 0, kTableMessageRefBitsHere ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +// TrailsMessageExtent: the extent Trails's maps command on the message wire, from +// the FRAMING alone (docs/SPEC-TABLES.md §2.8, §3.3, §6.5). +inline bool TrailsMessageExtent( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & at ) +{ + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { return false; } + if ( ref == 0 ) { return true; } + if ( ref > (uint64_t) vocabulary.count ) { return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + if ( TableMessageReserved( entry.id ) ) { return false; } + if ( entry.id == 0x124250ad5a5b6d14ull && entry.kind == 14 && entry.elem_kind == 13 ) // steps + { + uint64_t n = 0; + if ( !r.get( n, TableBitsRequired( entry.min, entry.max ) ) ) { return false; } + n += (uint64_t) entry.min; + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( TrailsStepsEntry ) + at += (int64_t) n * (int64_t) sizeof( TrailsStepsEntry ); // the whole array FIRST + for ( uint64_t i = 0; i < n; i++ ) // then, entry by entry in key order + { + if ( !TrailsStepsEntryMessageExtent( r, vocabulary, index_bits, at ) ) { return false; } + } + continue; + } + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { return false; } + } +} + +// The BITPACKED body's read (docs/SPEC-TABLES.md §3.3): the declared +// defaults first, then whatever the wire says, field by field. An entry this +// build cannot name is skipped by its SHAPE and counted; one whose kind is +// not this field's is a kind mismatch and skipped the same way. +inline bool TrailsLoadMessageBody( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, Trails & value ) +{ + (void) nodes; (void) index_bits; + TrailsReset( value ); + for ( ;; ) + { + uint64_t ref = 0; + if ( !r.get( ref, vocabulary.ref_bits ) ) { report->malformed = true; return false; } + if ( ref == 0 ) { return true; } // the body ENDS AT ITS OWN ZERO REFERENCE + if ( ref > (uint64_t) vocabulary.count ) { report->malformed = true; return false; } + const TableMessageEntry & entry = TableVocabularyEntryAt( vocabulary, ref ); + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, IS + // MALFORMED (§3.1, §3.3): the node table is the ROOT body's first + // field and is read before this walk begins, so meeting one here is + // a second numbering wherever it sits + if ( TableMessageReserved( entry.id ) ) { report->malformed = true; return false; } + switch ( entry.id ) + { + case 0x124250ad5a5b6d14ull: // steps + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 14 || entry.elem_kind != 13 ) + { + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + uint64_t count = 0; + if ( !r.get( count, TableBitsRequired( entry.min, entry.max ) ) ) { report->malformed = true; return false; } + count += (uint64_t) entry.min; + TableMapFill fill = TableMapFillBegin( nodes, value.steps, (uint32_t) count ); + if ( !fill.ok ) { report->malformed = true; return false; } // the measure and the load disagree + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + const TrailsStepsEntryMessageKeyRead read = TrailsStepsEntryMessageReadKey( r, vocabulary, index_bits ); + if ( read.malformed ) { report->malformed = true; return false; } + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; report->widened++; } + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets to + // EMPTY, ONE kind_mismatch is counted for it, and the rest of its + // entries are stepped over by their shapes + report->kind_mismatch++; + TableMapFillReset( fill ); + r.offset = read.end; + for ( uint64_t j = i + 1; j < count; j++ ) { if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { report->malformed = true; return false; } } + break; + } + if ( read.over ) { report->clamped++; r.offset = read.end; continue; } // dropped whole, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) { report->malformed = true; return false; } // DESCENDING: not a body any conforming writer produced + TrailsStepsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset by the + // decode below, so LAST WINS WHOLE, and the count excludes it. + slot = TableMapFillLast( fill ); + report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { report->malformed = true; return false; } + if ( !TrailsStepsEntryLoadMessageBody( r, vocabulary, report, nodes, index_bits, *slot ) ) { return false; } + if ( r.offset != read.end ) { report->malformed = true; return false; } // the scan and the decode disagree about where the entry ends + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + break; + } + case 0xbf82010f6f71eae9ull: // after + { + // THE KIND MISMATCH IS FOUND IN THE ANNOUNCEMENT, not on the body. + // A RANGE that moved is not one: the shapes differ and the entry + // carries the SENDER's, so the field decodes and clamps (§4). + if ( entry.kind != 4 || entry.elem_kind != 0 ) + { + if ( entry.elem_kind == 0 && TableKindWidens( entry.kind, 4 ) ) + { + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + report->widened++; + break; + } + report->kind_mismatch++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + { + const int64_t width = entry.value_bits; + uint64_t raw = 0; + if ( width < 0 || !r.get( raw, width ) ) { report->malformed = true; return false; } + int64_t decoded_wide = (int64_t) raw; + if ( entry.packing == 1 ) { decoded_wide = (int64_t) ( raw + (uint64_t) entry.base_lo ); } + else if ( width > 0 && width < 64 ) + { + const uint64_t sign = uint64_t(1) << ( width - 1 ); + if ( ( raw & sign ) != 0 ) { decoded_wide = (int64_t) ( raw | ~( ( uint64_t(1) << width ) - 1 ) ); } + } + if ( decoded_wide < -2147483648ll ) { decoded_wide = -2147483648ll; report->clamped++; } + if ( decoded_wide > 2147483647ll ) { decoded_wide = 2147483647ll; report->clamped++; } + int32_t decoded_v = (int32_t) decoded_wide; + value.after = decoded_v; + } + break; + } + default: + report->unknown++; + if ( !TableMessageSkip( r, vocabulary, index_bits, entry ) ) { report->malformed = true; return false; } + break; + } + } +} + +// TrailsStepsEntryWireExtent: the extent TrailsStepsEntry's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool TrailsStepsEntryWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; // the scan's framing damage is the LOAD's to report + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { return true; } + if ( field_ref == 0 ) { return true; } + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { return true; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { return true; } + uint8_t field_kind = r.get8(); + if ( field_id == 0x7ce4fd9430e80ceaull && field_kind == 14 ) // value: an unbounded array + { + uint64_t list_len = 0; + if ( !r.getleb( list_len ) || !r.room( list_len ) ) { return true; } + const uint8_t * list_body = r.buffer + r.offset; + r.offset += (int64_t) list_len; + if ( !TableListWireExtent( list_body, (int64_t) list_len, at, (int64_t) sizeof( TableRef ), (int64_t) alignof( TableRef ), 17, 1, NULL, ids, reason ) ) { return false; } + continue; + } + if ( !r.skip( field_kind ) ) { return true; } + } +} + +// TrailsStepsEntryExtentAt: the node extent TrailsStepsEntry's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as TrailsStepsEntryExtentPack advances it (§2.8, §2.9). +template +inline bool TrailsStepsEntryExtentAt( const Ctx & ctx, const TrailsStepsEntry & value, int64_t & at ) +{ + { + TableListCursor cursor = TableListElements( ctx, value.value ); + if ( !cursor.ok ) { return false; } + at = ( at + (int64_t) alignof( TableRef ) - 1 ) & ~( (int64_t) alignof( TableRef ) - 1 ); + at += (int64_t) cursor.count * (int64_t) sizeof( TableRef ); // the whole array FIRST + } + return true; +} + +// the whole extent of one node, from a fresh offset: what a pack reserves +// for it beside the record's own storage. +template +inline int64_t TrailsStepsEntryExtent( const Ctx & ctx, const TrailsStepsEntry & value ) +{ + int64_t at = 0; + if ( !TrailsStepsEntryExtentAt( ctx, value, at ) ) { return -1; } + return at; +} + +// TrailsStepsEntryExtentPack: carve TrailsStepsEntry's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset TrailsStepsEntryExtentAt advances (§2.8, §2.9). +template +inline bool TrailsStepsEntryExtentPack( const Ctx & ctx, const TrailsStepsEntry & src, TrailsStepsEntry & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + { + TableListCursor cursor = TableListElements( ctx, src.value ); + if ( !cursor.ok ) { return false; } + at = ( at + (int64_t) alignof( TableRef ) - 1 ) & ~( (int64_t) alignof( TableRef ) - 1 ); + const int64_t bytes = (int64_t) cursor.count * (int64_t) sizeof( TableRef ); + if ( at + bytes > capacity ) { return false; } + TableRef * placed = (TableRef *) ( extent + at ); + at += bytes; + dst.value.count = cursor.count; + dst.value.padding = 0; + dst.value.elements.value = cursor.count > 0 ? (int64_t) ( (uint8_t *) placed - (const uint8_t *) &dst.value.elements ) : 0; + for ( int32_t i = 0; i < cursor.count; i++ ) // INDEX order, live elements only + { + memcpy( (void *) ( placed + i ), (const void *) &cursor[i], sizeof( TableRef ) ); // trivially copyable, by construction + } + } + return true; +} + +// TrailsWireExtent: the extent Trails's lists and maps command, from the FRAMING alone. +// It reads no field value, so a caller can refuse a number it did not +// expect before one byte is allocated (docs/SPEC-TABLES.md §6.5). +inline bool TrailsWireExtent( const uint8_t * body, int64_t length, int64_t & at, const TableIdTable * ids, TableRefuseReason & reason ) +{ + TableReport scratch; // the scan's framing damage is the LOAD's to report + TableReader r( body, length, &scratch, ids ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { return true; } + if ( field_ref == 0 ) { return true; } + if ( ids == NULL || field_ref > (uint64_t) ids->count ) { return true; } + const uint64_t field_id = ids->at( field_ref ); + if ( !r.has( 1 ) ) { return true; } + uint8_t field_kind = r.get8(); + if ( field_id == 0x124250ad5a5b6d14ull && field_kind == 14 ) // steps + { + uint64_t map_len = 0; + if ( !r.getleb( map_len ) || !r.room( map_len ) ) { return true; } + const uint8_t * map_body = r.buffer + r.offset; + r.offset += (int64_t) map_len; + if ( !TableMapWireExtent( map_body, (int64_t) map_len, at, (int64_t) sizeof( TrailsStepsEntry ), (int64_t) alignof( TrailsStepsEntry ), &TrailsStepsEntryWireExtent, ids, reason ) ) { return false; } + continue; + } + if ( !r.skip( field_kind ) ) { return true; } + } +} + +// TrailsExtentAt: the node extent Trails's lists and maps take, PRE-ORDER, advancing +// the running offset exactly as TrailsExtentPack advances it (§2.8, §2.9). +template +inline bool TrailsExtentAt( const Ctx & ctx, const Trails & value, int64_t & at ) +{ + { + TableMapCursor cursor = TableMapOrder( ctx, value.steps ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( TrailsStepsEntry ) + at += (int64_t) cursor.count * (int64_t) sizeof( TrailsStepsEntry ); // the whole array FIRST + for ( int32_t i = 0; i < cursor.count; i++ ) // then, entry by entry in key order + { + if ( !TrailsStepsEntryExtentAt( ctx, *cursor[i], at ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// the whole extent of one node, from a fresh offset: what a pack reserves +// for it beside the record's own storage. +template +inline int64_t TrailsExtent( const Ctx & ctx, const Trails & value ) +{ + int64_t at = 0; + if ( !TrailsExtentAt( ctx, value, at ) ) { return -1; } + return at; +} + +// TrailsExtentPack: carve Trails's arrays out of the node's extent and copy the +// entries in ASCENDING key order and the elements in INDEX order, PRE-ORDER, +// advancing the same running offset TrailsExtentAt advances (§2.8, §2.9). +template +inline bool TrailsExtentPack( const Ctx & ctx, const Trails & src, Trails & dst, uint8_t * extent, int64_t & at, int64_t capacity ) +{ + { + TableMapCursor cursor = TableMapOrder( ctx, src.steps ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; + const int64_t bytes = (int64_t) cursor.count * (int64_t) sizeof( TrailsStepsEntry ); + if ( at + bytes > capacity ) { TableMapRelease( cursor ); return false; } + TrailsStepsEntry * placed = (TrailsStepsEntry *) ( extent + at ); + at += bytes; + dst.steps.count = cursor.count; + dst.steps.padding = 0; + dst.steps.entries.value = cursor.count > 0 ? (int64_t) ( (uint8_t *) placed - (const uint8_t *) &dst.steps.entries ) : 0; + for ( int32_t i = 0; i < cursor.count; i++ ) + { + memcpy( (void *) ( placed + i ), (const void *) cursor[i], sizeof( TrailsStepsEntry ) ); // trivially copyable, by construction + } + for ( int32_t i = 0; i < cursor.count; i++ ) + { + if ( !TrailsStepsEntryExtentPack( ctx, *cursor[i], placed[i], extent, at, capacity ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// ---- Trails.steps: the builder's five and the side index (§2.8) ---- + +// INSERT: the key is copied, the value is handed back at its defaults to +// fill. A DUPLICATE key REPLACES — the value is reset and the same entry +// handed back, key and address unchanged — so a caller that wants to know +// writes Find first. NULL is NOT INSERTED: a key longer than the bound, +// because a truncated key would be a merged entry, and an arena that +// cannot carve another segment, alike. +// +// It is a WRAPPER: TableMapPlace owns the lookup, the reset, the +// allocation and the key copy, and this half is the bound and the +// const char * key's length. Nothing here mutates an entry (§2.8). +inline TableList * TrailsStepsInsert( TableWorker & worker, TableMap & map, uint32_t key ) +{ + TrailsStepsEntry * entry = TableMapPlace( worker, map, key ); + return entry != NULL ? TableEntryValue( entry ) : NULL; +} + +// FIND on the builder: the same linear scan, O( n ) key compares over the +// segments in insertion order. NULL when absent. The builder builds NO +// INDEX, and that is a rule — the sort happens once, at Lock, Save or +// Cook, and every lookup that matters runs over the sorted region. +inline TableList * TrailsStepsFind( TableArena & arena, TableMap & map, uint32_t key ) +{ + TrailsStepsEntry * found = TableMapScan( arena, map, key ); + return found != NULL ? TableEntryValue( found ) : NULL; +} + +// ERASE: marks the entry DEAD, one bit in the segment's slot and not in the +// entry table. False when absent. Its storage is held until the builder +// resets and never reused mid-build, because reusing a slot would make "an +// entry's address is stable" false for exactly one case. +inline bool TrailsStepsErase( TableArena & arena, TableMap & map, uint32_t key ) +{ + return TableMapErase( arena, map, key ); +} + +// EACH on the builder: INSERTION order, live entries only. +inline TableMapEach TrailsStepsEach( const TableArena & arena, const TableMap & map ) +{ + return TableMapEachOf( arena, map ); +} + +// ---- the OPTIONAL INDEX: caller-owned, built at load, never stored ---- +// +// Open addressing with linear probing over the sorted array, for a map large +// enough that log n compares over a cold array cost more than one hash and a +// probe. ITS HASH AND ITS LOAD FACTOR ARE NOT A CROSS-PORT CONTRACT: the +// index is never stored, so no golden, no cook-check rule and no +// build-version line ever names either. What a port is held to is the +// CONTRACT of the lookup — the same value the sorted array's Find returns +// for the same key, and no allocation past the storage the caller handed in. +inline int64_t TrailsStepsIndexMeasure( const TableMap & map ) +{ + return (int64_t) TableMapIndexSlots( map.count ) * (int64_t) sizeof( int32_t ); +} + +inline TableMapIndex TrailsStepsIndex( const TableMap & map, void * storage, int64_t bytes ) +{ + TableMapIndex index; + const int32_t slots = TableMapIndexSlots( map.count ); + if ( storage == NULL || bytes < (int64_t) slots * (int64_t) sizeof( int32_t ) ) { return index; } + index.slots = (int32_t *) storage; + index.capacity = slots; + for ( int32_t i = 0; i < slots; i++ ) { index.slots[i] = 0; } + const TrailsStepsEntry * entries = map.Entries(); + for ( int32_t i = 0; i < map.count; i++ ) // ONE PASS over the sorted array + { + int32_t at = (int32_t) ( TableMapHash( (uint64_t) entries[i].key ) & (uint64_t) ( slots - 1 ) ); + while ( index.slots[at] != 0 ) { at = ( at + 1 ) & ( slots - 1 ); } + index.slots[at] = i + 1; // slots are ENTRY INDICES; 0 is an empty slot + } + index.good = true; + return index; +} + +inline const TableList * TrailsStepsIndexFind( const TableMapIndex & index, const TableMap & map, uint32_t key ) +{ + if ( !index.good ) { return map.Find( key ); } // an index that did not build is not a wrong answer + const TrailsStepsEntry * entries = map.Entries(); + int32_t at = (int32_t) ( TableMapHash( (uint64_t) key ) & (uint64_t) ( index.capacity - 1 ) ); + for ( int32_t probe = 0; probe < index.capacity; probe++ ) + { + const int32_t slot = index.slots[at]; + if ( slot == 0 ) { return NULL; } + if ( TableEntryOrder( entries[slot - 1], key ) == 0 ) { return TableEntryFound( entries + slot - 1 ); } + at = ( at + 1 ) & ( index.capacity - 1 ); + } + return NULL; +} + +// ---- TrailsStepsEntry.value: the builder's three (§2.9) ---- + +// ADD: the element is appended and handed back to fill. On a []*T that is +// the SLOT at null, which ItemEmplace fills as it fills any pointer slot, +// and a second slot may hold the same reference: two slots, one node. +// NULL means NOT ADDED: an arena that cannot carve another segment, or a +// count at the int32 cap. A caller that needs the reason checks size(). +inline TableRef * TrailsStepsEntryValueAdd( TableWorker & worker, TableList & list ) +{ + return TableListPlace( worker, list ); +} + +// ERASE, by the element's own pointer: marks it DEAD, one bit in the +// segment's slot and not in the element storage. False when the pointer is +// not this list's. Storage is held until the builder resets. INDICES ARE +// NOT STABLE ACROSS AN ERASE: what was index 3 is index 2 in the next Save. +inline bool TrailsStepsEntryValueErase( TableArena & arena, TableList & list, const TableRef * element ) +{ + return TableListErase( arena, list, element ); +} + +// EACH on the builder: INDEX order, live elements only, yielding the +// element Add handed back. +inline TableListEach TrailsStepsEntryValueEach( const TableArena & arena, const TableList & list ) +{ + return TableListEachOf( arena, list ); +} + +// TrailsStepsEntryNumber: number everything TrailsStepsEntry POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool TrailsStepsEntryNumber( const Ctx & ctx, TableNumbering & numbering, const TrailsStepsEntry & value ) +{ + { // value: a by-value edge, elements in INDEX order (§2.9, §3.1) + TableListCursor cursor_value = TableListElements( ctx, value.value ); + if ( !cursor_value.ok ) { return false; } + for ( int32_t i = 0; i < cursor_value.count; i++ ) + { + { + const Item * pointee = ItemAt( ctx, cursor_value[i] ); // value + if ( pointee != NULL ) + { + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( numbering.seen, (const void *) pointee, + (int64_t) ( numbering.count + 2 ), taken, slot ); // its index, if this is its first visit + if ( entry == NULL ) { return false; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return false; } // a data cycle + } + else + { + TableNodeEntry node; + node.node = (const void *) pointee; + node.type_id = 0x52cfa1d198476806ull; // fnv1a64( "Item" ) + node.type_slot = 65; // its slot in the unit's vocabulary (§3.3) + node.measure = &TableNodeMeasureThunk; + node.save = &TableNodeSaveThunk; + node.message_measure = &TableNodeMessageMeasureThunk; + node.message_save = &TableNodeMessageSaveThunk; + if ( !TableNumberingAppend( numbering, node ) ) { return false; } + if ( !ItemNumber( ctx, numbering, *pointee ) ) { return false; } + TablePackMapClose( numbering.seen, (const void *) pointee, slot ); + } + } + } + } + } + return true; +} + +// TrailsStepsEntryPackMeasure: the packed region bytes of everything TrailsStepsEntry POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t TrailsStepsEntryPackMeasure( const Ctx & ctx, TablePackMap & seen, const TrailsStepsEntry & value ) +{ + int64_t bytes = 0; + { // value: a by-value edge, elements in INDEX order (§2.9, §3.1) + TableListCursor cursor_value = TableListElements( ctx, value.value ); + if ( !cursor_value.ok ) { return -1; } + for ( int32_t i = 0; i < cursor_value.count; i++ ) + { + { + const Item * pointee = ItemAt( ctx, cursor_value[i] ); // value + if ( pointee != NULL ) + { + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( seen, (const void *) pointee, 0, taken, slot ); + if ( entry == NULL ) { return -1; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return -1; } // a data cycle + } + else + { + int64_t inner = ItemPackMeasure( ctx, seen, *pointee ); + if ( inner < 0 ) { return -1; } + TablePackMapClose( seen, (const void *) pointee, slot ); + bytes += TableAlignUp64( (int64_t) sizeof( Item ) ) + inner; + } + } + } + } + } + return bytes; +} + +// TrailsStepsEntryPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool TrailsStepsEntryPackEdges( const Ctx & ctx, TablePackMap & seen, const TrailsStepsEntry & src, TrailsStepsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool TrailsStepsEntryPack( const Ctx & ctx, TablePackMap & seen, const TrailsStepsEntry & src, TrailsStepsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( TrailsStepsEntry ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( TrailsStepsEntry ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !TrailsStepsEntryExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return TrailsStepsEntryPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool TrailsStepsEntryPackEdges( const Ctx & ctx, TablePackMap & seen, const TrailsStepsEntry & src, TrailsStepsEntry & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + { // value: a by-value edge, elements in INDEX order (§2.9, §3.1) + TableListCursor cursor_value = TableListElements( ctx, src.value ); + if ( !cursor_value.ok ) { return false; } + TableRef * placed_value = (TableRef *) ( dst.value.elements.value != 0 ? ( (uint8_t *) &dst.value.elements + dst.value.elements.value ) : NULL ); + for ( int32_t i = 0; i < cursor_value.count; i++ ) + { + { + placed_value[i].value = 0; // value + const Item * pointee = ItemAt( ctx, cursor_value[i] ); + if ( pointee != NULL ) + { + int64_t at = TableAlignUp64( used ); // where it WOULD land, if this is its first visit + bool taken = false; + int64_t slot = 0; + const TablePackEntry * entry = TablePackMapReach( seen, (const void *) pointee, at, taken, slot ); + if ( entry == NULL ) { return false; } // the map could not grow + if ( !taken ) + { + if ( entry->open != 0 ) { return false; } // a data cycle + placed_value[i].value = (int64_t) ( ( base + entry->offset ) - (const uint8_t *) &placed_value[i] ); // the one body it already has + } + else + { + if ( at + (int64_t) sizeof( Item ) > capacity ) { return false; } + used = at + TableAlignUp64( (int64_t) sizeof( Item ) ); + Item * child = new ( base + at ) Item; // lifetime only: the Pack below memcpy's the whole node over it + placed_value[i].value = (int64_t) ( ( base + at ) - (const uint8_t *) &placed_value[i] ); + if ( !ItemPack( ctx, seen, *pointee, *child, base, capacity, used ) ) { return false; } + TablePackMapClose( seen, (const void *) pointee, slot ); + } + } + } + } + } + return true; +} + +// TrailsNumber: number everything Trails POINTS AT, in first-visit order — +// the fields in declaration order, a by-value edge descended in place. +// A reference to an entry whose descent is still OPEN is a data cycle, +// named here rather than recursed away (docs/SPEC-TABLES.md §3.1). +template +inline bool TrailsNumber( const Ctx & ctx, TableNumbering & numbering, const Trails & value ) +{ + { // steps: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_steps = TableMapOrder( ctx, value.steps ); + if ( !cursor_steps.ok ) { return false; } + for ( int32_t i = 0; i < cursor_steps.count; i++ ) + { + if ( !TrailsStepsEntryNumber( ctx, numbering, *cursor_steps[i] ) ) { TableMapRelease( cursor_steps ); return false; } + } + TableMapRelease( cursor_steps ); + } + return true; +} + +// TrailsPackMeasure: the packed region bytes of everything Trails POINTS AT. +// ONE VISIT PER NODE: `seen` carries the first-visit numbering (§3.1), so a +// node two references name is measured ONCE and packed once, and a +// reference to a node whose descent is still open is a data cycle, refused. +template +inline int64_t TrailsPackMeasure( const Ctx & ctx, TablePackMap & seen, const Trails & value ) +{ + int64_t bytes = 0; + { // steps: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_steps = TableMapOrder( ctx, value.steps ); + if ( !cursor_steps.ok ) { return -1; } + for ( int32_t i = 0; i < cursor_steps.count; i++ ) + { + int64_t inner = TrailsStepsEntryPackMeasure( ctx, seen, *cursor_steps[i] ); + if ( inner < 0 ) { TableMapRelease( cursor_steps ); return -1; } + bytes += inner; + } + TableMapRelease( cursor_steps ); + } + return bytes; +} + +// TrailsPack: copy src into dst (already placed), then lay every pointee out +// depth-first behind it, in FIELD ORDER, by bump allocation. +// +// ONE NODE, ONE BODY (§6.2): `seen` holds every node already placed and +// where it landed, so a node's FIRST reference lays it out and every later +// reference points BACK at that one body. A region delta therefore has no +// required sign (§6.3), and sharing and a back-reference are one fact. A +// reference to a node whose descent is still OPEN is a cycle, and this +// refuses it rather than packing one. +template +inline bool TrailsPackEdges( const Ctx & ctx, TablePackMap & seen, const Trails & src, Trails & dst, uint8_t * base, int64_t capacity, int64_t & used ); + +template +inline bool TrailsPack( const Ctx & ctx, TablePackMap & seen, const Trails & src, Trails & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + memcpy( (void *) &dst, (const void *) &src, sizeof( Trails ) ); // trivially copyable, by construction + int64_t at = 0; + uint8_t * extent = (uint8_t *) &dst + TableAlignUp64( (int64_t) sizeof( Trails ) ); + const int64_t room = capacity - ( (int64_t) ( extent - base ) ); + if ( !TrailsExtentPack( ctx, src, dst, extent, at, room ) ) { return false; } + return TrailsPackEdges( ctx, seen, src, dst, base, capacity, used ); +} + +template +inline bool TrailsPackEdges( const Ctx & ctx, TablePackMap & seen, const Trails & src, Trails & dst, uint8_t * base, int64_t capacity, int64_t & used ) +{ + { // steps: a by-value edge, entries in ASCENDING key order (§2.8, §3.1) + TableMapCursor cursor_steps = TableMapOrder( ctx, src.steps ); + if ( !cursor_steps.ok ) { return false; } + TrailsStepsEntry * placed_steps = (TrailsStepsEntry *) ( dst.steps.entries.value != 0 ? ( (uint8_t *) &dst.steps.entries + dst.steps.entries.value ) : NULL ); + for ( int32_t i = 0; i < cursor_steps.count; i++ ) + { + if ( !TrailsStepsEntryPackEdges( ctx, seen, *cursor_steps[i], placed_steps[i], base, capacity, used ) ) { TableMapRelease( cursor_steps ); return false; } + } + TableMapRelease( cursor_steps ); + } + return true; +} + +// ---- Trails: the variable-length life (docs/SPEC-TABLES.md §2, §6, §9) ---- +// +// MUTABLE: TrailsBuilder — allocate nodes, wire them together, then Lock. +// CONST: one packed region, root at its base. Lock produces it and Load +// produces it, so a locked structure and a loaded one are the +// SAME representation with one view API. There is no unlock: +// re-editing means loading the const form into a fresh builder. +// Trails is never held by value — a file-format-scale structure is a region +// and a root pointer, not a struct you copy. + +struct TrailsBuilder +{ + TableArena arena; + TableWorker main; // the calling thread's allocation front + TableRef root_ref; + uint8_t * region = NULL; // the packed const form, produced by Lock() + int64_t region_bytes = 0; + + // THE ALLOCATOR IS THE BUILDER'S, and everything this structure ever + // allocates goes through it: the arena's segments, Lock's identity map, + // the packed region, the wire walks' numbering, and the tool path's node + // directory. Name your own and a profiler sees every byte under it. + TrailsBuilder( TableAllocator allocator = TableDefaultAllocator() ) + { + TableArenaInit( arena, allocator ); + main.arena = &arena; + TableSlot slot = main.Alloc(); + root_ref = slot.ref; + } + ~TrailsBuilder() { TableArenaShutdown( arena ); arena.allocator.free( arena.allocator.context, region ); } + TrailsBuilder( const TrailsBuilder & ) = delete; + TrailsBuilder & operator=( const TrailsBuilder & ) = delete; + + // Alloc a node in THIS thread's slab: no lock, no atomic per node. + // The result is usable both as the node pointer and as the reference + // to store in a pointer field. + template TableSlot Alloc() { return main.Alloc(); } + // a BYTE BUFFER's node of exactly `length` bytes (docs/SPEC-TABLES.md §2.5): + // the bytes to write through, and the reference to store in a *bytes + // or *string slot; a blob past a slab takes a span of its own + TableBytesSlot AllocBytes( int64_t length ) { return main.AllocBytes( length ); } + TableStringSlot AllocString( int64_t length ) { return main.AllocString( length ); } + // one worker per thread; allocate on your own, and synchronize your own + // writes to nodes another worker allocated + TableWorker Worker() { TableWorker worker; worker.arena = &arena; return worker; } + + // GetRoot/AsConst, not Root/Const: a member function hides the type + // name it shares, and `table Root` is this spec's own canonical + // example. The checker refuses a table named after any member here, + // so the remaining spellings cannot collide either. + Trails * GetRoot() { return arena.locked ? NULL : (Trails *) TableArenaAt( arena, (uint32_t) root_ref.value ); } + bool Locked() const { return arena.locked; } + const Trails * AsConst() const { return (const Trails *) region; } + const uint8_t * Region() const { return region; } + int64_t RegionBytes() const { return region_bytes; } + + // Lock is ONE WAY and it is the compaction: the segmented arena becomes + // one exact-packed region with zero slack, references rewritten + // self-relative, and the mutable life released. Single-threaded: call + // it after the workers have joined. + bool Lock(); +}; + +inline bool TrailsBuilder::Lock() +{ + if ( arena.locked ) { return region != NULL; } + if ( root_ref.null() ) { return false; } + TableArenaCtx ctx = { &arena }; + const Trails & root = *(const Trails *) TableArenaAt( arena, (uint32_t) root_ref.value ); + // The ROOT takes the map's first entry: it is packed at offset 0, and its + // descent is open for the whole walk (docs/SPEC-TABLES.md §3.1). + TablePackMap seen; + TablePackMapInit( seen, arena.allocator ); + bool root_taken = false; + int64_t root_slot = 0; + int64_t below = -1; + if ( TablePackMapReach( seen, (const void *) &root, 0, root_taken, root_slot ) != NULL ) + { + below = TrailsPackMeasure( ctx, seen, root ); + } + if ( below < 0 ) { TablePackMapShutdown( seen ); return false; } // a data cycle, named at the reference that closes it + int64_t root_extent = TrailsExtent( ctx, root ); + if ( root_extent < 0 ) { TablePackMapShutdown( seen ); return false; } // the sort could not run + int64_t total = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ) + below; + // the AUTHORING path may allocate (§6.5), and it does so through the + // builder's own pair. The region comes back ZEROED, which is the + // allocator's contract: a packed region carries node padding. + uint8_t * packed = (uint8_t *) arena.allocator.alloc( arena.allocator.context, total ); + if ( packed == NULL ) { TablePackMapShutdown( seen ); return false; } + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + Trails * destination = new ( packed ) Trails; // lifetime only: the Pack below memcpy's the whole node over it + // The pack walk RE-DERIVES the same numbering rather than carrying the + // measure's — nothing passes between them, which is what makes + // `used == total` below a real check and not a tautology (§3.1). The + // map keeps the capacity the measure paid for, so the second walk + // rehashes nothing. + TablePackMapReset( seen ); + if ( TablePackMapReach( seen, (const void *) &root, 0, root_taken, root_slot ) == NULL || + !TrailsPack( ctx, seen, root, *destination, packed, total, used ) || used != total ) + { + TablePackMapShutdown( seen ); + arena.allocator.free( arena.allocator.context, packed ); + return false; + } + TablePackMapShutdown( seen ); + region = packed; + region_bytes = total; + arena.locked = true; // MONOTONIC: there is no unlock + TableArenaShutdown( arena ); + return true; +} + +// ---- Trails on the wire: the FLAT NODE TABLE (docs/SPEC-TABLES.md §3.1) ---- +// +// A pointered save writes every reachable node ONCE, into a node table under +// the reserved id 0xFFFF, and a pointer field rides as a u32 INDEX into it +// under kind 17. No pointer edge is a nesting level, so a chain's length is +// not a depth and two references to one node are one node. + +// TrailsNodeStorage: the region bytes one record commands, or -1 for a type id +// this build cannot name — which keeps its index and reads null. A BYTE +// BUFFER's record commands its header and its bytes (docs/SPEC-TABLES.md §2.5), +// which is the one answer the record's LENGTH decides, and a blob past the +// size cap answers kTableNodeRefused with its reason (§3.1, §6.5). +// A MAP'S ENTRIES RIDE IN THEIR HOLDER'S EXTENT (docs/SPEC-TABLES.md §2.8), +// so a record's storage is its type's PLUS N x sizeof( Entry ) at every +// depth, summed from the FRAMING: N is framing and not a value, and this +// reads no field. kTableNodeRefused is a wire whose N its L cannot carry. +inline int64_t TrailsNodeStorage( uint64_t type_id, int64_t length, TableRefuseReason & reason ) +{ + (void) length; // no byte buffer below this root: every node's storage is its type's + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( (int64_t) sizeof( Item ) ); // Item + default: break; + } + (void) reason; // no blob and no extent below this root: nothing here refuses + return -1; +} + +// TrailsNodePlace: start one record's node's lifetime in the storage pass one +// reserved for it, holding exactly the declared defaults — a byte buffer's +// header holds its length, and its bytes come in pass two. +inline void TrailsNodePlace( uint64_t type_id, uint8_t * at, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: { Item * node = new ( at ) Item; ItemReset( *node ); break; } // Item + default: break; + } +} + +// TrailsNodeRecordBytes: one record's OWN storage, before the extent its maps +// take (docs/SPEC-TABLES.md §2.8) — where a node's extent begins. +inline int64_t TrailsNodeRecordBytes( uint64_t type_id ) +{ + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( (int64_t) sizeof( Item ) ); // Item + default: break; + } + return 0; +} + +// TrailsNodeAlloc: the TOOL's path — one record's node in the builder's arena. +// Zero is the arena's null, and it is also what a type id this build cannot +// name answers. +inline uint32_t TrailsNodeAlloc( uint64_t type_id, TableWorker & worker, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return (uint32_t) worker.Alloc().ref.value; // Item + default: break; + } + return 0; +} + +// TrailsNodeBody: PASS TWO's half — decode one record's body into the storage it +// already owns. +inline void TrailsNodeBody( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = TrailsNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? TrailsNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ItemLoadBody( r, *(Item *) at ); break; // Item + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// TrailsNodeMessageStorage: the region bytes one record commands on the message +// wire, or -1 for a type id this build cannot name. A table's is its own +// storage plus the extent its maps take; a byte buffer's is its header and +// its bytes, which is the one answer the record's LENGTH decides. +inline int64_t TrailsNodeMessageStorage( uint64_t type_id, int64_t extent, int64_t length ) +{ + (void) length; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Item ) ) + extent ); // Item + default: break; + } + return -1; +} + +// TrailsNodeMessageExtent: step over one TABLE record's body, tallying the extent +// its maps take where its type has any (§2.8). A type this build cannot +// name is stepped over by its announced shapes and takes no extent. +inline bool TrailsNodeMessageExtent( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, int64_t & extent ) +{ + extent = 0; + (void) type_id; // no map below any node this root can name + return TableMessageSkipBody( r, vocabulary, index_bits ); +} + +// TrailsNodeMessageBody: PASS TWO's half, which decodes one record's body into +// storage it already owns, its map entries carved from its own extent. +inline bool TrailsNodeMessageBody( uint64_t type_id, TableBitReader & r, const TableVocabulary & vocabulary, TableReport * report, const TableNodeMap & nodes, int64_t index_bits, uint8_t * at ) +{ + TableExtentCarve carve; + carve.at = at + TrailsNodeRecordBytes( type_id ); + carve.left = 0; + { + // the extent this record was placed with, re-read from the framing + TableBitReader walk = r; + int64_t extent = 0; + if ( !TrailsNodeMessageExtent( type_id, walk, vocabulary, index_bits, extent ) ) { report->malformed = true; return false; } + carve.left = extent; + } + TableExtentCarve * const outer = nodes.carve; + nodes.carve = &carve; + (void) nodes; (void) index_bits; // every node this root can name is a FIXED table + bool ok = false; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemLoadMessageBody( r, vocabulary, report, index_bits, *(Item *) at ); break; // Item + // a record this dispatch cannot name never reaches here: pass one left it absent + default: report->malformed = true; break; + } + nodes.carve = outer; + return ok; +} + +// The numbering both wire walks derive, and NEITHER CARRIES THE OTHER'S: the +// root takes index 1 and its entry stays open for the whole walk, so a +// reference back at it is the cycle it is (§3.1). +template +inline bool TrailsNumberFrom( const Ctx & ctx, TableNumbering & numbering, const Trails & root ) +{ + bool taken = false; + int64_t slot = 0; + if ( TablePackMapReach( numbering.seen, (const void *) &root, (int64_t) kTableNodeIndexRoot, taken, slot ) == NULL ) { return false; } + return TrailsNumber( ctx, numbering, root ); +} + +template +inline int64_t TrailsMeasureWire( const Ctx & ctx, const Trails & root, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bytes = -1; + if ( TrailsNumberFrom( ctx, numbering, root ) ) + { + TableIds ids; + bytes = TrailsMeasureBody( ctx, numbering, ids, root ); + if ( bytes >= 0 ) + { + const int64_t table = TableNodeTableMeasure( ctx, ids, numbering ); + // the FORM BYTE, the ROOT BODY — its own fields, the node table + // and the terminator — and the ID TABLE (docs/SPEC-TABLES.md §3) + bytes = table < 0 || ids.overflow ? -1 : 1 + bytes + table + TableIdsBytes( ids ); + } + } + TableNumberingShutdown( numbering ); + return bytes; +} + +template +inline int64_t TrailsSaveWire( const Ctx & ctx, const Trails & root, uint8_t * buffer, int64_t capacity, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + if ( !TrailsNumberFrom( ctx, numbering, root ) ) { TableNumberingShutdown( numbering ); return -1; } + TableWriter w( buffer, capacity ); + TableIds ids; + w.put8( kTableWireForm ); // the FORM BYTE is the whole header (§3) + // the root's own fields, then the node table's field, then the + // terminator: a reader that gives up inside the table has already + // decoded the ROOT'S OWN FIELDS (§3.1) + bool ok = TrailsSaveBodyFields( ctx, numbering, w, ids, root ) && TableNodeTableSave( ctx, w, ids, numbering ); + TableNumberingShutdown( numbering ); + if ( !ok || ids.overflow ) { return -1; } + w.put8( 0 ); // the ZERO REFERENCE that ends the root body + TableIdsWrite( w, ids ); + if ( w.overflow ) { return -1; } // the caller's buffer was too small + return w.offset; // == TrailsMeasure( root ) +} + +inline int64_t TrailsMeasure( const Trails * root, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return TrailsMeasureWire( ctx, *root, allocator ); +} + +inline int64_t TrailsSave( const Trails * root, uint8_t * buffer, int64_t capacity, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return TrailsSaveWire( ctx, *root, buffer, capacity, allocator ); +} + +inline int64_t TrailsMeasure( const TrailsBuilder & builder ) +{ + if ( builder.region != NULL ) { return TrailsMeasure( builder.AsConst(), builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return TrailsMeasureWire( ctx, *(const Trails *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), builder.arena.allocator ); +} + +inline int64_t TrailsSave( const TrailsBuilder & builder, uint8_t * buffer, int64_t capacity ) +{ + if ( builder.region != NULL ) { return TrailsSave( builder.AsConst(), buffer, capacity, builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return TrailsSaveWire( ctx, *(const Trails *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), buffer, capacity, builder.arena.allocator ); +} + +// ---- Trails on the MESSAGE wire: the batch over a region (docs/SPEC-TABLES.md §3.3) ---- + +// TrailsMessageBodyBits: one root body's bits at bit position `at` of the batch, +// with the numbering derived from the graph, the node table FIRST, then the +// fields, then the zero reference. Measure derives the numbering and save +// derives the same one, and nothing passes between them (§3.1). +template +inline int64_t TrailsMessageBodyBits( const Ctx & ctx, const Trails & root, int64_t at, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bits = -1; + if ( TrailsNumberFrom( ctx, numbering, root ) ) + { + const int64_t index_bits = TableBitsRequired( 0, numbering.count + 1 ); + const int64_t table = TableMessageNodeTableMeasure( ctx, numbering, index_bits, at ); + if ( table >= 0 ) + { + const int64_t body = TrailsMeasureMessageBody( ctx, numbering, index_bits, at + table, root ); + bits = body < 0 ? -1 : table + body; + } + } + TableNumberingShutdown( numbering ); + return bits; +} + +template +inline bool TrailsMessageBodySave( const Ctx & ctx, const Trails & root, TableBitWriter & w, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + bool ok = false; + if ( TrailsNumberFrom( ctx, numbering, root ) ) + { + const int64_t index_bits = TableBitsRequired( 0, numbering.count + 1 ); + ok = TableMessageNodeTableSave( ctx, numbering, index_bits, w ) && TrailsSaveMessageBody( ctx, numbering, index_bits, w, root ); + } + TableNumberingShutdown( numbering ); + return ok && !w.overflow; +} + +// THE PRIMITIVE IS A BATCH (§3.3): a number of ROOTS in one buffer, one count +// and one continuous bit stream, each body carrying its own numbering. A +// root is a locked region's, `builder.AsConst()`, or a loaded one's. M above +// 256 is a refusal by name, batch_too_large, with nothing written. +inline int64_t TrailsMeasureMessages( const Trails * const * roots, int64_t count, TableReport * report, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( roots == NULL || count < 1 ) { return -1; } + if ( count > kTableMessageBatchMax ) { TableMessageRefuseBatch( report ); return -1; } + TableRegionCtx ctx; + int64_t bits = 8; // the body count + for ( int64_t i = 0; i < count; i++ ) + { + if ( roots[i] == NULL ) { return -1; } + const int64_t body = TrailsMessageBodyBits( ctx, *roots[i], bits, allocator ); + if ( body < 0 ) { return -1; } + bits += body; + } + return 1 + ( bits + 7 ) / 8; +} + +inline int64_t TrailsSaveMessages( const Trails * const * roots, int64_t count, uint8_t * buffer, int64_t capacity, TableReport * report, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( roots == NULL || count < 1 ) { return -1; } + if ( count > kTableMessageBatchMax ) { TableMessageRefuseBatch( report ); return -1; } + TableMessageBatch batch; + if ( !TableMessageBatchBegin( batch, buffer, capacity, count ) ) { return -1; } + TableRegionCtx ctx; + for ( int64_t i = 0; i < count; i++ ) + { + if ( roots[i] == NULL || !TrailsMessageBodySave( ctx, *roots[i], batch.w, allocator ) ) { return -1; } + batch.written++; + } + return TableMessageBatchEnd( batch ); // == TrailsMeasureMessages( roots, count, report, allocator ) +} + +// TrailsMessageRecordScan: one node record's type id and the extent its maps +// take, or a blob's length, the reader left after the record. A type id +// reference of 0, one past E, or one naming anything but a kind-0 entry is +// damage, as §3.1 and §3.3 say. +inline bool TrailsMessageRecordScan( TableBitReader & r, const TableVocabulary & vocabulary, int64_t index_bits, uint64_t & type_id, int64_t & extent, int64_t & length ) +{ + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { return false; } + TableMessageEntry type_entry; + if ( !TableMessageNameEntry( vocabulary, type_ref, type_entry ) ) { return false; } + type_id = type_entry.id; + extent = 0; + length = 0; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + // A BLOB RECORD CARRIES A LENGTH AT THIRTY-TWO RAW BITS, then ALIGNS, + // then the bytes verbatim (§3.3) + uint64_t n = 0; + if ( !r.get( n, 32 ) || !r.align() || !r.skip( (int64_t) n * 8 ) ) { return false; } + length = (int64_t) n; + return true; + } + return TrailsNodeMessageExtent( type_id, r, vocabulary, index_bits, extent ); +} + +// TrailsMessageBodyStorage: one body's node count and data bytes from the FRAMING +// alone, the reader left at the next body. The node table is walked record +// by record, a table record's body stepped over by its announced shapes and +// a blob's by its length, then the root's own fields. False is a numbering +// that could not be sized; `complete` false is a ROOT body whose own framing +// gave out, which the load meets as damage inside this body after the +// bodies before it were delivered, so the batch is sized through this body +// and no further (§3.3). +inline bool TrailsMessageBodyStorage( TableBitReader & r, const TableVocabulary & vocabulary, int64_t & records, int64_t & data, bool & complete ) +{ + complete = true; + records = 0; + data = 0; + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { return false; } + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !TrailsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { return false; } + const int64_t storage = TrailsNodeMessageStorage( type_id, extent, length ); + if ( storage > 0 ) { data += storage; } // a type id this build cannot name commands none + records++; + } + int64_t root_extent = 0; + if ( !TrailsMessageExtent( r, vocabulary, index_bits, root_extent ) ) { complete = false; } + data += TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + return true; +} + +// TrailsLoadMeasure's MESSAGE overload: the exact region bytes ONE BATCH needs, +// which is one measurement, one allocation and one bounds check for however +// many bodies ride (§3.3, §6.5). It is a scan by the announced shapes and +// reads no field value. The answer is the data bytes plus the attribution, +// one node directory a body, and -1 for a wire it cannot size: no vocabulary, +// another form, or framing that gives out. +inline int64_t TrailsLoadMeasure( const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, int64_t * attribution_bytes = NULL ) +{ + TableReport ignored; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, &ignored ); + if ( bodies < 0 ) { return -1; } + int64_t data = 0, attribution = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + int64_t records = 0, body_data = 0; + bool complete = true; + if ( !TrailsMessageBodyStorage( br.r, vocabulary, records, body_data, complete ) ) { return -1; } + data += body_data; + attribution += ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( !complete ) { break; } // damage inside this body: the load delivers the ones before it + } + if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; } + return data + attribution; +} + +// TrailsLoadMessageBodyInto: one body of a batch into the region at `used`. Its +// chunk is the node DIRECTORY, then the records in wire order, then the root +// and the extent its maps take, so every offset a pass needs is known when +// the pass reaches it. PASS ONE fills the numbering from the framing and +// places every node; PASS TWO decodes each record's body into the storage it +// owns; the ROOT's own body decodes last, so every index it carries resolves +// against a numbering already known whole. +inline bool TrailsLoadMessageBodyInto( TableBitReader & r, const TableVocabulary & vocabulary, TableReport * out, uint8_t * region, int64_t region_bytes, int64_t & used, const Trails * & root_out ) +{ + // the node table opens the body, or the body has none + int64_t count = 0; + if ( !TableMessageNodeTableOpen( r, vocabulary, count ) ) { out->malformed = true; return false; } + const int64_t directory_bytes = ( count + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( used + directory_bytes > region_bytes ) { out->malformed = true; return false; } + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + used ); + used += directory_bytes; + const int64_t index_bits = TableBitsRequired( 0, count + 1 ); + TableNodeMap nodes; + nodes.base = region; + nodes.entries = directory; + nodes.count = count + 1; + nodes.good = false; + + // PASS ONE: the numbering from the framing, every node placed, no body read + const int64_t records_start = r.offset; + int32_t unknown_records = 0; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_id = 0; + int64_t extent = 0, length = 0; + if ( !TrailsMessageRecordScan( r, vocabulary, index_bits, type_id, extent, length ) ) { out->malformed = true; return false; } + const int64_t storage = TrailsNodeMessageStorage( type_id, extent, length ); + directory[k + 1].type_id = type_id; + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS INDEX, is + // counted once here and not once per pointer, and every reference + // to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + continue; + } + if ( used + storage > region_bytes ) { out->malformed = true; return false; } + directory[k + 1].offset = (uint64_t) used; + TrailsNodePlace( type_id, region + used, length ); + used += storage; + } + const int64_t fields_start = r.offset; + int64_t root_extent = 0; + { + TableBitReader walk = r; + if ( !TrailsMessageExtent( walk, vocabulary, index_bits, root_extent ) ) { out->malformed = true; return false; } + } + const int64_t root_bytes = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + if ( used + root_bytes > region_bytes ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) used; + directory[0].type_id = 0xb4578774a78fb150ull; + Trails * root = new ( region + used ) Trails; // lifetime only: LoadMessageBody's first act is TrailsReset + TrailsReset( *root ); + root_out = root; + TableExtentCarve root_carve; + root_carve.at = region + used + TableAlignUp64( (int64_t) sizeof( Trails ) ); + root_carve.left = root_extent; + used += root_bytes; + nodes.good = true; + out->unknown += unknown_records; + + // PASS TWO: each record's body into its own storage, in wire order + r.offset = records_start; + for ( int64_t k = 0; k < count; k++ ) + { + uint64_t type_ref = 0; + if ( !r.get( type_ref, vocabulary.ref_bits ) ) { out->malformed = true; return false; } + const uint64_t type_id = directory[k + 1].type_id; + if ( type_id == kTableBytesTypeId || type_id == kTableStringTypeId ) + { + uint64_t length = 0; + if ( !r.get( length, 32 ) || !r.align() || !r.has( (int64_t) length * 8 ) ) { out->malformed = true; return false; } + if ( directory[k + 1].offset != kTableNodeAbsent && length > 0 ) { memcpy( region + directory[k + 1].offset + kTableBlobHeader, r.buffer + r.offset / 8, (size_t) length ); } + r.offset += (int64_t) length * 8; + continue; + } + if ( directory[k + 1].offset == kTableNodeAbsent ) + { + if ( !TableMessageSkipBody( r, vocabulary, index_bits ) ) { out->malformed = true; return false; } + continue; + } + if ( !TrailsNodeMessageBody( type_id, r, vocabulary, out, nodes, index_bits, region + directory[k + 1].offset ) ) { return false; } + } + if ( r.offset != fields_start ) { out->malformed = true; return false; } // the two passes disagree about the table's extent + + // and the ROOT's own body last + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + return TrailsLoadMessageBody( r, vocabulary, out, nodes, index_bits, *root ); +} + +// TrailsLoadMessages: decode a BATCH into the caller's exact-sized region and +// write each body's root into `roots`. `count` is IN and OUT: the storage the +// caller has room for, then what it got. M above the capacity is a refusal +// by name with count holding the wire's M; damage inside body k delivers +// bodies 1 to k - 1 and count says k - 1 (§3.3). LOAD IS A SCAN: it follows +// no reference, so there is no depth cap and no visited set. NULL roots +// beyond count are not bodies. +inline bool TrailsLoadMessages( const Trails ** roots, int64_t * count, uint8_t * region, int64_t region_bytes, const TableVocabulary & vocabulary, const uint8_t * buffer, int64_t bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + if ( roots == NULL || count == NULL ) { out->malformed = true; return false; } + const int64_t capacity = *count; + *count = 0; + TableMessageBatchReader br; + const int64_t bodies = TableMessageBatchOpen( br, vocabulary, buffer, bytes, out ); + if ( bodies < 0 ) { return false; } + if ( bodies > capacity ) { *count = bodies; TableMessageRefuseBatch( out ); return false; } + if ( region == NULL || region_bytes < 0 || ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return false; } + memset( region, 0, (size_t) region_bytes ); + int64_t used = 0; + for ( int64_t b = 0; b < bodies; b++ ) + { + roots[b] = NULL; + if ( !TrailsLoadMessageBodyInto( br.r, vocabulary, out, region, region_bytes, used, roots[b] ) ) { *count = b; return false; } + br.remaining--; + } + *count = bodies; + return TableMessageBatchClose( br ); +} + +// TrailsLoadMeasure: the exact region bytes a wire buffer will need, and it is +// ONE SCAN — a record's type id gives its storage size, its length gives the +// next record — reading no field value at all, so the caller owns the +// allocation and can refuse a number it did not expect (§6.5). +// +// It reports the DATA bytes and the ATTRIBUTION bytes separately, because the +// attribution is the wire's numbering made resident (§6.3) and a caller may +// release it once Load returns. The answer is their sum. +inline int64_t TrailsLoadMeasure( const uint8_t * wire_file, int64_t wire_file_bytes, int64_t * attribution_bytes = NULL, TableRefuseReason * reason_out = NULL ) +{ + TableReport ignored; + TableIdTable ids_table; + int64_t body_bytes = 0; + // a FORM BYTE this build does not carry is refused by name (§3, §6.5); + // a trailer that cannot be read whole is damage and names no reason + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict == TableOpenRefused ) { if ( reason_out != NULL ) { *reason_out = unknown_form; } return -1; } + if ( verdict != TableOpenOk ) { return -1; } + // ANY BYTE BETWEEN THE ROOT'S TERMINATOR AND THE TABLE'S FIRST ENTRY + // IS MALFORMED (docs/SPEC-TABLES.md §3): the two ends of the file have + // met, nothing is decoded, and no region is sized from it. + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) { return -1; } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &ignored, &ids_table ); + TableRefuseReason reason = count_over_length; + int64_t root_extent = 0; + if ( !TrailsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { if ( reason_out != NULL ) { *reason_out = reason; } return -1; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + int64_t records = 0; + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = TrailsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { if ( reason_out != NULL ) { *reason_out = reason; } return -1; } // an N the record's framing cannot carry, or a blob past the cap (§2.8, §2.9, §3.1) + if ( storage > 0 ) { data += storage; } // a type id this build cannot name commands none + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( attribution_bytes != NULL ) { *attribution_bytes = attribution; } + return data + attribution; +} + +// TrailsLoad: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +inline const Trails * TrailsLoad( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Trails ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !TrailsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = TrailsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0xb4578774a78fb150ull; + Trails * root = new ( region ) Trails; // lifetime only: LoadBody's first act is TrailsReset + TrailsReset( *root ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = TrailsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + TrailsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + if ( nodes.good ) { out->unknown += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + TrailsNodeBody( type_id, sub, nodes, region + directory[k + 1].offset ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Trails ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + TrailsLoadBody( r, nodes, *root ); + return root; +} + +// TrailsLoadBuilder: the TOOL's path — the same tolerant decode into a fresh +// builder, so loaded data can be edited and locked again. The numbering is +// the same one; what differs is where a node lives and therefore what a +// resolved slot holds — an arena offset here, a self-relative delta there. +inline bool TrailsLoadBuilder( TrailsBuilder & builder, const uint8_t * wire_file, int64_t wire_file_bytes, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; } + return false; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return false; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + Trails * root = builder.GetRoot(); + if ( root == NULL ) { out->malformed = true; return false; } + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) { records++; } + } + // the AUTHORING side may allocate (§6.5), and this is the tool's path. + // It goes through the builder's own pair, like everything else the + // builder reaches, and the entries come back zeroed. + const TableAllocator allocator = builder.arena.allocator; + TableNodeDirEntry * directory = (TableNodeDirEntry *) allocator.alloc( allocator.context, ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ) ); + if ( directory == NULL ) { out->malformed = true; return false; } + directory[0].offset = (uint64_t) builder.root_ref.value; + directory[0].type_id = 0xb4578774a78fb150ull; + TableNodeMap nodes; + nodes.base = NULL; + nodes.entries = directory; + nodes.count = records + 1; + nodes.arena = true; // a resolved slot holds the node's ARENA OFFSET here + nodes.worker = &builder.main; // and a map's entries and a list's elements are the arena's, not a node extent's (§2.8, §2.9) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + uint32_t at = TrailsNodeAlloc( type_id, builder.main, length ); + if ( at == 0 ) + { + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + } + else + { + directory[k + 1].offset = (uint64_t) at; + } + directory[k + 1].type_id = type_id; + k++; + } + nodes.good = TableNodeScanWhole( scan ); + if ( nodes.good ) { out->unknown += unknown_records; } else { out->malformed = true; } + } + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + TrailsNodeBody( type_id, sub, nodes, TableArenaAt( builder.arena, (uint32_t) directory[k + 1].offset ) ); + } + k++; + } + } + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.worker = &builder.main; + nodes.carve = &root_carve; + bool ok = TrailsLoadBody( r, nodes, *root ); + // A COUNT ABOVE THE int32 CAP is this path's refusal (docs/SPEC-TABLES.md + // §2.9): the partial builder is the caller's to discard, and the report + // holds what it held when the count was met + ok = ok && !nodes.refused; + allocator.free( allocator.context, directory ); + return ok; +} + +template +inline int64_t TrailsStepsEntryMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + if ( value.key != 0 ) { bytes += TableLebBytes( ids.ref( 0x3dc94a19365b10ecull ) ) + 1 + 4; } // key + { + // value: a kind 14 array of kind 17 elements, INDEX order (§2.9) + TableListCursor cursor_value = TableListElements( ctx, value.value ); + if ( !cursor_value.ok ) { return -1; } // the slot and the head disagree + if ( cursor_value.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( cursor_value.count ) ); // the element kind byte and the count + for ( int32_t elem_i_value = 0; elem_i_value < cursor_value.count; elem_i_value++ ) + { + { + const Item * slot_pointee_value = ItemAt( ctx, cursor_value[elem_i_value] ); + uint64_t slot_index_value = 0; + if ( slot_pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee_value, slot_index_value ) ) { return -1; } + body_value += TableLebBytes( slot_index_value ); + } + } + bytes += TableLebBytes( ref_value ) + 1 + TableLebBytes( (uint64_t) ( body_value ) ) + ( body_value ); + } + } + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool TrailsStepsEntrySaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( value.key != 0 ) + { + w.putleb( ids.ref( 0x3dc94a19365b10ecull ) ); w.put8( 8 ); // key + w.put32( uint32_t( value.key ) ); + } + { + TableListCursor cursor_value = TableListElements( ctx, value.value ); // value + if ( !cursor_value.ok ) { return false; } + if ( cursor_value.count > 0 ) // an EMPTY list elides, the by-value rule (§3) + { + const uint64_t ref_value = ids.ref( 0x7ce4fd9430e80ceaull ); + int64_t body_value = 0; + body_value += 1 + TableLebBytes( (uint64_t) ( cursor_value.count ) ); // the element kind byte and the count + for ( int32_t elem_i_value = 0; elem_i_value < cursor_value.count; elem_i_value++ ) + { + { + const Item * slot_pointee_value = ItemAt( ctx, cursor_value[elem_i_value] ); + uint64_t slot_index_value = 0; + if ( slot_pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee_value, slot_index_value ) ) { return false; } + body_value += TableLebBytes( slot_index_value ); + } + } + w.putleb( ref_value ); w.put8( 14 ); w.putleb( (uint64_t) body_value ); // value + w.put8( 17 ); w.putleb( (uint64_t) ( cursor_value.count ) ); + for ( int32_t elem_i_value = 0; elem_i_value < cursor_value.count; elem_i_value++ ) + { + { + const Item * slot_pointee_value = ItemAt( ctx, cursor_value[elem_i_value] ); + uint64_t slot_index_value = 0; + if ( slot_pointee_value != NULL && !TableNumberingIndex( numbering, (const void *) slot_pointee_value, slot_index_value ) ) { return false; } + w.putleb( slot_index_value ); + } + } + } + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool TrailsStepsEntrySaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !TrailsStepsEntrySaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool TrailsStepsEntryLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, TrailsStepsEntry & value, TableRetain * retain, const TableRetainPath & path ) +{ + TrailsStepsEntryReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x3dc94a19365b10ecull: // key + { + if ( kind != 8 ) + { + if ( TableKindWidens( kind, 8 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + uint64_t widened_v = 0; + if ( !TableReadUnsignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = (uint32_t) widened_v; + value.key = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + uint32_t decoded_v = uint32_t( r.get32( ) ); + value.key = decoded_v; + break; + } + case 0x7ce4fd9430e80ceaull: // value + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + // A BODY TOO SHORT FOR ITS OWN HEADER is INERT (§4): the field keeps + // the value it has, no counter is raised, and the walk continues past L. + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + const bool counted_ok = r.getleb( count ); + if ( !counted_ok ) { r.report->malformed = true; } + // AN ELEMENT KIND THAT DISAGREES with the reader's declaration is §3's + // element-kind rule: the field reads EMPTY and one kind_mismatch counts + else if ( elem_kind != 17 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + else + { + // THE COUNT IS THE DATA'S (§2.9): there is no bound, so clamped + // cannot fire on it. A count above the int32 storage cap is the + // fill's refusal, and it moves no counter. + TableListFill fill = TableListFillBegin( nodes, value.value, count ); + if ( fill.refused ) { nodes.refused = true; return false; } + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + // elements are BOUNDED by the field body: a count the length cannot + // cover keeps the decoded prefix, flags malformed, and the parent + // continues at the next field + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + for ( uint64_t i = 0; i < count; i++ ) + { + TableRef * slot = TableListFillNext( fill ); + if ( slot == NULL ) { r.report->malformed = true; break; } // the arena could not carve + bool landed = false; + do + { + { + uint64_t node_index_value = 0; + if ( !sub.getleb( node_index_value ) ) { r.report->malformed = true; break; } + TableNodeResolve( nodes, ( *slot ), node_index_value, 0x52cfa1d198476806ull, r.report ); // *Item + } + landed = true; + } while ( 0 ); + if ( !landed ) { TableListFillDrop( fill ); break; } // the element's own framing gave out before it decoded + } + TableListFillEnd( fill ); + } + } + r.offset = body_end; // excess bytes and slack skip via the length + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +template +inline int64_t TrailsMeasureBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + int64_t bytes = 1; // the ZERO REFERENCE that ends the body + { + // steps: a kind 14 array of kind 13 elements, ASCENDING (§2.8) + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); + if ( !order_steps.ok ) { return -1; } // the sort could not run + if ( order_steps.count > 0 ) + { + const uint64_t ref_steps = ids.ref( 0x124250ad5a5b6d14ull ); + int64_t body_steps = 1 + TableLebBytes( (uint64_t) order_steps.count ); // the element kind byte and the count + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_steps = TrailsStepsEntryMeasureBodyRetain( ctx, numbering, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return -1; } + body_steps += TableLebBytes( (uint64_t) ( elem_steps ) ) + ( elem_steps ); // BUT THE ENTRY ALWAYS RIDES: identity here is the key + } + bytes += TableLebBytes( ref_steps ) + 1 + TableLebBytes( (uint64_t) ( body_steps ) ) + ( body_steps ); + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) { bytes += TableLebBytes( ids.ref( 0xbf82010f6f71eae9ull ) ) + 1 + 4; } // after + bytes += TableRetainTailMeasure( retain, ids, path ); + return bytes; +} + +template +inline bool TrailsSaveBodyFieldsRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + { + TableMapCursor order_steps = TableMapOrder( ctx, value.steps ); // steps + if ( !order_steps.ok ) { return false; } + if ( order_steps.count > 0 ) // an EMPTY map elides, the by-value rule (§3) + { + const uint64_t ref_steps = ids.ref( 0x124250ad5a5b6d14ull ); + int64_t body_steps = 1 + TableLebBytes( (uint64_t) order_steps.count ); + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_steps = TrailsStepsEntryMeasureBodyRetain( ctx, numbering, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_steps < 0 ) { TableMapRelease( order_steps ); return false; } + body_steps += TableLebBytes( (uint64_t) ( elem_steps ) ) + ( elem_steps ); + } + w.putleb( ref_steps ); w.put8( 14 ); w.putleb( (uint64_t) body_steps ); + w.put8( 13 ); w.putleb( (uint64_t) order_steps.count ); + for ( int32_t i = 0; i < order_steps.count; i++ ) + { + const int64_t elem_len_steps = TrailsStepsEntryMeasureBodyRetain( ctx, numbering, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ); + if ( elem_len_steps < 0 ) { TableMapRelease( order_steps ); return false; } + w.putleb( (uint64_t) elem_len_steps ); + if ( !TrailsStepsEntrySaveBodyRetain( ctx, numbering, w, ids, *order_steps[i], retain, TableRetainStepInto( path, 0, (uint32_t) ( i ) ) ) ) { TableMapRelease( order_steps ); return false; } + } + } + TableMapRelease( order_steps ); + } + if ( value.after != 0 ) + { + w.putleb( ids.ref( 0xbf82010f6f71eae9ull ) ); w.put8( 4 ); // after + w.put32( uint32_t( value.after ) ); + } + if ( !TableRetainTailSave( retain, ids, w, path ) ) { return false; } + return !w.overflow; +} + +template +inline bool TrailsSaveBodyRetain( const Ctx & ctx, const TableNumbering & numbering, TableWriter & w, TableRetainIds & ids, const Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + if ( !TrailsSaveBodyFieldsRetain( ctx, numbering, w, ids, value, retain, path ) ) { return false; } + w.put8( 0 ); // the ZERO REFERENCE that ends the body + return !w.overflow; +} + +inline bool TrailsLoadBodyRetain( TableReader & r, const TableNodeMap & nodes, Trails & value, TableRetain * retain, const TableRetainPath & path ) +{ + TrailsReset( value ); // prefill declared defaults in place, then overlay + // A RETAINED RECORD DIES WITH THE BODY OCCURRENCE THAT CARRIED IT + // (docs/SPEC-TABLES.md §6.6): this body is being established, so + // whatever an earlier occurrence of it left is discarded before the + // winning one is read. The discard moves neither counter. + TableRetainDiscardBody( retain, path ); + for ( ;; ) + { + uint64_t field_ref = 0; + if ( !r.getleb( field_ref ) ) { r.report->malformed = true; return false; } + if ( field_ref == 0 ) return true; // the body ENDS AT ITS OWN ZERO REFERENCE + if ( r.ids == NULL || field_ref > (uint64_t) r.ids->count ) { r.report->malformed = true; return false; } // a reference ABOVE the entry count + const uint64_t field_id = r.ids->at( field_ref ); + if ( !r.has( 1 ) ) { r.report->malformed = true; return false; } + uint8_t kind = r.get8(); + if ( ( field_id == kTableNodeTableFieldId && r.nested ) || field_id == kTableBuildVersionFieldId || field_id == kTableMessageVocabularyFieldId ) + { + // A RESERVED ID IN ANY BODY BUT THE ONE WHOSE TRANSPORT IT IS, + // IS MALFORMED (docs/SPEC-TABLES.md §3.1, §3.3). The node + // table's is the ROOT body's alone, on the numbering's own + // rule — a second numbering cannot exist — and the BUILD + // VERSION's rides in the announcement and nowhere else. That + // body stops and the parent reads on past its L. + r.report->malformed = true; + return false; + } + switch ( field_id ) + { + case 0x124250ad5a5b6d14ull: // steps + { + if ( kind != 14 ) + { + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + uint64_t body_len = 0; + if ( !r.getleb( body_len ) || !r.room( body_len ) ) { r.report->malformed = true; return false; } + int64_t body_end = r.offset + (int64_t) body_len; + if ( body_len >= 2 ) + { + uint8_t elem_kind = r.get8(); + uint64_t count = 0; + if ( !r.getleb( count ) ) { r.report->malformed = true; r.offset = body_end; break; } + // A MAP HEADER WHOSE ELEMENT KIND IS NOT 13 is the ordinary array + // kind mismatch of §4, and nothing about a map is special-cased + if ( elem_kind != 13 ) { r.report->kind_mismatch++; r.offset = body_end; break; } + // THE READ COMMITS TO REPLACE HERE (docs/SPEC-TABLES.md §6.6): the + // records under this field go with the value it is about to lose. + TableRetainDiscardField( retain, path, 0 ); + TableMapFill fill = TableMapFillBegin( nodes, value.steps, (uint32_t) count ); + if ( !fill.ok ) { r.report->malformed = true; r.offset = body_end; break; } + TableReader sub( r.buffer + r.offset, body_end - r.offset, r.report, r.ids ); + uint32_t last_key = 0; + bool landed = false; + bool map_widened = false; + for ( uint64_t i = 0; i < count; i++ ) + { + uint64_t elem_len = 0; + if ( !sub.getleb( elem_len ) || !sub.room( elem_len ) ) { r.report->malformed = true; break; } + const uint8_t * elem_body = sub.buffer + sub.offset; + sub.offset += (int64_t) elem_len; + TrailsStepsEntryKeyRead read = TrailsStepsEntryReadKey( elem_body, (int64_t) elem_len, r.ids ); + // A KEY KIND THE DECLARATION WIDENS: the map counts ONE widened (§2.8, §4) + if ( read.widened && !map_widened ) { map_widened = true; r.report->widened++; } + // THE KEY KIND IS CHECKED FIRST: a key read under another kind + // desynchronizes the rest of the scan, and the honest answer to a + // body whose key is not this reader's kind is the KIND, not the + // framing damage that follows from it. + if ( read.kind_bad ) + { + // A MAP WITH HALF ITS KEYS IS NOT A MAP (§2.8): the map resets + // to EMPTY, ONE kind_mismatch is counted for it, and the rest + // is skipped. Events counted inside earlier entries stand. + r.report->kind_mismatch++; + TableMapFillReset( fill ); + break; + } + if ( read.malformed ) { r.report->malformed = true; break; } + if ( read.over ) { r.report->clamped++; continue; } // skipped by its L, one count per entry + const int order = landed ? TableKeyOrder( (uint64_t) last_key, (uint64_t) read.key ) : -1; + if ( order > 0 ) + { + // DESCENDING: not a body any conforming writer produced. The map + // keeps the ascending prefix it has, the rest skips by the map's + // L, and the PARENT reads on past the field's length (§4). + r.report->malformed = true; + break; + } + TrailsStepsEntry * slot = NULL; + if ( order == 0 ) + { + // EQUAL: a DUPLICATE. The slot that entry took is reset to the + // entry's defaults by the decode below, so LAST WINS WHOLE and an + // elided field of the repeat reads as its default. The map's + // count excludes it. + slot = TableMapFillLast( fill ); + r.report->duplicate++; + } + else + { + slot = TableMapFillNext( fill ); // ASCENDING: the next slot + } + if ( slot == NULL ) { r.report->malformed = true; break; } + { + TableReader elem( elem_body, (int64_t) elem_len, r.report, r.ids ); + TrailsStepsEntryLoadBodyRetain( elem, nodes, *slot, retain, TableRetainStepInto( path, 0, (uint32_t) ( fill.map->count - 1 ) ) ); + } + last_key = read.key; // the WIRE keys of the entries that LAND + landed = true; + } + TableMapFillEnd( fill ); + } + r.offset = body_end; // the remaining entries skip by the map's L + break; + } + case 0xbf82010f6f71eae9ull: // after + { + if ( kind != 4 ) + { + if ( TableKindWidens( kind, 4 ) ) + { + // WIDENED (§4): a kind that grew since the writer decodes + // exactly at its own width, the value lands, one widened counts + int64_t widened_v = 0; + if ( !TableReadSignedAt( r, kind, widened_v ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = (int32_t) widened_v; + value.after = decoded_v; + r.report->widened++; + break; + } + // AT A POSITION THE READER DOES NAME, a field under + // kind 31 or kind 32 takes this same rule and no other (§3) + r.report->kind_mismatch++; + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + if ( !r.has( 4 ) ) { r.report->malformed = true; return false; } + int32_t decoded_v = int32_t( r.get32( ) ); + value.after = decoded_v; + break; + } + case 0xffffffffffffffffull: + { + if ( !r.skip( kind ) ) { r.report->malformed = true; return false; } + break; + } + default: + { + r.report->unknown++; + if ( !TableRetainCapture( retain, r, path, field_id, kind ) ) { r.report->malformed = true; return false; } + break; + } + } + } +} + +// TrailsNodeBodyRetain: PASS TWO's half — decode one record's body into the storage it +// already owns. +// EACH NODE BODY IS A PATH ROOT of its own (docs/SPEC-TABLES.md §6.6): the +// index is the region directory's, which Load fills from the wire's framing +// and nothing afterwards renumbers. +inline void TrailsNodeBodyRetain( uint64_t type_id, TableReader & r, const TableNodeMap & nodes, uint8_t * at, TableRetain * retain, uint32_t node ) +{ + // the node's own EXTENT, where its lists' and maps' arrays are carved + // from, PRE-ORDER as the bodies decode (docs/SPEC-TABLES.md §2.8, §2.9). + // The tool's path carries a worker instead: there the arrays are the + // arena's. + TableExtentCarve carve; + carve.worker = nodes.worker; + if ( carve.worker == NULL ) + { + TableRefuseReason reason = count_over_length; // pass one already refused what this could refuse + const int64_t storage = TrailsNodeStorage( type_id, r.size, reason ); + const int64_t record = storage > 0 ? TrailsNodeRecordBytes( type_id ) : 0; + carve.at = at + record; + carve.left = storage > record ? storage - record : 0; + } + nodes.carve = &carve; + (void) nodes; // every node this root can name is a FIXED table + switch ( type_id ) + { + case 0x52cfa1d198476806ull: ItemLoadBodyRetain( r, *(Item *) at, retain, TableRetainPathRoot( (const void *) at, node ) ); break; // Item + default: break; + } + nodes.carve = NULL; // the cursor is ONE node's, and this node's body is done +} + +// TrailsLoadRetain: decode the tolerant wire into the caller's exact-sized region and +// return the root. LOAD IS A SCAN, and that is the whole of its bound: it +// follows no reference, so there is no depth cap, no visited set and no +// ordering rule on the indices. Partial results are kept, as everywhere on +// this wire — the report says what happened. NULL means the CALLER's buffer +// was wrong. +// UNDER RETENTION it also fills the caller's two stores with the fields +// this build cannot name, and the report carries what it could not keep +// (docs/SPEC-TABLES.md §6.6). It is Load's own path and nothing else: the +// reader's data is exactly what it would have been with retention off. +inline const Trails * TrailsLoadRetain( uint8_t * region, int64_t region_bytes, const uint8_t * wire_file, int64_t wire_file_bytes, TableRetain * retain, TableReport * report ) +{ + TableReport ignored; + TableReport * out = report != NULL ? report : &ignored; + // THE FORM BYTE IS READ FIRST, then the trailer, and only then a body: + // a file that is both a newer form and damaged is a REFUSAL and never + // damage (docs/SPEC-TABLES.md §3). + TableIdTable ids_table; + int64_t body_bytes = 0; + const TableOpenVerdict verdict = TableOpen( wire_file, wire_file_bytes, ids_table, body_bytes ); + if ( verdict != TableOpenOk ) + { + if ( verdict == TableOpenDamaged ) { out->malformed = true; } else { out->refused = true; if ( wire_file_bytes > 0 && wire_file[0] == kTableWireMessageForm ) { out->reason = message_form_as_file; } else { out->reason = newer_form; } } + return NULL; + } + if ( TableBodyEndsEarly( wire_file + 1, body_bytes, ids_table ) ) + { + out->malformed = true; // a byte no field claims, before the table (§3) + return NULL; + } + const uint8_t * const wire = wire_file + 1; + const int64_t wire_bytes = body_bytes; + if ( region == NULL || region_bytes < (int64_t) sizeof( Trails ) ) { out->malformed = true; return NULL; } + if ( ( ( (uintptr_t) region ) & ( kTableAlign - 1 ) ) != 0 ) { out->malformed = true; return NULL; } + memset( region, 0, (size_t) region_bytes ); + uint64_t type_id = 0; + const uint8_t * body = NULL; + int64_t length = 0; + + // the record count and the data bytes, from the FRAMING alone + TableRefuseReason reason = count_over_length; // LoadMeasure is where a caller reads it; a Load past a refusal is malformed + int64_t root_extent = 0; + if ( !TrailsWireExtent( wire, wire_bytes, root_extent, &ids_table, reason ) ) { out->malformed = true; return NULL; } + int64_t data = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + int64_t records = 0; + { + TableReport counting; + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, &counting, &ids_table ); + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + records++; + int64_t storage = TrailsNodeStorage( type_id, length, reason ); + if ( storage == kTableNodeRefused ) { out->malformed = true; return NULL; } + if ( storage > 0 ) { data += storage; } + } + } + int64_t attribution = ( records + 1 ) * (int64_t) sizeof( TableNodeDirEntry ); + if ( data + attribution > region_bytes ) { out->malformed = true; return NULL; } + + TableNodeMap nodes; + nodes.base = region; + nodes.entries = (const TableNodeDirEntry *) ( region + data ); + nodes.count = records + 1; + TableNodeDirEntry * directory = (TableNodeDirEntry *) ( region + data ); + directory[0].offset = 0; // position 0 is the ROOT, at offset 0 (§6.3) + directory[0].type_id = 0xb4578774a78fb150ull; + Trails * root = new ( region ) Trails; // lifetime only: LoadBody's first act is TrailsReset + TrailsReset( *root ); + + // LoadRetain RESETS BOTH STORES and writes into neither id list: a + // retained record carries its field's identity in the record itself, + // with every reference resolved (docs/SPEC-TABLES.md §6.6). The buffer + // belongs to this region from here on. + TableRetainReset( retain, nodes, region ); + + // PASS ONE: fill the numbering from the framing, so that an index + // resolves whichever way it points. It reads no body. + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t used = TableAlignUp64( TableAlignUp64( (int64_t) sizeof( Trails ) ) + root_extent ); + int64_t k = 0; + int32_t unknown_records = 0; // counted once the scan is known whole + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + int64_t storage = TrailsNodeStorage( type_id, length, reason ); + if ( storage <= 0 ) + { + // a record whose type id this build cannot name KEEPS ITS + // INDEX, is counted once here and not once per pointer, and + // every reference to it reads null (§3.1) + unknown_records++; + directory[k + 1].offset = kTableNodeAbsent; + directory[k + 1].type_id = type_id; + } + else + { + directory[k + 1].offset = (uint64_t) used; + directory[k + 1].type_id = type_id; + TrailsNodePlace( type_id, region + used, length ); + used += storage; + } + k++; + } + nodes.good = TableNodeScanWhole( scan ); + // the table is whole or it is nothing: a scan that failed counts + // malformed and NOT the unknowns it met on the way, because the + // numbering they belonged to does not exist (§3.1) + // A NODE RECORD whose type id this reader cannot name is one of the + // SIX EXCLUDED CLASSES (§6.6): it is a whole node, and putting one + // back means renumbering a graph the writer numbers from its own edges. + if ( nodes.good ) { out->unknown += unknown_records; out->retain_lost += unknown_records; } else { out->malformed = true; } + } + + // PASS TWO: decode each body into its own storage. A forward index + // resolves without scratch, because pass one already placed every node. + if ( nodes.good ) + { + TableNodeScan scan = TableNodeScanBegin( wire, wire_bytes, out, &ids_table ); + int64_t k = 0; + while ( TableNodeScanNext( scan, type_id, body, length ) ) + { + if ( directory[k + 1].offset != kTableNodeAbsent ) + { + TableReader sub( body, length, out, &ids_table ); + TrailsNodeBodyRetain( type_id, sub, nodes, region + directory[k + 1].offset, retain, (uint32_t) ( k + 2 ) ); + } + k++; + } + } + + // and the ROOT's own body last, so every index it carries resolves + // against a numbering already known good or already known bad + TableReader r( wire, wire_bytes, out, &ids_table ); + r.nested = false; // the ROOT body, the one that carries the node table + TableExtentCarve root_carve; + root_carve.at = region + TableAlignUp64( (int64_t) sizeof( Trails ) ); + root_carve.left = root_extent; + nodes.carve = &root_carve; // the ROOT's extent is its own, like every node's + TrailsLoadBodyRetain( r, nodes, *root, retain, TableRetainPathRoot( (const void *) root, 1 ) ); + return root; +} + +// TrailsMeasureRetain and TrailsSaveRetain: the pair, with the retained tail in +// every body it belongs to (docs/SPEC-TABLES.md §6.6). They drop the same +// records under the same walk, so Measure's answer is the size the save +// writes even where a record could not be placed. +template +inline int64_t TrailsMeasureWireRetain( const Ctx & ctx, const Trails & root, TableRetain * retain, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + int64_t bytes = -1; + auto retain_measure = []( const Ctx & c, const TableNumbering & nn, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> int64_t + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemMeasureBodyRetain( ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return -1; + }; + if ( TrailsNumberFrom( ctx, numbering, root ) ) + { + TableRetainIds ids( retain ); + if ( retain != NULL ) { retain->id_used = 0; } // one walk fills the list, and the save's own walk refills it + bytes = TrailsMeasureBodyRetain( ctx, numbering, ids, root, retain, TableRetainPathRoot( (const void *) &root, 1 ) ); + if ( bytes >= 0 ) + { + const int64_t table = TableNodeTableMeasureRetain( ctx, ids, numbering, retain, retain_measure ); + bytes = table < 0 || ids.overflow ? -1 : 1 + bytes + table + TableRetainIdsBytes( ids ); + } + } + TableNumberingShutdown( numbering ); + return bytes; +} + +template +inline int64_t TrailsSaveWireRetain( const Ctx & ctx, const Trails & root, TableRetain * retain, uint8_t * buffer, int64_t capacity, TableReport * report ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, TableDefaultAllocator() ); + auto retain_measure = []( const Ctx & c, const TableNumbering & nn, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> int64_t + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemMeasureBodyRetain( ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return -1; + }; + auto retain_save = []( const Ctx & c, const TableNumbering & nn, TableWriter & ww, TableRetainIds & ii, uint64_t type_id, const void * node, TableRetain * rt ) -> bool + { + const TableRetainPath at = TableRetainPathRoot( node, 0 ); + (void) c; (void) nn; (void) ww; (void) ii; (void) rt; (void) at; + switch ( type_id ) + { + case 0x52cfa1d198476806ull: return ItemSaveBodyRetain( ww, ii, *(const Item *) node, rt, at ); // Item + default: break; + } + return false; + }; + if ( !TrailsNumberFrom( ctx, numbering, root ) ) { TableNumberingShutdown( numbering ); return -1; } + TableWriter w( buffer, capacity ); + TableRetainIds ids( retain ); + if ( retain != NULL ) { retain->id_used = 0; } + TableRetainClearPlaced( retain ); + w.put8( kTableWireForm ); // the FORM BYTE is the whole header (§3) + // the root's own fields, then the RETAINED TAIL, then the node table's + // field: a retained field is one of the root's own values, and the tail + // is pinned before the large and damage-prone part (§6.6, §3.1) + bool ok = TrailsSaveBodyFieldsRetain( ctx, numbering, w, ids, root, retain, TableRetainPathRoot( (const void *) &root, 1 ) ) && + TableNodeTableSaveRetain( ctx, w, ids, numbering, retain, retain_measure, retain_save ); + TableNumberingShutdown( numbering ); + if ( !ok || ids.overflow ) { return -1; } + w.put8( 0 ); // the ZERO REFERENCE that ends the root body + TableRetainIdsWrite( w, ids ); + if ( w.overflow ) { return -1; } // the caller's buffer was too small + // THE SAVE'S OWN SHARE OF retain_lost, read after the save (§6.6): every + // record the walk did not place, counted once. + TableRetainCountLost( retain, report ); + return w.offset; +} + +inline int64_t TrailsMeasureRetain( const Trails * root, TableRetain * retain, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return TrailsMeasureWireRetain( ctx, *root, retain, allocator ); +} + +// SaveRetain REFUSES A NULL REPORT and returns -1 (docs/SPEC-TABLES.md +// §6.6): the save is the only place a caller learns that a record was +// dropped, so the report is required here where it is optional everywhere +// else. A surface that let a caller retain, save and never find out would +// be a promise it could not check. +inline int64_t TrailsSaveRetain( const Trails * root, TableRetain * retain, uint8_t * buffer, int64_t capacity, TableReport * report ) +{ + if ( root == NULL || report == NULL ) { return -1; } + TableRegionCtx ctx; + return TrailsSaveWireRetain( ctx, *root, retain, buffer, capacity, report ); +} + +// TrailsSaveRetainMessages: RETENTION WRITING FORM 2 IS REFUSED BY NAME +// (docs/SPEC-TABLES.md §3.3). It is a MISUSE refusal on §6.6's own +// precedent and never a silent drop, and the two answers are named: a +// caller that must carry unknowns across a rewrite writes the FILE form, +// which carries its own table and takes §6.6 unchanged, and a RELAY +// forwards the sending peer's announcement and its batch bytes verbatim. +template +inline int64_t TrailsSaveRetainMessages( Args &&... ) +{ + static_assert( sizeof...( Args ) == (size_t) -1, + "Trails: a form 2 writer names entries through slots of a vocabulary the compiler settled, and a retained id is one this build's closure does not contain, so it has neither a slot nor an announced shape. Retention writing the MESSAGE form is refused by name (docs/SPEC-TABLES.md §3.3). Write the FILE form, which carries its own table and takes §6.6 unchanged, or relay the sender's announcement and batch bytes verbatim." ); + return -1; +} + +// ---- the cooked form: point at a cook (docs/SPEC-TABLES.md §7) ---- + +// TrailsOpen: match the header and POINT. On a match the bytes ARE what this +// build wrote, in this build's layout and this build's byte order, so there +// is nothing to validate and nothing to fix up and the root comes back as it +// lies. On ANY refusal it returns NULL and NAMES the refusal in the caller's +// TableRefuseReason, the first failing clause in §7's order (a wrong build +// version is a re-cook, a foreign order a cross-endian cook, a truncated +// file a bad download, an unaligned base the caller's own buffer), and the +// caller falls back to a wire load, which is the path that carries every +// version. The reason is written on the refusal path only; a caller that +// passes nothing gets the null alone. +// +// It is O(1) IN THE FILE'S SIZE — the header and nothing per node — so a one +// megabyte cook and a one gigabyte cook open in the same time, and a mapped +// file's pages are touched only as they are used. That is a property of +// touching nothing at open rather than a separate mechanism. +// +// A REFERENCE INSIDE THE REGION IS DEREFERENCED THROUGH TrailsAt: the slot holds +// the signed self-relative byte delta of §6.3, so a deref is one add and +// needs no base pointer, a whole region relocates by plain memcpy, and a +// delta of zero is null. +// +// There is ONE entry point and no tolerant twin: a build either wrote this +// file or it did not, and the build version is what says which. Validating a +// file whose provenance a person doubts is schema cook-check, offline, +// over the ATTRIBUTION part beside the data — a person's decision, never a +// parameter on a load. +inline const Trails * TrailsOpen( const void * bytes, uint64_t length, TableRefuseReason * reason = NULL ) +{ + return (const Trails *) TableCookOpen( bytes, length, (uint64_t) sizeof( Trails ), (uint64_t) alignof( Trails ), reason ); +} + +// ---- the cooked form: WRITE a cook (docs/SPEC-TABLES.md §7.6) ---- +// +// The bytes are `schema cook`'s, and the tool stays the reference: the two +// writers are held to one file, byte for byte, in both byte orders. A cook is +// content-addressed by (asset hash, build version), so two writers of one +// instance produce ONE artifact or the pair means nothing. + +template inline bool TrailsStepsEntryCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const TrailsStepsEntry & value, TableByteOrder order ); +template inline bool TrailsCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Trails & value, TableByteOrder order ); + +template inline bool TrailsStepsEntryCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const TrailsStepsEntry & value, TableByteOrder order ) +{ + (void) ctx; (void) region; // no reference resolves in this body: a list's and a map's slots are the extent writer's, and the class was decided elsewhere in the closure + table_cook_put( at + 0, (uint64_t) value.key, 4, order ); + table_cook_put( at + 8, 0, 8, order ); // value: the array's delta, filled by the extent writer + table_cook_put( at + 16, 0, 4, order ); // and its count + return true; +} + +template inline bool TrailsCookBody( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Trails & value, TableByteOrder order ) +{ + (void) ctx; (void) region; // no reference resolves in this body: a list's and a map's slots are the extent writer's, and the class was decided elsewhere in the closure + table_cook_put( at + 0, 0, 8, order ); // steps: the array's delta, filled by the extent writer + table_cook_put( at + 8, 0, 4, order ); // and its count + table_cook_put( at + 16, (uint64_t) value.after, 4, order ); + return true; +} + +template inline bool TrailsStepsEntryCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const TrailsStepsEntry & value, TableByteOrder order ); +template inline bool TrailsCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const Trails & value, TableByteOrder order ); + +// TrailsStepsEntryCookExtent: TrailsStepsEntry's arrays into the node's extent, PRE-ORDER, a map's entries +// in ASCENDING key order and a list's elements in INDEX order, each through its +// own cook writer (§2.8, §2.9, §7.6). +template inline bool TrailsStepsEntryCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const TrailsStepsEntry & value, TableByteOrder order ) +{ + { // value: an unbounded array + TableListCursor cursor = TableListElements( ctx, value.value ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( TableRef ) + uint8_t * array = extent + at; + at += (int64_t) cursor.count * 8; // the whole array FIRST + // the SIXTEEN BYTES of the slot: the self-relative delta, then the count + table_cook_put( record + 8, cursor.count > 0 ? (uint64_t) (int64_t) ( array - ( record + 8 ) ) : 0, 8, order ); + table_cook_put( record + 16, (uint64_t) (uint32_t) cursor.count, 4, order ); + for ( int32_t i = 0; i < cursor.count; i++ ) // INDEX order, live elements only + { + if ( !table_cook_ref( region, array + i * 8, (const void *) ItemAt( ctx, cursor[i] ), order ) ) { return false; } + } + } + return true; +} + +// TrailsCookExtent: Trails's arrays into the node's extent, PRE-ORDER, a map's entries +// in ASCENDING key order and a list's elements in INDEX order, each through its +// own cook writer (§2.8, §2.9, §7.6). +template inline bool TrailsCookExtent( const Ctx & ctx, const TableCookRegion & region, uint8_t * extent, int64_t & at, uint8_t * record, const Trails & value, TableByteOrder order ) +{ + (void) region; // a table element's and an entry's references resolve through their own bodies + { // steps + TableMapCursor cursor = TableMapOrder( ctx, value.steps ); + if ( !cursor.ok ) { return false; } + at = ( at + 7 ) & ~(int64_t) 7; // at alignof( TrailsStepsEntry ) + uint8_t * array = extent + at; + at += (int64_t) cursor.count * 24; // the whole array FIRST + // the SIXTEEN BYTES of the slot: the self-relative delta, then the count + table_cook_put( record + 0, cursor.count > 0 ? (uint64_t) (int64_t) ( array - ( record + 0 ) ) : 0, 8, order ); + table_cook_put( record + 8, (uint64_t) (uint32_t) cursor.count, 4, order ); + for ( int32_t i = 0; i < cursor.count; i++ ) + { + if ( !TrailsStepsEntryCookBody( ctx, region, array + i * 24, *cursor[i], order ) ) { return false; } + } + for ( int32_t i = 0; i < cursor.count; i++ ) // then, entry by entry in key order + { + if ( !TrailsStepsEntryCookExtent( ctx, region, extent, at, array + i * 24, *cursor[i], order ) ) { TableMapRelease( cursor ); return false; } + } + TableMapRelease( cursor ); + } + return true; +} + +// TrailsStepsEntryCookNode: one node, the record, then the extent its lists and maps take (§2.8, §2.9). +template inline bool TrailsStepsEntryCookNode( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const TrailsStepsEntry & value, TableByteOrder order ) +{ + if ( !TrailsStepsEntryCookBody( ctx, region, at, value, order ) ) { return false; } + int64_t extent_at = 0; + if ( !TrailsStepsEntryCookExtent( ctx, region, at + 24, extent_at, at, value, order ) ) { return false; } + return extent_at == TrailsStepsEntryExtent( ctx, value ); // the extent written is the extent measured, or no header is written +} + +// TrailsCookNode: one node, the record, then the extent its lists and maps take (§2.8, §2.9). +template inline bool TrailsCookNode( const Ctx & ctx, const TableCookRegion & region, uint8_t * at, const Trails & value, TableByteOrder order ) +{ + if ( !TrailsCookBody( ctx, region, at, value, order ) ) { return false; } + int64_t extent_at = 0; + if ( !TrailsCookExtent( ctx, region, at + 24, extent_at, at, value, order ) ) { return false; } + return extent_at == TrailsExtent( ctx, value ); // the extent written is the extent measured, or no header is written +} + +// TrailsCookLayout: the tool's own Layout (docs/SPEC-TABLES.md §7.2) over one +// numbering — the root at zero, then every node in index order at +// align_up( offset, alignof ) for its OWN type, no slack between them, the +// data length rounded to the greatest alignment among them and never below +// eight. The offsets go into the region's table when it has one, and are only +// summed when it does not (a measure). A type id the numbering carries that +// this root cannot name is the two walks disagreeing, and it is refused. +// A NODE'S SIZE DEPENDS ON ITS VALUE where a list or a map rides in its extent +// (docs/SPEC-TABLES.md §2.8), so the layout takes the resolution context +// the numbering walked and reads the same arrays that walk read. +template +inline bool TrailsCookLayout( const Ctx & ctx, const Trails & root, const TableNumbering & numbering, TableCookRegion & region ) +{ + region.numbering = &numbering; + region.count = numbering.count + 1; + const int64_t root_extent = TrailsExtent( ctx, root ); + if ( root_extent < 0 ) { return false; } + int64_t offset = 24 + root_extent; // the root at zero, its extent behind it + int64_t align = 8; + if ( region.offsets != NULL ) { region.offsets[0] = 0; } + for ( int64_t k = 0; k < numbering.count; k++ ) + { + int64_t size = 0; + int64_t node_align = 0; + switch ( numbering.entries[k].type_id ) + { + case 0x52cfa1d198476806ull: size = 4; node_align = 4; break; // Item + default: return false; + } + offset = ( offset + node_align - 1 ) & ~( node_align - 1 ); + if ( region.offsets != NULL ) { region.offsets[k + 1] = offset; } + offset += size; + if ( node_align > align ) { align = node_align; } + } + region.bytes = ( offset + align - 1 ) & ~( align - 1 ); + region.align = align; + return true; +} + +// TrailsCookMeasureFrom: the whole cooked file's bytes for one graph — the header, +// the data part and the attribution part (§7.1). IT DEPENDS ON THE VALUE, +// because the answer is the numbering: the depth-first walk of §3.1 is run +// here and run again by the write, and neither carries the other's (§7.6). A +// data cycle is refused by the walk and answers -1. +template +inline int64_t TrailsCookMeasureFrom( const Ctx & ctx, const Trails & root, TableAllocator allocator ) +{ + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + TableCookRegion region; + int64_t bytes = -1; + if ( TrailsNumberFrom( ctx, numbering, root ) && TrailsCookLayout( ctx, root, numbering, region ) ) + { + const int64_t data_offset = ( kTableCookHeaderBytes + region.align - 1 ) & ~( region.align - 1 ); + bytes = data_offset + region.bytes + region.count * (int64_t) sizeof( TableNodeDirEntry ); + } + TableNumberingShutdown( numbering ); + return bytes; +} + +// TrailsCookFrom: write one cooked file of a pointered graph, in the byte order +// the caller names. The bytes are `schema cook`'s, byte for byte (§7.6). +// +// THE CALLER OWNS THE OUTPUT and nothing is allocated toward it. What is +// allocated is the numbering — the identity map, the entry array and one +// offset per node — through the pair handed in, and released before this +// returns (§6.5, §13.9). A capacity short of the measure writes nothing. +// +// THE HEADER IS WRITTEN LAST. A reference the numbering did not carry is +// found while a body is being written, and a write that refuses there has +// already put bytes in the buffer; with no magic ahead of them, no Open can +// mistake them for a cook. +template +inline bool TrailsCookFrom( const Ctx & ctx, const Trails & root, void * out, uint64_t capacity, TableByteOrder order, TableAllocator allocator ) +{ + if ( out == NULL ) { return false; } + TableNumbering numbering; + TableNumberingInit( numbering, allocator ); + TableCookRegion region; + bool ok = TrailsNumberFrom( ctx, numbering, root ); + if ( ok ) + { + region.offsets = (int64_t *) allocator.alloc( allocator.context, ( numbering.count + 1 ) * (int64_t) sizeof( int64_t ) ); + ok = region.offsets != NULL && TrailsCookLayout( ctx, root, numbering, region ); + } + if ( ok ) + { + const int64_t data_offset = ( kTableCookHeaderBytes + region.align - 1 ) & ~( region.align - 1 ); + const int64_t attribution = region.count * (int64_t) sizeof( TableNodeDirEntry ); + const int64_t need = data_offset + region.bytes + attribution; + ok = (uint64_t) need <= capacity; + if ( ok ) + { + uint8_t * raw = (uint8_t *) out; + memset( raw, 0, (size_t) need ); // EVERY BYTE NO FIELD COVERS IS ZERO (§7.2) + region.base = raw + data_offset; + // the DATA part: the root at the region's base, then every numbered + // node at the offset the layout gave it, each through its own writer + ok = TrailsCookNode( ctx, region, region.base, root, order ); + for ( int64_t k = 0; ok && k < numbering.count; k++ ) + { + uint8_t * at = region.base + region.offsets[k + 1]; + const void * node = numbering.entries[k].node; + switch ( numbering.entries[k].type_id ) + { + case 0x52cfa1d198476806ull: ok = ItemCookNode( ctx, region, at, *(const Item *) node, order ); break; // Item + default: ok = false; break; + } + } + // the ATTRIBUTION part: the node directory (§6.3), one entry per node + // in index order, for `schema cook-check` + uint8_t * entry = raw + data_offset + region.bytes; + table_cook_put( entry, 0, 8, order ); + table_cook_put( entry + 8, 0xb4578774a78fb150ull, 8, order ); // the root: fnv1a64( "Trails" ) + for ( int64_t k = 0; k < numbering.count; k++ ) + { + entry += sizeof( TableNodeDirEntry ); + table_cook_put( entry, (uint64_t) region.offsets[k + 1], 8, order ); + table_cook_put( entry + 8, numbering.entries[k].type_id, 8, order ); + } + // and the HEADER (§7.1), every word a u64 in the order the file is + // produced in; the two RESERVED words are the memset's zeros + if ( ok ) + { + table_cook_put( raw + 0, TableCookMagic, 8, order ); + table_cook_put( raw + 8, BuildVersion, 8, order ); + table_cook_put( raw + 16, (uint64_t) ( order == TableByteOrder::Big ? 2 : 1 ), 8, order ); + table_cook_put( raw + 24, (uint64_t) region.bytes, 8, order ); + table_cook_put( raw + 32, (uint64_t) attribution, 8, order ); + table_cook_put( raw + 40, (uint64_t) region.align, 8, order ); + } + } + } + allocator.free( allocator.context, region.offsets ); + TableNumberingShutdown( numbering ); + return ok; +} + +// TrailsCookMeasure / TrailsCook over a REGION root — a locked builder's AsConst, a +// region TrailsLoad produced, or an opened cook — with the pair the numbering +// allocates through as an optional last argument, as the wire's own entries +// take it (§13.9). +inline int64_t TrailsCookMeasure( const Trails * root, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return -1; } + TableRegionCtx ctx; + return TrailsCookMeasureFrom( ctx, *root, allocator ); +} + +inline bool TrailsCook( const Trails * root, void * out, uint64_t capacity, TableByteOrder order, TableAllocator allocator = TableDefaultAllocator() ) +{ + if ( root == NULL ) { return false; } + TableRegionCtx ctx; + return TrailsCookFrom( ctx, *root, out, capacity, order, allocator ); +} + +// and over a BUILDER, locked or not: the builder's own pair, and the arena +// encoding while it is still mutable (§6.3). +inline int64_t TrailsCookMeasure( const TrailsBuilder & builder ) +{ + if ( builder.region != NULL ) { return TrailsCookMeasure( builder.AsConst(), builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return -1; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return TrailsCookMeasureFrom( ctx, *(const Trails *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), builder.arena.allocator ); +} + +inline bool TrailsCook( const TrailsBuilder & builder, void * out, uint64_t capacity, TableByteOrder order ) +{ + if ( builder.region != NULL ) { return TrailsCook( builder.AsConst(), out, capacity, order, builder.arena.allocator ); } + if ( builder.root_ref.null() ) { return false; } // the root allocation failed + TableArenaCtx ctx = { &builder.arena }; + return TrailsCookFrom( ctx, *(const Trails *) TableArenaAt( builder.arena, (uint32_t) builder.root_ref.value ), out, capacity, order, builder.arena.allocator ); +} + +// ---- relocatability, enforced: the wire is a pure length-prefixed +// stream AND the decoded storage is pointer-free — every closure type +// must stay trivially copyable and standard-layout, so instances can be +// memcpy'd, mmap'd, shared across processes, and walked through +// descriptor offsets. A failure here means a pointer, virtual or +// non-trivial member crept into generated storage. +// +// They ask the COMPILER ITSELF, which is what every C++ standard library +// answers the same two questions with — and it costs this header no +// include at all. +// A pointer FIELD is a TableRef — eight bytes and no address — so the +// property holds in BOTH forms: a fixed-size table is one relocatable +// struct, and a packed region is one relocatable block whose references +// are self-relative and therefore survive a plain memcpy. +static_assert( __is_trivially_copyable( TrailsStepsEntry ), "TrailsStepsEntry must stay relocatable" ); +static_assert( __is_standard_layout( TrailsStepsEntry ), "TrailsStepsEntry must stay standard-layout for offsetof" ); +static_assert( __is_trivially_copyable( Trails ), "Trails must stay relocatable" ); +static_assert( __is_standard_layout( Trails ), "Trails must stay standard-layout for offsetof" ); + +// ---- the cook's layout contract (docs/SPEC-TABLES.md §20.3) ---- +// +// The compiler derived every number below from the declaration and folded it +// into the BUILD VERSION; these asserts are this compiler saying whether it +// agrees. The model is not self-evidently right — on 32-bit System V +// alignof(uint64_t) is 4, not 8 — which is precisely why it is asserted +// rather than assumed. +static_assert( sizeof( TrailsStepsEntry ) == 24, "TrailsStepsEntry's sizeof moved: the build version was taken over 24, so a cook of it would not be this build's file (docs/SPEC-TABLES.md §20.3)" ); +static_assert( alignof( TrailsStepsEntry ) == 8, "TrailsStepsEntry's alignof moved: the build version was taken over 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( TrailsStepsEntry, key ) == 0, "TrailsStepsEntry's field key moved: the build version was taken over offset 0 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( TrailsStepsEntry, value ) == 8, "TrailsStepsEntry's field value moved: the build version was taken over offset 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( sizeof( Trails ) == 24, "Trails's sizeof moved: the build version was taken over 24, so a cook of it would not be this build's file (docs/SPEC-TABLES.md §20.3)" ); +static_assert( alignof( Trails ) == 8, "Trails's alignof moved: the build version was taken over 8 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( Trails, steps ) == 0, "Trails's field steps moved: the build version was taken over offset 0 (docs/SPEC-TABLES.md §20.3)" ); +static_assert( offsetof( Trails, after ) == 16, "Trails's field after moved: the build version was taken over offset 16 (docs/SPEC-TABLES.md §20.3)" ); + +static_assert( alignof( TableRef ) <= kTableAlign, "TrailsStepsEntry.value: an unbounded array's element alignment must fit the arena's" ); + +// ---- reflection descriptors (tables only, docs/SPEC-TABLES.md) ---- + +inline const TableTypeInfo * TrailsStepsEntryTableType(); +inline const TableTypeInfo * TrailsTableType(); +// The descriptors are CONSTANT-INITIALISED data, and a field's target is +// the ADDRESS of another descriptor. These declarations are what let a +// self- or mutually-referential graph — Node naming itself through *Node — +// be expressed as constant data instead of a lazy link, which could not +// have been written race-free OR recursion-safe. The whole reflection +// surface is therefore immutable: read it from any thread, any time. +extern const TableTypeInfo TrailsStepsEntryTableInfo; +extern const TableTypeInfo TrailsTableInfo; + +inline const TableFieldInfo TrailsStepsEntryTableFields[] = { + { "key", "key", "uint32", 0x3dc94a19365b10ecull, 8, false, false, NULL, NULL, false, false, 0, (uint32_t) offsetof( TrailsStepsEntry, key ), (uint32_t) sizeof( TrailsStepsEntry::key ), 0xffffffffu, 0xffffffffu, NULL, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, + { "value", "value", "Item", 0x7ce4fd9430e80ceaull, 17, true, true, []( const void * slot ) -> const void * { return (const void *) ItemAt( *(const TableRef *) slot ); }, []( TableWorker & worker, void * slot ) -> void * { return (void *) ItemEmplace( worker, *(TableRef *) slot ); }, true, false, 0, (uint32_t) offsetof( TrailsStepsEntry, value ), (uint32_t) sizeof( TableRef ), (uint32_t) offsetof( TrailsStepsEntry, value.count ), 0xffffffffu, &ItemTableInfo, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, []( TableWorker & worker, void * slot, const char *, int32_t, int64_t ) -> void * { return (void *) TableListPlace( worker, *(TableList *) slot ); }, "", TableDocNone, 0, NULL }, +}; +inline const TableTypeInfo TrailsStepsEntryTableInfo = { "TrailsStepsEntry", (uint32_t) sizeof( TrailsStepsEntry ), 2, TrailsStepsEntryTableFields, +[]( void * p ) { TrailsStepsEntryReset( *(TrailsStepsEntry *) p ); }, true, TableDocNone, 0, NULL }; +inline const TableTypeInfo * TrailsStepsEntryTableType() { return &TrailsStepsEntryTableInfo; } + +inline const TableFieldInfo TrailsTableFields[] = { + { "steps", "steps", "map[uint32]*Item", 0x124250ad5a5b6d14ull, 13, true, false, NULL, NULL, true, false, 0, (uint32_t) offsetof( Trails, steps ), (uint32_t) sizeof( TrailsStepsEntry ), (uint32_t) offsetof( Trails, steps.count ), 0xffffffffu, &TrailsStepsEntryTableInfo, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, []( TableWorker & worker, void * slot, const char *, int32_t, int64_t key_value ) -> void * { return (void *) TableMapPlace( worker, *(TableMap *) slot, (uint32_t) key_value ); }, "", TableDocNone, 0, NULL }, + { "after", "after", "int32", 0xbf82010f6f71eae9ull, 4, false, false, NULL, NULL, false, false, 0, (uint32_t) offsetof( Trails, after ), (uint32_t) sizeof( Trails::after ), 0xffffffffu, 0xffffffffu, NULL, false, 0.0, 0.0, 0, NULL, -1, NULL, NULL, NULL, NULL, NULL, NULL, NULL, "", TableDocNone, 0, NULL }, +}; +inline const TableTypeInfo TrailsTableInfo = { "Trails", (uint32_t) sizeof( Trails ), 2, TrailsTableFields, +[]( void * p ) { TrailsReset( *(Trails *) p ); }, true, TableDocNone, 0, NULL }; +inline const TableTypeInfo * TrailsTableType() { return &TrailsTableInfo; } + +// ---- the text form (docs/SPEC-TABLES.md §16) ---- + +// Trails in and out of a JSON text (docs/SPEC-TABLES.md §16.7): read into a +// builder, written from a region's const root. A node named more than once +// carries `&node` in the text. Defined in TrailsTable.cpp; link it to use them. +bool TrailsFromJson( TrailsBuilder & builder, const char * text, int64_t bytes, TableReport * report ); +int64_t TrailsToJsonMeasure( const Trails * root, TableAllocator allocator = TableDefaultAllocator() ); +int64_t TrailsToJson( const Trails * root, char * buffer, int64_t capacity, TableAllocator allocator = TableDefaultAllocator() ); + +} // namespace mapdemo diff --git a/testdata/wire/tables/map_conn.bin b/testdata/wire/tables/map_conn.bin index edea0d793ba149ca8def62db0e5acf7b144b48ef..1435cfef8e18eb439f9519c27fc30af087dc7cfa 100644 GIT binary patch delta 162 zcmbQl*2*r#$jEuO;N+zY1s_Q!zPoH}x7a2Mr86@z2~I4?k(nT18J9lgY9$}Te<0xJ zeZ}*_V9MVoHGB*}`8x*%+``^%%+~9929uvWm$64dBsV&0ZGe*yOc0?7Y>41wO{Qp$ sz&4HqhAGh}CO0$bakLc}ujW-?5t+P!$$=waWB>A!_V6u}-!mx#09BkptpET3 delta 53 zcmV-50LuS`2a*O70RagaKAaA4E3V`M4(bI4=LL}xXOV?tlLP_5lhpx?lYs(TljQ;} LlN$p+lVbxSygd(vr3iMlh|#4-#ci0pbZ3Kth4T!vIJLu(XJ<0%;kx?937- uMxgBt4UH*4nt_2uh-Zl*h{0eW1SA~3{FnCl`~Uxc1_mdF0}db|AOHa9I61$xq%DXMOsnyOMU{c%1PdUcz~Ny4qy$(h{;&XP8Mf@q5++8V s?F|izDL|TmfklXCp&^LDz#s&~4qyIDJN^Ct|33qR6T<-kkPr|60Iv8fHvj+t From 31c4a5de9678613649320cc103e511937ee7f988 Mon Sep 17 00:00:00 2001 From: Rowan Date: Mon, 7 Sep 2026 06:02:16 -0400 Subject: [PATCH 5/5] tables: the predicate's comment carries no em dash (#666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit House rule, and the refusal citation is §2.4 and §15, which is what the diagnostic itself names. Co-Authored-By: Claude Opus 5 --- internal/codegen/cpptable/maps.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/codegen/cpptable/maps.go b/internal/codegen/cpptable/maps.go index 71bb628a1..b042629c4 100644 --- a/internal/codegen/cpptable/maps.go +++ b/internal/codegen/cpptable/maps.go @@ -82,13 +82,13 @@ func mapKeyOrderType(f *ir.Field) string { return "uint64_t" } -// mapValueIsPointer reports a `map[K]*T` — the value is ONE pointer SLOT, so +// mapValueIsPointer reports a `map[K]*T`, whose value is ONE pointer SLOT, so // the const form's Find answers the resolved `const T *` (docs/SPEC-TABLES.md // §2.8). AN ARRAY OF POINTERS IS NOT THIS CASE. `[N]*T`, `[..N]*T` and `[]*T` // store a `TableRef` PER ELEMENT (§2.1, §4.2), so each takes the arm its array // form takes and carries `TableRef` as the element; a predicate that read // `Type.Pointer` alone sent all three here and spelled `At` on an array. -// `[E]*T` is refused by name (§11) and reaches no arm at all. +// `[E]*T` is refused by name (§2.4, §15) and reaches no arm at all. func mapValueIsPointer(f *ir.Field) bool { value := ir.MapValueField(f) return value.Type.Pointer && value.Array == ir.ArrayNone && value.KeyEnum == ""