tables: the C++ codec for unbounded arrays (#531)

compiler/cook.go

@@ -117,12 +117,6 @@ func orderWord(big bool) string {
 // It is a person's decision to run it, not a parameter on a load: the runtime
 // keeps one `Open` that matches the header and points.
 func (c *Compiler) CookCheck(u *ir.Unit, root string, file []byte) (CookReport, error) {
-	if err := refuseToolMaps(u); err != nil {
-		return CookReport{}, err
-	}
-	if err := refuseToolLists(u); err != nil {
-		return CookReport{}, err
-	}
 	m := tabletext.NewModel(u)
 	res, err := tablecook.Check(m, file)
 	if err != nil {

compiler/tableslists.go

@@ -1,8 +1,8 @@
 // The UNBOUNDED ARRAY cross-target refusal (docs/SPEC-TABLES.md §2.9, §11):
-// its own file, per the registry split — a construct's refusal adds a file
-// beside builtin.go rather than growing it. Every target's Generate calls
-// [refuseLists], because no code generator carries the construct yet; the
-// carrier registry the map's file has lands here with the first carrier.
+// its own file, per the registry split: a construct's carrier registry and
+// its refusal add a file beside builtin.go rather than growing it. A target
+// that carries the construct registers through [registerListCarrier] from its
+// own file's init, and every other target's Generate calls [refuseLists].
 package compiler
 
 import (
@@ -11,29 +11,40 @@ import (
 	"github.com/mas-bandwidth/schema/v2/ir"
 )
 
-// refuseLists is the named refusal every target gives a unit whose table
+// listTargets is the canonical name of every built-in target whose table
+// backend carries an UNBOUNDED ARRAY (docs/SPEC-TABLES.md §2.9). refuseLists
+// names them.
+var listTargets []string
+
+// registerListCarrier is what a carrying target's file calls from its init,
+// beside its registerBuiltin call.
+func registerListCarrier(name string) { listTargets = append(listTargets, name) }
+
+// refuseLists is the named refusal every PORT gives a unit whose table
 // closure declares a `[]T` (docs/SPEC-TABLES.md §2.9, §11, §15).
 //
 // An unbounded array is a VARIABLE-CLASS construct, and the variable class is
 // the C++ reference's alone — the arena, the region, the node extent and the
-// walks a list's elements ride in are all the reference's. The LANGUAGE takes
-// the spelling and the tool's WIRE and TEXT halves carry it, so `pack` and
-// `unpack` read and write one, and a generator that emitted a codec for it
-// would emit one that never met an element array. So every target refuses by
-// name until the reference lands the codec.
+// walks a list's elements ride in are all the reference's. So the reference
+// carries the codec, registers through [registerListCarrier] from its own
+// init and never reaches here, and every port refuses loudly rather than
+// emitting a codec that never met an element array.
 func refuseLists(u *ir.Unit, target string) error {
 	fields := ir.ListFields(u)
 	if len(fields) == 0 {
 		return nil
 	}
-	return fmt.Errorf("unit declares an unbounded array in a table closure (%s) — no code generator carries `[]T` yet, %s included: the language takes the spelling and the tool's `pack` and `unpack` read and write it, and the C++ reference lands the codec first (docs/SPEC-TABLES.md §2.9, §11, §15). Declare the array at a bound, [..N]T, which is the same bytes, and remove the bound when the reference carries it",
-		englishList(fields), target)
+	carry, flags := carriers(listTargets)
+	return fmt.Errorf("unit declares an unbounded array in a table closure (%s): a []T is %s only today, and the %s form is a named follow-on. Generate with %s, or declare the array at a bound, [..N]T, which is the same bytes (docs/SPEC-TABLES.md §2.9, §11, §15)",
+		englishList(fields), englishList(carry), target, englishList(flags))
 }
 
 // refuseToolLists is the TOOL's COOK refusal (docs/SPEC-TABLES.md §2.9, §15),
 // the map's own, one construct over: a unit whose table closure declares a
-// `[]T` is refused by name at the tool's COOK surfaces, because
-// internal/tablecook does not lay out the element arrays yet.
+// `[]T` is refused by name at the tool's COOK and UNCOOK surfaces, because
+// internal/tablecook does not lay out the element arrays yet. `cook-check`
+// is not among them: its scan carries §7.4's element-array clause, and a cook
+// the C++ reference wrote is checked there.
 //
 // It is here, at the surface, rather than in the engine, and it is NAMED
 // rather than left to the layout. Without it the engine lays out a region
@@ -44,6 +55,6 @@ func refuseToolLists(u *ir.Unit) error {
 	if len(fields) == 0 {
 		return nil
 	}
-	return fmt.Errorf("unit declares an unbounded array in a table closure (%s) — the tool's WIRE and TEXT halves carry the construct, and its COOK half does not, so this command would lay out a region short of the element arrays rather than refusing; the C++ reference lands the cook (--lang cpp) (docs/SPEC-TABLES.md §2.9, §15)",
+	return fmt.Errorf("unit declares an unbounded array in a table closure (%s): the tool's WIRE and TEXT halves carry the construct and `cook-check` reads one, and its COOK half does not, so this command would lay out a region short of the element arrays rather than refusing. The C++ reference carries the cook (--lang cpp) (docs/SPEC-TABLES.md §2.9, §15)",
 		englishList(fields))
 }

compiler/tableslists_test.go

@@ -1,9 +1,8 @@
 package compiler
 
 // The UNBOUNDED ARRAY's cross-target refusals (docs/SPEC-TABLES.md §2.9, §11,
-// §15): no code generator carries the construct yet, so every one of them
-// refuses a unit that declares one BY NAME, and none of them refuses a
-// list-free unit for it.
+// §15): the C++ reference carries the codec, every port refuses a unit that
+// declares one BY NAME, and none of them refuses a list-free unit for it.
 
 import (
 	"strings"
@@ -26,22 +25,38 @@ table Save
 }
 `
 
-// TestEveryTargetRefusesAList: the refusal is a REFUSAL and not a silent
-// emission of an array whose elements no backend laid out.
-func TestEveryTargetRefusesAList(t *testing.T) {
+// TestListsAreRefusedByEveryPort: the refusal is a REFUSAL and not a silent
+// emission of an array whose elements no port laid out, and the reference
+// does not refuse.
+func TestListsAreRefusedByEveryPort(t *testing.T) {
 	u := unitFromSource(t, listSrc)
 	c := New()
 	for _, target := range c.Targets() {
-		_, err := c.Generate(u, target, Options{})
-		if err == nil {
-			t.Errorf("--lang %s emitted for a unit declaring an unbounded array", target)
-			continue
-		}
-		for _, want := range []string{"unbounded array", "Save.placements", "Save.scores", "[..N]T"} {
-			if !strings.Contains(err.Error(), want) {
-				t.Errorf("--lang %s: the refusal does not name %q: %v", target, want, err)
+		t.Run(target, func(t *testing.T) {
+			_, err := c.Generate(u, target, Options{})
+			if target == "cpp" {
+				if err != nil {
+					t.Fatalf("--lang cpp refused an unbounded array: the reference carries the codec (schema#531): %v", err)
+				}
+				return
 			}
-		}
+			if err == nil {
+				t.Fatalf("--lang %s emitted for a unit declaring an unbounded array", target)
+			}
+			for _, want := range []string{"unbounded array", "Save.placements", "Save.scores", "[..N]T", "cpp"} {
+				if !strings.Contains(err.Error(), want) {
+					t.Errorf("--lang %s: the refusal does not name %q: %v", target, want, err)
+				}
+			}
+		})
+	}
+}
+
+// TestListCarrierIsTheReferenceAlone: exactly one target carries the
+// construct, and it is the C++ reference (docs/SPEC-TABLES.md §2.9, §15).
+func TestListCarrierIsTheReferenceAlone(t *testing.T) {
+	if len(listTargets) != 1 || listTargets[0] != "cpp" {
+		t.Fatalf("listTargets = %v, want exactly [cpp]: the variable class is the reference's (docs/SPEC-TABLES.md §2.9, §15)", listTargets)
 	}
 }
 
@@ -75,11 +90,27 @@ func TestListFieldsNamesWhatAnAuthorWrote(t *testing.T) {
 	}
 }
 
+// TestListRefusalNamesTheCarrier: what a port's refusal says: the carrier,
+// the flag that generates, and the fields an author wrote.
+func TestListRefusalNamesTheCarrier(t *testing.T) {
+	err := refuseLists(unitFromSource(t, listSrc), "go")
+	if err == nil {
+		t.Fatalf("refuseLists accepted a list-bearing unit for a non-carrier")
+	}
+	for _, want := range []string{"a []T is cpp only today", "Save.placements", "--lang cpp"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the carrier-form refusal does not name %q: %v", want, err)
+		}
+	}
+}
+
 // TestTheToolsCookRefusesAList: the tool's WIRE and TEXT halves carry the
-// construct and its COOK half does not, so the cook surfaces refuse by name
-// rather than laying out a region short of the element arrays.
+// construct and `cook-check` reads one, and its COOK half does not, so the
+// cook and uncook surfaces refuse by name rather than laying out a region
+// short of the element arrays.
 func TestTheToolsCookRefusesAList(t *testing.T) {
-	err := refuseToolLists(unitFromSource(t, listSrc))
+	u := unitFromSource(t, listSrc)
+	err := refuseToolLists(u)
 	if err == nil {
 		t.Fatal("the tool's cook accepted a unit declaring an unbounded array")
 	}
@@ -91,4 +122,16 @@ func TestTheToolsCookRefusesAList(t *testing.T) {
 	if err := refuseToolLists(unitFromSource(t, mapSrc)); err != nil {
 		t.Fatalf("the tool refused a list-free unit: %v", err)
 	}
+	c := New()
+	if _, _, _, err := c.Cook(u, "Save", nil, CookOptions{}); err == nil || !strings.Contains(err.Error(), "unbounded array") {
+		t.Errorf("the tool's cook did not refuse a list-bearing unit by name: %v", err)
+	}
+	if _, err := c.Uncook(u, "Save", nil); err == nil || !strings.Contains(err.Error(), "unbounded array") {
+		t.Errorf("the tool's uncook did not refuse a list-bearing unit by name: %v", err)
+	}
+	// cook-check reaches its scan: the refusal it answers for an empty file is
+	// the header's, not the construct's
+	if _, err := c.CookCheck(u, "Save", nil); err == nil || strings.Contains(err.Error(), "unbounded array") {
+		t.Errorf("cook-check refused a list-bearing unit by construct rather than reading the file: %v", err)
+	}
 }

compiler/tablesmaps.go

@@ -39,16 +39,17 @@ func refuseMaps(u *ir.Unit, target string) error {
 }
 
 // refuseToolMaps is the TOOL's COOK refusal (docs/SPEC-TABLES.md §2.8, §15): a unit
-// whose table closure declares a map is refused by name at every table surface
-// the tool has — pack, unpack, cook, cook-check and uncook — because
-// internal/tablewire does not carry the construct yet.
+// whose table closure declares a map is refused by name at the tool's COOK and
+// UNCOOK surfaces, because internal/tablecook does not lay out the entry
+// arrays yet. `cook-check` is not among them: its scan refuses a map SLOT by
+// name where it meets one (internal/tablecook), so a cook of a map-free root
+// in a unit that declares a map elsewhere is checked as any other is.
 //
 // It is here, at the surface, rather than in the engine, and it is NAMED rather
-// than left to the decoder. Without it the engine meets a kind 14 field whose
-// element kind is 13, decodes nothing into a slot it has no shape for, and
-// reports FRAMING DAMAGE — an answer that sends its reader looking for a
-// corrupt file when the file is fine and the reader is the one that is short.
-// A refusal that says which is which is the whole difference.
+// than left to the layout. Without it the engine lays out a region short of
+// the entry arrays and a reader meets a slot pointing past its holder's
+// extent, which is a corrupt file with nothing saying who wrote it. A refusal
+// that says which is which is the whole difference.
 func refuseToolMaps(u *ir.Unit) error {
 	fields := ir.MapFields(u)
 	if len(fields) == 0 {

compiler/tablesmaps_test.go

@@ -271,17 +271,18 @@ func TestMapEntryIsNotARoot(t *testing.T) {
 }
 
 // TestToolRefusesMapsByName: the tool's WIRE and TEXT halves carry maps now
-// (docs/SPEC-TABLES.md §2.8), and its COOK half does not — so the cook
-// surfaces refuse a map-bearing unit BY NAME rather than laying out an entry
-// array they have no placement for. Without the refusal a caller gets a cook
-// whose region is short of the entries, which is worse than a diagnostic.
+// (docs/SPEC-TABLES.md §2.8), and its COOK half does not. So the cook and
+// uncook surfaces refuse a map-bearing unit BY NAME rather than laying out an
+// entry array they have no placement for. Without the refusal a caller gets a
+// cook whose region is short of the entries, which is worse than a diagnostic.
+// `cook-check` refuses at the SLOT instead, where its scan meets one, and
+// internal/tablecook's TestCookCheckMapSlotRefusedByName holds that.
 func TestToolRefusesMapsByName(t *testing.T) {
 	u := unitFromSource(t, mapSrc)
 	c := New()
 	surfaces := map[string]func() error{
-		"Cook":      func() error { _, _, _, err := c.Cook(u, "Fleet", nil, CookOptions{}); return err },
-		"Uncook":    func() error { _, err := c.Uncook(u, "Fleet", nil); return err },
-		"CookCheck": func() error { _, err := c.CookCheck(u, "Fleet", nil); return err },
+		"Cook":   func() error { _, _, _, err := c.Cook(u, "Fleet", nil, CookOptions{}); return err },
+		"Uncook": func() error { _, err := c.Uncook(u, "Fleet", nil); return err },
 	}
 	for name, call := range surfaces {
 		t.Run(name, func(t *testing.T) {

compiler/target_cpp.go

@@ -22,13 +22,6 @@ func (cppTarget) Generate(u *ir.Unit, _ Options) (map[string][]byte, error) {
 	if err := refusePacketVoidArms(u, "cpp"); err != nil {
 		return nil, err
 	}
-	// the UNBOUNDED ARRAY is OWED in this target (docs/SPEC-TABLES.md §2.9):
-	// the reference lands the codec first and registers through
-	// registers a carrier when it does. Until then it refuses one by name
-	// rather than emitting an array whose elements it never laid out.
-	if err := refuseLists(u, "cpp"); err != nil {
-		return nil, err
-	}
 	files, err := cpp.Generate(u)
 	if err != nil {
 		return nil, err
@@ -53,5 +46,6 @@ func init() {
 	registerBuiltin(cppTarget{}, true, true, true, true)
 	registerWideTextCarrier("cpp") // the C++ reference carries wstring(N) on the packet wire (SPEC §4.12)
 	registerOptionalArrayCarrier("cpp")
-	registerMapCarrier("cpp") // the C++ reference carries the map codecs (docs/SPEC-TABLES.md §2.8)
+	registerMapCarrier("cpp")  // the C++ reference carries the map codecs (docs/SPEC-TABLES.md §2.8)
+	registerListCarrier("cpp") // and the unbounded array codec (docs/SPEC-TABLES.md §2.9)
 }

docs/COMPARISON-TABLES.md

@@ -214,7 +214,7 @@ the source list at the end.
 | Optional fields | `?T` on a nested table, a type, an enum, a flags mask, a scalar and a bounded array: the value plus a `_present` bool, fixed size, no allocation (§2.3); on a string, on `bytes` and on a value whose closure is variable, a named follow-on (§15) | optional scalars via `= null`; references null when absent (FB-schema) | explicit presence: proto2 all, proto3 `optional`, editions explicit by default (PB-presence) |
 | Defaults | `= v` on scalars; part of the wire contract because a default is elided; changing one is a silent edit the baseline refuses (§4, §4.1, §18) | scalar defaults; "don't change existing default values" (FB-evolution) | proto2 `[default]`; proto3 zero, not serialized; editions serialize set defaults (PB-presence) |
 | Union | `union` with implicit `None`; arms by name hash, so add, remove, reorder freely; an arm IS a field line, so its type is any type a field's is — a `table` inside a table closure included — and an arm may carry no payload at all (§2.6, §5) | `union` of tables; structs and strings experimental; vectors of unions C++ only (FB-schema) | `oneof`; `Any` for open typing (PB-proto3) |
-| Arrays | `[N]T`, `[..N]T`, `[A..B]T` on both wires, and `[]T` unbounded in a table body, whose count the data decides (§2.9, taken by the front end and by the tool, emitted by no backend yet); the bound is not wire identity, so `[]T` and `[..N]T` are the same bytes (§2, §4) | `[T]` unbounded; `[T:N]` in structs only (FB-schema) | `repeated`, unbounded, packed scalars (PB-proto3) |
+| Arrays | `[N]T`, `[..N]T`, `[A..B]T` on both wires, and `[]T` unbounded in a table body, whose count the data decides (§2.9, carried by the C++ reference and the tool, refused by every port by name); the bound is not wire identity, so `[]T` and `[..N]T` are the same bytes (§2, §4) | `[T]` unbounded; `[T:N]` in structs only (FB-schema) | `repeated`, unbounded, packed scalars (PB-proto3) |
 | Enum-keyed arrays | `[E]T`: one slot per variant, no `None` slot, bad keys refused in every build, slots ride by name (§2.4, §3.2) | — | — |
 | Maps | `map[K]V` in a table body, a lookup over entries the wire carries as a sorted array of one generated `{ key, value }` table; keys are `string(N)` and the integer kinds, every other key refused by name; makes the holder variable (§2.8), in the reference and the tool | sorted vector of tables with `key` plus `LookupByKey` (FB-cpp) | `map<K,V>`, integral or string keys, order undefined (PB-proto3) |
 | Sorted lookup in a buffer | the map's `Find`: a binary search in place over the sorted entry array, the same call in a locked region, a loaded one and an opened cook, plus an optional index the caller builds at load and never stores (§2.8) | `key`, `CreateVectorOfSortedTables`, `LookupByKey` (FB-cpp) | — |
@@ -268,7 +268,7 @@ the source list at the end.
 | Union evolution | arms by name; add anywhere, remove, reorder (§2.6, §5) | append or explicit discriminant (FB-evolution) | adding is fine; moving an existing field into a oneof is unsafe (PB-editions) |
 | Flags evolution | append only, retire in place; the baseline refuses the rest (§4.1) | explicit values, any order (FB-schema) | — |
 | Array bound change | prefix kept, `clamped` counted; a short array fills with defaults (§4) | unbounded | unbounded |
-| Unbounded arrays | `[]T` and `[]*T` in a TABLE body, whose count the data decides (§2.9, taken by the front end and by the tool, emitted by no backend yet); the same bytes as `[..N]T`, so the bound is a declaration-side fact and moving between them is silent or a clamp. Refused in a `type` body by name, which is what keeps the packet wire bounded | unbounded vectors | unbounded repeated fields |
+| Unbounded arrays | `[]T` and `[]*T` in a TABLE body, whose count the data decides (§2.9, carried by the C++ reference and the tool, refused by every port by name); the same bytes as `[..N]T`, so the bound is a declaration-side fact and moving between them is silent or a clamp. Refused in a `type` body by name, which is what keeps the packet wire bounded | unbounded vectors | unbounded repeated fields |
 | `T` to `?T` to `*T` | `T` and `?T` are byte-identical for non-default content; to or from `*T` is a counted mismatch (§2.3, §4) | changes default semantics; required/optional changes break (FB-schema) | presence changes round-trip of defaults (PB-presence) |
 | Unknown fields on read | skipped by length, counted (§3, §4) | ignored (FB-evolution) | retained in the unknown set (PB-proto3) |
 | Unknown fields on rewrite | dropped and counted, by design; the writer has a schema | the buffer keeps them if forwarded whole (FB-evolution) | preserved and re-serialized (PB-proto3) |