tool: unbounded arrays in the front end, the IR and the tool's wire and text (#531)

compiler/cook.go

@@ -60,6 +60,9 @@ func (c *Compiler) Cook(u *ir.Unit, root string, wire []byte, opts CookOptions)
 	if err := refuseToolMaps(u); err != nil {
 		return nil, CookReport{}, TableReport{}, err
 	}
+	if err := refuseToolLists(u); err != nil {
+		return nil, CookReport{}, TableReport{}, err
+	}
 	var rep CookReport
 	m := tabletext.NewModel(u)
 	st := m.Lookup(root)
@@ -117,6 +120,9 @@ func (c *Compiler) CookCheck(u *ir.Unit, root string, file []byte) (CookReport,
 	if err := refuseToolMaps(u); err != nil {
 		return CookReport{}, err
 	}
+	if err := refuseToolLists(u); err != nil {
+		return CookReport{}, err
+	}
 	m := tabletext.NewModel(u)
 	res, err := tablecook.Check(m, file)
 	if err != nil {
@@ -150,6 +156,9 @@ func (c *Compiler) Uncook(u *ir.Unit, root string, file []byte) ([]byte, error)
 	if err := refuseToolMaps(u); err != nil {
 		return nil, err
 	}
+	if err := refuseToolLists(u); err != nil {
+		return nil, err
+	}
 	m := tabletext.NewModel(u)
 	st := m.Lookup(root)
 	if st == nil || !st.IsTable {

compiler/tableslists.go

@@ -0,0 +1,49 @@
+// The UNBOUNDED ARRAY cross-target refusal (docs/SPEC-TABLES.md §2.9, §11):
+// its own file, per the registry split — a construct's refusal adds a file
+// beside builtin.go rather than growing it. Every target's Generate calls
+// [refuseLists], because no code generator carries the construct yet; the
+// carrier registry the map's file has lands here with the first carrier.
+package compiler
+
+import (
+	"fmt"
+
+	"github.com/mas-bandwidth/schema/v2/ir"
+)
+
+// refuseLists is the named refusal every target gives a unit whose table
+// closure declares a `[]T` (docs/SPEC-TABLES.md §2.9, §11, §15).
+//
+// An unbounded array is a VARIABLE-CLASS construct, and the variable class is
+// the C++ reference's alone — the arena, the region, the node extent and the
+// walks a list's elements ride in are all the reference's. The LANGUAGE takes
+// the spelling and the tool's WIRE and TEXT halves carry it, so `pack` and
+// `unpack` read and write one, and a generator that emitted a codec for it
+// would emit one that never met an element array. So every target refuses by
+// name until the reference lands the codec.
+func refuseLists(u *ir.Unit, target string) error {
+	fields := ir.ListFields(u)
+	if len(fields) == 0 {
+		return nil
+	}
+	return fmt.Errorf("unit declares an unbounded array in a table closure (%s) — no code generator carries `[]T` yet, %s included: the language takes the spelling and the tool's `pack` and `unpack` read and write it, and the C++ reference lands the codec first (docs/SPEC-TABLES.md §2.9, §11, §15). Declare the array at a bound, [..N]T, which is the same bytes, and remove the bound when the reference carries it",
+		englishList(fields), target)
+}
+
+// refuseToolLists is the TOOL's COOK refusal (docs/SPEC-TABLES.md §2.9, §15),
+// the map's own, one construct over: a unit whose table closure declares a
+// `[]T` is refused by name at the tool's COOK surfaces, because
+// internal/tablecook does not lay out the element arrays yet.
+//
+// It is here, at the surface, rather than in the engine, and it is NAMED
+// rather than left to the layout. Without it the engine lays out a region
+// short of the element arrays and a reader meets a slot pointing past its
+// holder's extent, which is a corrupt file with nothing saying who wrote it.
+func refuseToolLists(u *ir.Unit) error {
+	fields := ir.ListFields(u)
+	if len(fields) == 0 {
+		return nil
+	}
+	return fmt.Errorf("unit declares an unbounded array in a table closure (%s) — the tool's WIRE and TEXT halves carry the construct, and its COOK half does not, so this command would lay out a region short of the element arrays rather than refusing; the C++ reference lands the cook (--lang cpp) (docs/SPEC-TABLES.md §2.9, §15)",
+		englishList(fields))
+}

compiler/tableslists_test.go

@@ -0,0 +1,94 @@
+package compiler
+
+// The UNBOUNDED ARRAY's cross-target refusals (docs/SPEC-TABLES.md §2.9, §11,
+// §15): no code generator carries the construct yet, so every one of them
+// refuses a unit that declares one BY NAME, and none of them refuses a
+// list-free unit for it.
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/mas-bandwidth/schema/v2/ir"
+)
+
+const listSrc = `package fixture
+
+table Placement
+{
+    x float32
+}
+
+table Save
+{
+    placements []Placement
+    scores     []int32
+}
+`
+
+// TestEveryTargetRefusesAList: the refusal is a REFUSAL and not a silent
+// emission of an array whose elements no backend laid out.
+func TestEveryTargetRefusesAList(t *testing.T) {
+	u := unitFromSource(t, listSrc)
+	c := New()
+	for _, target := range c.Targets() {
+		_, err := c.Generate(u, target, Options{})
+		if err == nil {
+			t.Errorf("--lang %s emitted for a unit declaring an unbounded array", target)
+			continue
+		}
+		for _, want := range []string{"unbounded array", "Save.placements", "Save.scores", "[..N]T"} {
+			if !strings.Contains(err.Error(), want) {
+				t.Errorf("--lang %s: the refusal does not name %q: %v", target, want, err)
+			}
+		}
+	}
+}
+
+// TestNoTargetRefusesAListFreeUnit: the refusal is the CONSTRUCT's and not a
+// tax on every unit.
+func TestNoTargetRefusesAListFreeUnit(t *testing.T) {
+	u := unitFromSource(t, mapSrc)
+	if got := ir.ListFields(u); len(got) != 0 {
+		t.Fatalf("ListFields = %v for a list-free unit", got)
+	}
+	c := New()
+	for _, target := range c.Targets() {
+		if _, err := c.Generate(u, target, Options{}); err != nil && strings.Contains(err.Error(), "unbounded array") {
+			t.Errorf("--lang %s refused a list-free unit over unbounded arrays: %v", target, err)
+		}
+	}
+}
+
+// TestListFieldsNamesWhatAnAuthorWrote: the refusal names `Table.field`,
+// sorted, so a reader goes to a declaration and not to a generated name.
+func TestListFieldsNamesWhatAnAuthorWrote(t *testing.T) {
+	got := ir.ListFields(unitFromSource(t, listSrc))
+	want := []string{"Save.placements", "Save.scores"}
+	if len(got) != len(want) {
+		t.Fatalf("ListFields = %v, want %v", got, want)
+	}
+	for i := range want {
+		if got[i] != want[i] {
+			t.Fatalf("ListFields = %v, want %v", got, want)
+		}
+	}
+}
+
+// TestTheToolsCookRefusesAList: the tool's WIRE and TEXT halves carry the
+// construct and its COOK half does not, so the cook surfaces refuse by name
+// rather than laying out a region short of the element arrays.
+func TestTheToolsCookRefusesAList(t *testing.T) {
+	err := refuseToolLists(unitFromSource(t, listSrc))
+	if err == nil {
+		t.Fatal("the tool's cook accepted a unit declaring an unbounded array")
+	}
+	for _, want := range []string{"Save.placements", "WIRE and TEXT halves carry", "--lang cpp"} {
+		if !strings.Contains(err.Error(), want) {
+			t.Errorf("the cook refusal does not name %q: %v", want, err)
+		}
+	}
+	if err := refuseToolLists(unitFromSource(t, mapSrc)); err != nil {
+		t.Fatalf("the tool refused a list-free unit: %v", err)
+	}
+}

compiler/target_c.go

@@ -28,6 +28,9 @@ func (cTarget) Generate(u *ir.Unit, _ Options) (map[string][]byte, error) {
 	if err := refuseMaps(u, "c"); err != nil {
 		return nil, err
 	}
+	if err := refuseLists(u, "c"); err != nil {
+		return nil, err
+	}
 	files, err := cgen.Generate(u)
 	if err != nil {
 		return nil, err

compiler/target_cpp.go

@@ -22,6 +22,13 @@ func (cppTarget) Generate(u *ir.Unit, _ Options) (map[string][]byte, error) {
 	if err := refusePacketVoidArms(u, "cpp"); err != nil {
 		return nil, err
 	}
+	// the UNBOUNDED ARRAY is OWED in this target (docs/SPEC-TABLES.md §2.9):
+	// the reference lands the codec first and registers through
+	// registers a carrier when it does. Until then it refuses one by name
+	// rather than emitting an array whose elements it never laid out.
+	if err := refuseLists(u, "cpp"); err != nil {
+		return nil, err
+	}
 	files, err := cpp.Generate(u)
 	if err != nil {
 		return nil, err

compiler/target_cs.go

@@ -25,6 +25,9 @@ func (csTarget) Generate(u *ir.Unit, _ Options) (map[string][]byte, error) {
 	if err := refuseMaps(u, "cs"); err != nil {
 		return nil, err
 	}
+	if err := refuseLists(u, "cs"); err != nil {
+		return nil, err
+	}
 	files, err := csharp.Generate(u)
 	if err != nil {
 		return nil, err

compiler/target_dart.go

@@ -25,6 +25,9 @@ func (dartTarget) Generate(u *ir.Unit, _ Options) (map[string][]byte, error) {
 	if err := refuseMaps(u, "dart"); err != nil {
 		return nil, err
 	}
+	if err := refuseLists(u, "dart"); err != nil {
+		return nil, err
+	}
 	files, err := dart.Generate(u)
 	if err != nil {
 		return nil, err

compiler/target_elixir.go

@@ -25,6 +25,9 @@ func (elixirTarget) Generate(u *ir.Unit, _ Options) (map[string][]byte, error) {
 	if err := refuseMaps(u, "elixir"); err != nil {
 		return nil, err
 	}
+	if err := refuseLists(u, "elixir"); err != nil {
+		return nil, err
+	}
 	files, err := elixir.Generate(u)
 	if err != nil {
 		return nil, err

compiler/target_go.go

@@ -25,6 +25,9 @@ func (goTarget) Generate(u *ir.Unit, _ Options) (map[string][]byte, error) {
 	if err := refuseMaps(u, "go"); err != nil {
 		return nil, err
 	}
+	if err := refuseLists(u, "go"); err != nil {
+		return nil, err
+	}
 	files, err := golang.Generate(u)
 	if err != nil {
 		return nil, err

compiler/target_java.go

@@ -25,6 +25,9 @@ func (javaTarget) Generate(u *ir.Unit, _ Options) (map[string][]byte, error) {
 	if err := refuseMaps(u, "java"); err != nil {
 		return nil, err
 	}
+	if err := refuseLists(u, "java"); err != nil {
+		return nil, err
+	}
 	files, err := java.Generate(u)
 	if err != nil {
 		return nil, err

compiler/target_javascript.go

@@ -27,6 +27,9 @@ func (jsTarget) Generate(u *ir.Unit, _ Options) (map[string][]byte, error) {
 	if err := refuseMaps(u, "js"); err != nil {
 		return nil, err
 	}
+	if err := refuseLists(u, "js"); err != nil {
+		return nil, err
+	}
 	files, err := js.Generate(u)
 	if err != nil {
 		return nil, err

compiler/target_rust.go

@@ -25,6 +25,9 @@ func (rustTarget) Generate(u *ir.Unit, _ Options) (map[string][]byte, error) {
 	if err := refuseMaps(u, "rust"); err != nil {
 		return nil, err
 	}
+	if err := refuseLists(u, "rust"); err != nil {
+		return nil, err
+	}
 	files, err := rust.Generate(u)
 	if err != nil {
 		return nil, err

docs/COMPARISON-TABLES.md

@@ -214,7 +214,7 @@ the source list at the end.
 | Optional fields | `?T` on a nested table, a type, an enum, a flags mask, a scalar and a bounded array: the value plus a `_present` bool, fixed size, no allocation (§2.3); on a string, on `bytes` and on a value whose closure is variable, a named follow-on (§15) | optional scalars via `= null`; references null when absent (FB-schema) | explicit presence: proto2 all, proto3 `optional`, editions explicit by default (PB-presence) |
 | Defaults | `= v` on scalars; part of the wire contract because a default is elided; changing one is a silent edit the baseline refuses (§4, §4.1, §18) | scalar defaults; "don't change existing default values" (FB-evolution) | proto2 `[default]`; proto3 zero, not serialized; editions serialize set defaults (PB-presence) |
 | Union | `union` with implicit `None`; arms by name hash, so add, remove, reorder freely; an arm IS a field line, so its type is any type a field's is — a `table` inside a table closure included — and an arm may carry no payload at all (§2.6, §5) | `union` of tables; structs and strings experimental; vectors of unions C++ only (FB-schema) | `oneof`; `Any` for open typing (PB-proto3) |
-| Arrays | `[N]T`, `[..N]T`, `[A..B]T` on both wires, and `[]T` unbounded in a table body, whose count the data decides (§2.9, specified and emitted nowhere yet); the bound is not wire identity, so `[]T` and `[..N]T` are the same bytes (§2, §4) | `[T]` unbounded; `[T:N]` in structs only (FB-schema) | `repeated`, unbounded, packed scalars (PB-proto3) |
+| Arrays | `[N]T`, `[..N]T`, `[A..B]T` on both wires, and `[]T` unbounded in a table body, whose count the data decides (§2.9, taken by the front end and by the tool, emitted by no backend yet); the bound is not wire identity, so `[]T` and `[..N]T` are the same bytes (§2, §4) | `[T]` unbounded; `[T:N]` in structs only (FB-schema) | `repeated`, unbounded, packed scalars (PB-proto3) |
 | Enum-keyed arrays | `[E]T`: one slot per variant, no `None` slot, bad keys refused in every build, slots ride by name (§2.4, §3.2) | — | — |
 | Maps | `map[K]V` in a table body, a lookup over entries the wire carries as a sorted array of one generated `{ key, value }` table; keys are `string(N)` and the integer kinds, every other key refused by name; makes the holder variable (§2.8), in the reference and the tool | sorted vector of tables with `key` plus `LookupByKey` (FB-cpp) | `map<K,V>`, integral or string keys, order undefined (PB-proto3) |
 | Sorted lookup in a buffer | the map's `Find`: a binary search in place over the sorted entry array, the same call in a locked region, a loaded one and an opened cook, plus an optional index the caller builds at load and never stores (§2.8) | `key`, `CreateVectorOfSortedTables`, `LookupByKey` (FB-cpp) | — |
@@ -268,7 +268,7 @@ the source list at the end.
 | Union evolution | arms by name; add anywhere, remove, reorder (§2.6, §5) | append or explicit discriminant (FB-evolution) | adding is fine; moving an existing field into a oneof is unsafe (PB-editions) |
 | Flags evolution | append only, retire in place; the baseline refuses the rest (§4.1) | explicit values, any order (FB-schema) | — |
 | Array bound change | prefix kept, `clamped` counted; a short array fills with defaults (§4) | unbounded | unbounded |
-| Unbounded arrays | `[]T` and `[]*T` in a TABLE body, whose count the data decides (§2.9, specified and emitted nowhere yet); the same bytes as `[..N]T`, so the bound is a declaration-side fact and moving between them is silent or a clamp. Refused in a `type` body by name, which is what keeps the packet wire bounded | unbounded vectors | unbounded repeated fields |
+| Unbounded arrays | `[]T` and `[]*T` in a TABLE body, whose count the data decides (§2.9, taken by the front end and by the tool, emitted by no backend yet); the same bytes as `[..N]T`, so the bound is a declaration-side fact and moving between them is silent or a clamp. Refused in a `type` body by name, which is what keeps the packet wire bounded | unbounded vectors | unbounded repeated fields |
 | `T` to `?T` to `*T` | `T` and `?T` are byte-identical for non-default content; to or from `*T` is a counted mismatch (§2.3, §4) | changes default semantics; required/optional changes break (FB-schema) | presence changes round-trip of defaults (PB-presence) |
 | Unknown fields on read | skipped by length, counted (§3, §4) | ignored (FB-evolution) | retained in the unknown set (PB-proto3) |
 | Unknown fields on rewrite | dropped and counted, by design; the writer has a schema | the buffer keeps them if forwarded whole (FB-evolution) | preserved and re-serialized (PB-proto3) |

docs/SPEC-TABLES.md

@@ -2755,11 +2755,13 @@ each:
 | a fixed-table user pays nothing | a list-free unit carries no list machinery, held by the zero-cost gate's header scan (§2.2) | every runtime carries the repeated codec | every runtime carries the vector |
 
 **BACKEND STATUS: OWED, not emitted.** This section is specified ahead of its
-implementation, on the same terms §3.3 and §6.6 take: no backend carries an
-unbounded array today, the front end refuses the spelling, and the corpus holds
-no `tables/lists`. The C++ REFERENCE and the TOOL land it first and every other
-backend refuses a unit that declares one, by name (§11), with the ports a named
-follow-on (§15). The corpus the implementation owes is `tables/lists`:
+implementation, on the same terms §3.3 and §6.6 take: **the FRONT END takes the
+spelling and holds every refusal above, and the TOOL's WIRE and TEXT halves
+carry the construct**, so `pack` and `unpack` read and write a `[]T` and the
+projections render it. **No CODE GENERATOR carries it**, every one of them
+refuses a unit that declares one by name (§11), and the corpus holds no
+`tables/lists`. The C++ REFERENCE lands the codec next and every other backend
+keeps refusing, with the ports a named follow-on (§15). The corpus the implementation owes is `tables/lists`:
 `list_empty` (an empty list beside a full one), `list_scalars`, `list_tables`,
 `list_shared` (two slots naming one node beside a null slot),
 `list_before_pointer` (the walk-order control above), `list_erased` (an erase

docs/USAGE.md

@@ -2169,9 +2169,10 @@ its quoted decimal spelling, entries in ascending key order.
 
 ### Unbounded arrays: `placements []Placement`
 
-*Specified, not yet emitted. The front end refuses the `[]T` spelling at
-parse, no backend carries the construct, and the corpus holds no
-`tables/lists` (SPEC-TABLES.md §2.9).*
+*Partly landed. The front end takes the `[]T` spelling and holds every
+refusal below, and `pack` and `unpack` read and write one. No backend carries
+the construct — every one of them refuses a unit that declares one, by name —
+and the corpus holds no `tables/lists` (SPEC-TABLES.md §2.9).*
 
 **An unbounded array is a counted array whose count the DATA decides.** It is
 the map with the key and the sort taken out: the same kind `14` body a

internal/ast/ast.go

@@ -153,6 +153,10 @@ const (
 	ArrayFixed ArrayKind = iota // [N]T
 	ArrayUpTo                   // [..N]T — sugar for [0..N] (SPEC §4.3)
 	ArrayRange                  // [Min..N]T
+	// ArrayList is `[]T` — an UNBOUNDED ARRAY, a counted array whose count
+	// the data decides (docs/SPEC-TABLES.md §2.9). The bracket carries no
+	// expression, so Lo and Hi are both nil.
+	ArrayList
 )
 
 type ArrayBound struct {

internal/baseline/baseline.go

@@ -350,6 +350,15 @@ func renderField(f *ir.Field) Field {
 	case f.Array == ir.ArrayCounted:
 		add("array", "bounded")
 		add("bound", strconv.FormatInt(f.ArrayBound, 10))
+	case f.Array == ir.ArrayList:
+		// AN UNBOUNDED ARRAY RENDERS AS `array=unbounded` WITH NO `bound=`
+		// (docs/SPEC-TABLES.md §2.9): the count is the data's, and the only
+		// ceiling is the int32 extent cap every array shares (§2.2). The
+		// missing token IS the capacity fact — a `bound=` appearing on an
+		// `array=` move is a capacity SHRINK and warns, and one vanishing is
+		// a capacity GROWTH and passes (§18.2) — so the token's absence
+		// carries the verdict the token itself would have carried.
+		add("array", "unbounded")
 	}
 	if f.Type.Kind == ir.TFixed {
 		// a fixed field's SCALE. The kind fixes the width and the signedness;