From 8b2ad79fdb309f2d3ee20b3ba09cb8e144717755 Mon Sep 17 00:00:00 2001 From: Thomas Hart Date: Fri, 11 Sep 2026 19:04:10 +0000 Subject: [PATCH] feat: Add hasMany and belongsTo eager loading Declare relations, batch related rows with a unique IN list, and nest them in memory so N parents cost one extra query instead of N. --- README.md | 25 ++++- src/index.ts | 1 + src/relations.ts | 205 +++++++++++++++++++++++++++++++++++++++++ test/relations.test.ts | 171 ++++++++++++++++++++++++++++++++++ 4 files changed, 401 insertions(+), 1 deletion(-) create mode 100644 src/relations.ts create mode 100644 test/relations.test.ts diff --git a/README.md b/README.md index dabf1f3..79712ec 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ A from-scratch type-safe ORM and a lab for the relational designs that break nai ## What this demonstrates -Most ORM tutorials stop at `findMany`. This repo goes the other way. It builds the internals of a small ORM (query builder, parameter binding, later migrations, relations, and a unit of work) and then uses that machinery against schemas that are actually hard: multi-tenant isolation, bitemporal history, trees, money, and soft delete. The point is to see where typed query construction earns its keep and where you should write SQL by hand. +Most ORM tutorials stop at `findMany`. This repo goes the other way. It builds the internals of a small ORM (query builder, parameter binding, migrations, relations, and later a unit of work) and then uses that machinery against schemas that are actually hard: multi-tenant isolation, bitemporal history, trees, money, and soft delete. The point is to see where typed query construction earns its keep and where you should write SQL by hand. ## Concepts demonstrated @@ -22,11 +22,17 @@ Most ORM tutorials stop at `findMany`. This repo goes the other way. It builds t - Migration journal with a zero-padded id prefix invariant - Dialect type mapping (`BOOLEAN`/`TIMESTAMPTZ` vs SQLite `INTEGER`/`TEXT`) - Expand/contract: in-place column ALTER is rejected; drop and add instead +- N+1 query problem versus batched eager loading +- `hasMany` / `belongsTo` cardinality (1:N nested arrays, N:1 nullable parent) +- Hash join in the client: `IN (...)` then group by foreign key +- Identity map so two children share one parent object +- JOIN cartesian product vs two-query nesting (parent rows are not duplicated) ## What's implemented - Type-safe SELECT/WHERE/JOIN query builder with prepared-statement binding - Declarative schema DSL + migration runner with up/down and a diff generator +- hasMany/belongsTo relations with eager loading to kill the N+1 problem ## Usage @@ -68,6 +74,23 @@ const to = defineSchema({ new Migrator(session, 'sqlite').migrateUp([{ id: '001_init', up: diffSchema(from, to), down: diffSchema(to, from) }]) ``` +Relations are declared once. Loading N parents then looping `WHERE userId = parent.id` is N extra queries. `eagerHasMany` compiles one `WHERE userId IN (...)` with unique keys, then nests children in memory. `belongsTo` does the inverse and reuses one parent object per id. + +```ts +import { belongsTo, eagerHasMany, hasMany } from 'orm-workbench' + +const userOrders = hasMany('orders', users, orders, users.id, orders.userId) +const orderUser = belongsTo('user', orders, users, orders.userId, users.id) + +const graph = eagerHasMany(session, userOrders, parents, { + id: orders.id, + userId: orders.userId, + totalCents: orders.totalCents, +}, { dialect: 'sqlite', orderBy: orders.id }) +``` + +A JOIN of users to orders repeats every user column once per order. The two-query path keeps parent rows unique and attaches `orders: []` when a user has none. + ```bash pnpm install pnpm run typecheck diff --git a/src/index.ts b/src/index.ts index 6bc8d8a..31a6e86 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,3 +2,4 @@ export * from './schema' export * from './query' export * from './ddl' export * from './migrate' +export * from './relations' diff --git a/src/relations.ts b/src/relations.ts new file mode 100644 index 0000000..3e56854 --- /dev/null +++ b/src/relations.ts @@ -0,0 +1,205 @@ +import type { SqlSession } from './migrate' +import { from, inList, type Dialect, type InferRow, type PreparedQuery } from './query' +import { quoteIdent, type AnyTable, type ColRef } from './schema' + +export type HasManyRel = { + readonly kind: 'hasMany' + readonly as: As + readonly parent: AnyTable + readonly child: AnyTable + readonly local: ColRef + readonly foreign: ColRef +} + +export type BelongsToRel = { + readonly kind: 'belongsTo' + readonly as: As + readonly child: AnyTable + readonly parent: AnyTable + readonly local: ColRef + readonly foreign: ColRef +} + +export type Relation = HasManyRel | BelongsToRel + +type SelectShape = Record> + +export type RelatedQueryOpts = { + dialect?: Dialect + orderBy?: ColRef + dir?: 'asc' | 'desc' +} + +function requireCol(table: AnyTable, col: ColRef, role: string): void { + if (col.table !== table.$alias) { + throw new Error(`${role} ${col.table}.${col.column} is not on ${table.$alias}`) + } + if (!(col.column in table.$columns)) { + throw new Error(`${role} unknown column ${table.$alias}.${col.column}`) + } +} + +function colKind(table: AnyTable, col: ColRef): string { + const def = table.$columns[col.column] + if (!def) throw new Error(`Unknown column ${table.$alias}.${col.column}`) + return def.kind +} + +function declareRel( + as: string, + owner: AnyTable, + localTable: AnyTable, + local: ColRef, + foreignTable: AnyTable, + foreign: ColRef, +): void { + quoteIdent(as) + if (as in owner.$columns) { + throw new Error(`Relation ${JSON.stringify(as)} collides with a column on ${owner.$name}`) + } + requireCol(localTable, local, 'local') + requireCol(foreignTable, foreign, 'foreign') + const left = colKind(localTable, local) + const right = colKind(foreignTable, foreign) + if (left !== right) { + throw new Error( + `Relation columns ${local.table}.${local.column} (${left}) and ${foreign.table}.${foreign.column} (${right}) differ`, + ) + } +} + +export function hasMany( + as: As, + parent: AnyTable, + child: AnyTable, + local: ColRef, + foreign: ColRef, +): HasManyRel { + declareRel(as, parent, parent, local, child, foreign) + return { kind: 'hasMany', as, parent, child, local, foreign } +} + +export function belongsTo( + as: As, + child: AnyTable, + parent: AnyTable, + local: ColRef, + foreign: ColRef, +): BelongsToRel { + declareRel(as, child, child, local, parent, foreign) + return { kind: 'belongsTo', as, child, parent, local, foreign } +} + +export function uniqueKeys(values: readonly unknown[]): unknown[] { + const seen = new Set() + const out: unknown[] = [] + for (const value of values) { + if (value === null || value === undefined) continue + if (seen.has(value)) continue + seen.add(value) + out.push(value) + } + return out +} + +function relatedTable(rel: Relation): AnyTable { + return rel.kind === 'hasMany' ? rel.child : rel.parent +} + +function assertJoinProjected(shape: SelectShape, col: ColRef): void { + const projected = shape[col.column] + if (!projected || projected.table !== col.table || projected.column !== col.column) { + throw new Error(`Select must project ${col.table}.${col.column} as ${col.column} to nest related rows`) + } +} + +function readKey(row: Record, column: string, table: string): unknown { + if (!(column in row)) { + throw new Error(`Row is missing ${table}.${column}; project that column under its own name`) + } + return row[column] +} + +export function relatedQuery( + rel: Relation, + keys: readonly unknown[], + shape: S, + opts: RelatedQueryOpts = {}, +): PreparedQuery> { + const table = relatedTable(rel) + assertJoinProjected(shape, rel.foreign) + if (opts.orderBy && opts.orderBy.table !== table.$alias) { + throw new Error(`orderBy ${opts.orderBy.table}.${opts.orderBy.column} is not on ${table.$alias}`) + } + let query = from(table) + .where(inList(rel.foreign as ColRef, uniqueKeys(keys))) + .select(shape) + if (opts.orderBy) query = query.orderBy(opts.orderBy, opts.dir ?? 'asc') + return query.compile(opts.dialect ?? 'postgres') +} + +export function nestHasMany, C extends Record>( + rel: HasManyRel, + parents: readonly P[], + children: readonly C[], +): Array

{ + const groups = new Map() + for (const child of children) { + const key = readKey(child, rel.foreign.column, rel.child.$alias) + if (key === null || key === undefined) continue + const bucket = groups.get(key) + if (bucket) bucket.push(child) + else groups.set(key, [child]) + } + return parents.map((parent) => { + const key = readKey(parent, rel.local.column, rel.parent.$alias) + const related = key === null || key === undefined ? [] : (groups.get(key) ?? []) + return { ...parent, [rel.as]: related.slice() } as P & { [K in As]: C[] } + }) +} + +export function nestBelongsTo, P extends Record>( + rel: BelongsToRel, + children: readonly C[], + parents: readonly P[], +): Array { + const byId = new Map() + for (const parent of parents) { + const key = readKey(parent, rel.foreign.column, rel.parent.$alias) + if (key === null || key === undefined) continue + if (!byId.has(key)) byId.set(key, parent) + } + return children.map((child) => { + const key = readKey(child, rel.local.column, rel.child.$alias) + const related = key === null || key === undefined ? null : (byId.get(key) ?? null) + return { ...child, [rel.as]: related } as C & { [K in As]: P | null } + }) +} + +export function eagerHasMany, S extends SelectShape>( + session: SqlSession, + rel: HasManyRel, + parents: readonly P[], + childSelect: S, + opts: RelatedQueryOpts = {}, +): Array

> }> { + const keys = uniqueKeys(parents.map((parent) => readKey(parent, rel.local.column, rel.parent.$alias))) + if (keys.length === 0) return nestHasMany(rel, parents, []) + const compiled = relatedQuery(rel, keys, childSelect, opts) + const children = session.all(compiled.sql, compiled.params) as Array> + return nestHasMany(rel, parents, children) +} + +export function eagerBelongsTo, S extends SelectShape>( + session: SqlSession, + rel: BelongsToRel, + children: readonly C[], + parentSelect: S, + opts: RelatedQueryOpts = {}, +): Array | null }> { + const keys = uniqueKeys(children.map((child) => readKey(child, rel.local.column, rel.child.$alias))) + if (keys.length === 0) return nestBelongsTo(rel, children, []) + const compiled = relatedQuery(rel, keys, parentSelect, opts) + const parents = session.all(compiled.sql, compiled.params) as Array> + return nestBelongsTo(rel, children, parents) +} diff --git a/test/relations.test.ts b/test/relations.test.ts new file mode 100644 index 0000000..fa007e8 --- /dev/null +++ b/test/relations.test.ts @@ -0,0 +1,171 @@ +import { expect, expectTypeOf, test } from 'vitest' +import { + alias, belongsTo, defineTable, eagerBelongsTo, eagerHasMany, hasMany, integer, + nestBelongsTo, nestHasMany, relatedQuery, text, uniqueKeys, type SqlSession, +} from '../src/index' + +const users = defineTable('users', { id: integer(), email: text() }) +const orders = defineTable('orders', { id: integer(), userId: integer(), sku: text() }) +const people = defineTable('people', { id: integer(), managerId: integer(), name: text() }) + +const userOrders = hasMany('orders', users, orders, users.id, orders.userId) +const orderUser = belongsTo('user', orders, users, orders.userId, users.id) +const reports = alias(people, 'reports') +const managerReports = hasMany('reports', people, reports, people.id, reports.managerId) + +const childSelect = { id: orders.id, userId: orders.userId, sku: orders.sku } +const parentSelect = { id: users.id, email: users.email } + +type SqliteDb = { + exec: (sql: string) => void + prepare: (sql: string) => { all: (...p: unknown[]) => unknown; run: (...p: unknown[]) => unknown } + close: () => void +} + +function sqlite(): { session: SqlSession; queries: () => number; close: () => void } { + const loaded = ( + globalThis as { process?: { getBuiltinModule?: (id: string) => { DatabaseSync: new (path: string) => SqliteDb } } } + ).process?.getBuiltinModule?.('node:sqlite') + if (!loaded) throw new Error('node:sqlite is not available in this runtime') + const db = new loaded.DatabaseSync(':memory:') + db.exec(` + CREATE TABLE "users" ("id" INTEGER PRIMARY KEY, "email" TEXT NOT NULL); + CREATE TABLE "orders" ("id" INTEGER PRIMARY KEY, "userId" INTEGER, "sku" TEXT NOT NULL); + CREATE TABLE "people" ("id" INTEGER PRIMARY KEY, "managerId" INTEGER, "name" TEXT NOT NULL); + INSERT INTO "users" ("id", "email") VALUES (1, 'a@x'), (2, 'b@x'), (3, 'c@x'); + INSERT INTO "orders" ("id", "userId", "sku") VALUES (10, 1, 'red'), (11, 1, 'blue'), (12, 2, 'green'); + INSERT INTO "people" ("id", "managerId", "name") VALUES (1, NULL, 'pat'), (2, 1, 'lee'), (3, 1, 'sam'), (4, 2, 'kim'); + `) + let queries = 0 + const session: SqlSession = { + exec: (sql, params) => { + queries += 1 + if (!params || params.length === 0) db.exec(sql) + else db.prepare(sql).run(...params) + }, + all: (sql, params) => { + queries += 1 + return db.prepare(sql).all(...(params ?? [])) as Record[] + }, + } + return { session, queries: () => queries, close: () => db.close() } +} + +test('relatedQuery batches unique keys into one IN list and skips nulls', () => { + const compiled = relatedQuery(userOrders, [1, 1, null, 2, undefined], childSelect) + expect(compiled.sql.split('\n')).toEqual([ + 'SELECT "orders"."id" AS "id", "orders"."userId" AS "userId", "orders"."sku" AS "sku"', + 'FROM "orders" AS "orders"', + 'WHERE "orders"."userId" IN ($1, $2)', + ]) + expect(compiled.params).toEqual([1, 2]) + const empty = relatedQuery(orderUser, [], parentSelect, { dialect: 'sqlite' }) + expect(empty.sql).toContain('WHERE 1 = 0') + expect(empty.params).toEqual([]) + expect(empty.sql).not.toMatch(/\$\d/) +}) + +test('hasMany nests children, empty parents get [], orphans are dropped', () => { + const parents = [{ id: 1, email: 'a@x' }, { id: 3, email: 'c@x' }, { id: 1, email: 'a-again' }] + const children = [ + { id: 11, userId: 1, sku: 'blue' }, + { id: 10, userId: 1, sku: 'red' }, + { id: 99, userId: 9, sku: 'ghost' }, + ] + const nested = nestHasMany(userOrders, parents, children) + expect(nested[0]?.orders.map((row) => row.sku)).toEqual(['blue', 'red']) + expect(nested[1]?.orders).toEqual([]) + expect(nested[2]?.orders.map((row) => row.sku)).toEqual(['blue', 'red']) + expect(nested[0]?.orders).not.toBe(nested[2]?.orders) + nested[0]?.orders.push({ id: 0, userId: 1, sku: 'x' }) + expect(nested[2]?.orders).toHaveLength(2) +}) + +test('belongsTo uses an identity map and leaves missing/null FKs as null', () => { + const parents = [{ id: 1, email: 'a@x' }, { id: 1, email: 'dup' }, { id: 2, email: 'b@x' }] + const children = [ + { id: 10, userId: 1, sku: 'red' }, + { id: 11, userId: 1, sku: 'blue' }, + { id: 12, userId: null, sku: 'anon' }, + { id: 13, userId: 9, sku: 'ghost' }, + ] + const nested = nestBelongsTo(orderUser, children, parents) + expect(nested[0]?.user?.email).toBe('a@x') + expect(nested[1]?.user).toBe(nested[0]?.user) + expect(nested[2]?.user).toBeNull() + expect(nested[3]?.user).toBeNull() +}) + +test('eager hasMany is two queries against N+1, including a parent with no children', () => { + const db = sqlite() + try { + const parents = db.session.all('SELECT "id", "email" FROM "users" ORDER BY "id"') + const n1Start = db.queries() + for (const parent of parents) { + db.session.all('SELECT "id", "userId", "sku" FROM "orders" WHERE "userId" = ?', [parent.id]) + } + expect(db.queries() - n1Start).toBe(parents.length) + + const eagerStart = db.queries() + const graph = eagerHasMany(db.session, userOrders, parents, childSelect, { + dialect: 'sqlite', + orderBy: orders.id, + }) + expect(db.queries() - eagerStart).toBe(1) + expect(graph.map((row) => row.orders.map((order) => order.sku))).toEqual([['red', 'blue'], ['green'], []]) + expectTypeOf(graph[0]!.orders[0]!.sku).toEqualTypeOf() + } finally { + db.close() + } +}) + +test('eager belongsTo loads unique parents once; empty and all-null skip the second query', () => { + const db = sqlite() + try { + const children = db.session.all('SELECT "id", "userId", "sku" FROM "orders" ORDER BY "id"') + const start = db.queries() + const graph = eagerBelongsTo(db.session, orderUser, children, parentSelect, { dialect: 'sqlite' }) + expect(db.queries() - start).toBe(1) + expect(graph.map((row) => row.user?.email ?? null)).toEqual(['a@x', 'a@x', 'b@x']) + expect(graph[0]?.user).toBe(graph[1]?.user) + + const skipped = db.queries() + expect(eagerHasMany(db.session, userOrders, [], childSelect, { dialect: 'sqlite' })).toEqual([]) + expect(eagerBelongsTo(db.session, orderUser, [{ id: 1, userId: null, sku: 'z' }], parentSelect, { dialect: 'sqlite' })[0]?.user).toBeNull() + expect(db.queries()).toBe(skipped) + } finally { + db.close() + } +}) + +test('self-relation hasMany groups through an alias; uniqueKeys drops nulls', () => { + const db = sqlite() + try { + const managers = db.session.all('SELECT "id", "managerId", "name" FROM "people" ORDER BY "id"') + const graph = eagerHasMany( + db.session, + managerReports, + managers, + { id: reports.id, managerId: reports.managerId, name: reports.name }, + { dialect: 'sqlite', orderBy: reports.id }, + ) + expect(graph.find((row) => row.name === 'pat')?.reports.map((row) => row.name)).toEqual(['lee', 'sam']) + expect(graph.find((row) => row.name === 'lee')?.reports.map((row) => row.name)).toEqual(['kim']) + expect(graph.find((row) => row.name === 'kim')?.reports).toEqual([]) + } finally { + db.close() + } + expect(uniqueKeys([null, 1, undefined, 1, 2])).toEqual([1, 2]) + expect(uniqueKeys([])).toEqual([]) +}) + +test('relation declarations and nest/load reject mismatched columns and missing keys', () => { + expect(() => hasMany('email', users, orders, users.id, orders.userId)).toThrow(/collides/) + expect(() => belongsTo('user', orders, users, orders.sku, users.id)).toThrow(/differ/) + expect(() => hasMany('orders', users, orders, users.email, orders.userId)).toThrow(/differ/) + expect(() => hasMany('orders', users, orders, orders.userId, orders.userId)).toThrow(/not on/) + expect(() => relatedQuery(userOrders, [1], { id: orders.id, sku: orders.sku })).toThrow(/project/) + expect(() => relatedQuery(userOrders, [1], childSelect, { orderBy: users.id })).toThrow(/orderBy/) + expect(() => nestHasMany(userOrders, [{ email: 'a@x' }], [])).toThrow(/missing/) + expect(() => nestBelongsTo(orderUser, [{ id: 1, sku: 'x' }], [])).toThrow(/missing/) +})