From 459f90b1e9edec7859d5a8d5a531f749a826d183 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:27:47 +0200 Subject: [PATCH 1/4] feat(database): defineRelations and findMany, one statement per level of with --- src/database.ts | 15 ++ src/database/relations.test-d.ts | 82 ++++++++ src/database/relations.test.ts | 192 +++++++++++++++++++ src/database/relations.ts | 293 +++++++++++++++++++++++++++++ tests/relations.clickhouse.test.ts | 72 +++++++ 5 files changed, 654 insertions(+) create mode 100644 src/database/relations.test-d.ts create mode 100644 src/database/relations.test.ts create mode 100644 src/database/relations.ts create mode 100644 tests/relations.clickhouse.test.ts diff --git a/src/database.ts b/src/database.ts index ba66164..7b9b94a 100644 --- a/src/database.ts +++ b/src/database.ts @@ -38,6 +38,21 @@ export { TransactionUnsupported, type TransactionError, } from "./database/errors" +export { + defineRelations, + findMany, + type FindManyOptions, + type FindManyRow, + type KeyColumn, + type Relation, + type RelationalSchema, + type RelationalTable, + type RelationBuilder, + type RelationOptions, + type RelationsConfig, + type RelationTables, + type WithOf, +} from "./database/relations" export { QuerySummary, withQuerySummary } from "./database/tracing" export { sql, type SqlIdentifier, type SqlRawText, type SqlTemplate } from "./database/sql" export type { DialectTransactions, IsolationLevel, TransactionSettings } from "./ch/dialect" diff --git a/src/database/relations.test-d.ts b/src/database/relations.test-d.ts new file mode 100644 index 0000000..05aa0eb --- /dev/null +++ b/src/database/relations.test-d.ts @@ -0,0 +1,82 @@ +// Type-level tests: relation declarations are checked against the tables, and +// `findMany` rows follow the `with` tree. + +import type { DateTime, Effect } from "effect" +import { expectTypeOf } from "expect-type" +import * as PG from "../postgres" +import * as Db from "../database" + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.int4, title: PG.text, at: PG.timestamptz }, + primaryKey: ["id"], +}) + +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +type User = { readonly id: number; readonly name: string } +type FindError = Db.DatabaseError | import("../postgres").QueryBuilderError | import("../postgres").CompiledQueryDecodeError + +// No `with`: the table's own row. +expectTypeOf(Db.findMany(schema.users)).toEqualTypeOf, FindError, Db.Database>>() +expectTypeOf(Db.findMany(schema.users, { where: ($) => [$.name.eq("ada")] })).toEqualTypeOf< + Effect.Effect, FindError, Db.Database> +>() + +// `many` is an array, `one` is a row or null, nested to any depth. +expectTypeOf( + Db.findMany(schema.users, { + with: { posts: { where: ($) => [$.title.eq("a")], with: { author: true } } }, + }), +).toEqualTypeOf< + Effect.Effect< + ReadonlyArray<{ + readonly id: number + readonly name: string + readonly posts: ReadonlyArray<{ + readonly id: number + readonly authorId: number + readonly title: string + readonly at: DateTime.Utc + readonly author: User | null + }> + }>, + FindError, + Db.Database + > +>() + +// `where` and `orderBy` on a nested level see that level's columns. +Db.findMany(schema.users, { + // @ts-expect-error -- `title` is a column of posts, not users + where: ($) => [$.title.eq("a")], +}) + +// A relation name `with` does not declare. +// @ts-expect-error -- users has no relation `author` +Db.findMany(schema.users, { with: { author: true } }) + +// Declarations are checked: target table, `to` column, `from` column. +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- no table `comments` + users: { posts: r.many("comments", { from: "id", to: "authorId" }) }, +})) +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- posts has no column `userId` + users: { posts: r.many("posts", { from: "id", to: "userId" }) }, +})) +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- users has no column `uid` + users: { posts: r.many("posts", { from: "uid", to: "authorId" }) }, +})) +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- a timestamptz decodes to DateTime, which is not a key + posts: { author: r.one("users", { from: "at", to: "id" }) }, +})) +Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + // @ts-expect-error -- `name` is already a column of users + users: { name: r.many("posts", { from: "id", to: "authorId" }) }, +})) diff --git a/src/database/relations.test.ts b/src/database/relations.test.ts new file mode 100644 index 0000000..59358ac --- /dev/null +++ b/src/database/relations.test.ts @@ -0,0 +1,192 @@ +// Relational reads (`defineRelations`, `findMany`) on PGlite: one statement per +// level of `with`, rows decoded through each level's own row codec. + +import { PgliteClient } from "@effect/sql-pglite" +import { assert, expect, layer } from "@effect/vitest" +import { Cause, DateTime, Effect, Exit, Layer } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import * as PG from "../postgres" +import { postgresDialect } from "../pg/dialect" +import * as Db from "../database" + +const statements: Array = [] + +const Live = Layer.effect( + Db.Database, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient + return Db.fromSqlClient(sql, { + dialect: postgresDialect, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + }), +).pipe(Layer.provideMerge(PgliteClient.layer({ postgresqlconf: "timezone = 'UTC'" }))) + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.column(PG.int4, { name: "author_id" }), title: PG.text }, + primaryKey: ["id"], +}) + +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +const setup = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE users (id int4 PRIMARY KEY, name text NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE posts (id int4 PRIMARY KEY, author_id int4 NOT NULL, title text NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO users VALUES (1, 'ada'), (2, 'bob'), (3, 'cy')`) + yield* Db.execute(Db.sql`INSERT INTO posts VALUES (10, 1, 'a1'), (11, 1, 'a2'), (12, 2, 'b1')`) +}) + +layer(Live, { excludeTestServices: true })("findMany on PGlite", (it) => { + it.effect("loads a many relation and a nested one, one statement per level", () => + Effect.gen(function* () { + yield* setup + statements.length = 0 + const users = yield* Db.findMany(schema.users, { + orderBy: ($) => [[$.id, "asc"]], + with: { posts: { orderBy: ($) => [[$.id, "asc"]], with: { author: true } } }, + }) + expect(users).toEqual([ + { + id: 1, + name: "ada", + posts: [ + { id: 10, authorId: 1, title: "a1", author: { id: 1, name: "ada" } }, + { id: 11, authorId: 1, title: "a2", author: { id: 1, name: "ada" } }, + ], + }, + { id: 2, name: "bob", posts: [{ id: 12, authorId: 2, title: "b1", author: { id: 2, name: "bob" } }] }, + { id: 3, name: "cy", posts: [] }, + ]) + expect(statements).toHaveLength(3) + expect(statements[1]).toMatch(/IN \(/) + }), + ) + + it.effect("a one relation is null when no row matches, and an empty level sends nothing", () => + Effect.gen(function* () { + yield* Db.execute(Db.sql`INSERT INTO posts VALUES (13, 99, 'orphan')`) + const orphan = yield* Db.findMany(schema.posts, { where: ($) => [$.id.eq(13)], with: { author: true } }) + expect(orphan).toEqual([{ id: 13, authorId: 99, title: "orphan", author: null }]) + statements.length = 0 + const none = yield* Db.findMany(schema.users, { where: ($) => [$.id.eq(404)], with: { posts: true } }) + expect(none).toEqual([]) + expect(statements).toHaveLength(1) + }), + ) + + it.effect("a nested where filters that level only", () => + Effect.gen(function* () { + const users = yield* Db.findMany(schema.users, { + where: ($) => [$.id.in_(1, 2)], + orderBy: ($) => [[$.id, "asc"]], + with: { posts: { where: ($) => [$.title.eq("a2")] } }, + }) + expect(users.map((user) => [user.name, user.posts.map((post) => post.title)])).toEqual([ + ["ada", ["a2"]], + ["bob", []], + ]) + }), + ) + + it.effect("nested rows decode through their column codecs", () => + Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE events (id int4 PRIMARY KEY, user_id int4 NOT NULL, at timestamptz NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO events VALUES (1, 1, '2026-01-02T03:04:05Z')`) + const Events = PG.table("events", { + columns: { id: PG.int4, userId: PG.column(PG.int4, { name: "user_id" }), at: PG.timestamptz }, + primaryKey: ["id"], + }) + const withEvents = Db.defineRelations({ users: Users, events: Events }, (r) => ({ + users: { events: r.many("events", { from: "id", to: "userId" }) }, + })) + const [ada] = yield* Db.findMany(withEvents.users, { where: ($) => [$.id.eq(1)], with: { events: true } }) + const at = ada!.events[0]!.at + assert(DateTime.isDateTime(at)) + expect(DateTime.formatIso(at)).toBe("2026-01-02T03:04:05.000Z") + }), + ) +}) + +// Tenant scope on every level: a separate database that refuses unscoped statements. +const Orgs = PG.table("orgs", { columns: { id: PG.text }, primaryKey: ["id"] }) +const Units = PG.table("units", { + columns: { id: PG.int4, orgId: PG.column(PG.text, { name: "org_id" }) }, + primaryKey: ["id"], + tenantColumn: "orgId", +}) +const Locks = PG.table("locks", { + columns: { id: PG.int4, orgId: PG.column(PG.text, { name: "org_id" }), unitId: PG.column(PG.int4, { name: "unit_id" }) }, + primaryKey: ["id"], + tenantColumn: "orgId", +}) +const tenancy = Db.defineRelations({ orgs: Orgs, units: Units, locks: Locks }, (r) => ({ + orgs: { units: r.many("units", { from: "id", to: "orgId" }) }, + units: { locks: r.many("locks", { from: "id", to: "unitId" }) }, +})) + +const Scoped = Layer.effect( + Db.Database, + Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient + return Db.fromSqlClient(sql, { + dialect: postgresDialect, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + }), +).pipe(Layer.provideMerge(PgliteClient.layer({ postgresqlconf: "timezone = 'UTC'" }))) + +const tenantSetup = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE orgs (id text PRIMARY KEY)`) + yield* Db.execute(Db.sql`CREATE TABLE units (id int4 PRIMARY KEY, org_id text NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE locks (id int4 PRIMARY KEY, org_id text NOT NULL, unit_id int4 NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO orgs VALUES ('a'), ('b')`) + yield* Db.execute(Db.sql`INSERT INTO units VALUES (1, 'a'), (2, 'b')`) + // Lock 21 points at unit 1 but belongs to org b: bad data a tenant pin must not leak. + yield* Db.execute(Db.sql`INSERT INTO locks VALUES (10, 'a', 1), (20, 'b', 2), (21, 'b', 1)`) +}) + +layer(Scoped, { excludeTestServices: true })("findMany tenant scope", (it) => { + it.effect("a level under one tenant's rows is pinned to that tenant", () => + Effect.gen(function* () { + yield* tenantSetup + statements.length = 0 + const units = yield* Db.findMany(tenancy.units, { where: ($) => [$.orgId.eq("a")], with: { locks: true } }) + expect(units).toEqual([{ id: 1, orgId: "a", locks: [{ id: 10, orgId: "a", unitId: 1 }] }]) + expect(statements[1]).toMatch(/org_id" = /) + }), + ) + + it.effect("an unscoped top level is refused before any nested level runs", () => + Effect.gen(function* () { + statements.length = 0 + const exit = yield* Effect.exit(Db.findMany(tenancy.units, { with: { locks: true } })) + assert(Exit.isFailure(exit)) + expect(Cause.squash(exit.cause)).toBeInstanceOf(Db.TenantScopeViolation) + expect(statements).toEqual([]) + }), + ) + + it.effect("an untenanted parent gives a tenanted level nothing to pin, so it is refused", () => + Effect.gen(function* () { + const exit = yield* Effect.exit(Db.findMany(tenancy.orgs, { with: { units: true } })) + assert(Exit.isFailure(exit)) + expect(Cause.squash(exit.cause)).toBeInstanceOf(Db.TenantScopeViolation) + }), + ) + + it.effect("crossTenant declares every level", () => + Effect.gen(function* () { + const units = yield* Db.findMany(tenancy.units, { + orderBy: ($) => [[$.id, "asc"]], + with: { locks: { orderBy: ($) => [[$.id, "asc"]] } }, + crossTenant: true, + }) + expect(units.map((unit) => unit.locks.map((lock) => lock.id))).toEqual([[10, 21], [20]]) + }), + ) +}) diff --git a/src/database/relations.ts b/src/database/relations.ts new file mode 100644 index 0000000..d5fae05 --- /dev/null +++ b/src/database/relations.ts @@ -0,0 +1,293 @@ +// Relational reads: relations declared over tables, and `findMany` with a +// nested `with`, as Drizzle's relational queries and Prisma's `include`. +// +// Each level of `with` is one ordinary builder query, `WHERE IN (...)` +// over the keys the level above returned. So every level compiles, decodes and +// derives its tenant scope the way any other query does, and `run` checks it. +// See design/relations.md for why this and not JSON aggregation. + +import { Effect } from "effect" +import type { CompiledQueryDecodeError } from "../ch/compile" +import type { QueryBuilderError } from "../ch/errors" +import type { Condition, Expr } from "../ch/expr" +import { from, type ColumnAccessor, type CHQuery } from "../ch/query" +import type { SelectRowOf, Table } from "../ch/table" +import type { ColumnDefs, InferTS } from "../ch/types" +import { Database } from "./database" +import type { DatabaseError } from "./errors" + +type AnyTable = Table + +/** The tables a relation schema covers, by the key relations use to name them. */ +export type RelationTables = Record + +type ColumnsOf = T extends Table ? Cols : never + +/** + * Columns a relation can join on: their decoded value is a string, number or + * bigint, which the loader matches by value. A `DateTime` or bytes column is + * not a key. + */ +export type KeyColumn = { + [K in keyof ColumnsOf]: NonNullable[K]>> extends string | number | bigint ? K : never +}[keyof ColumnsOf] & + string + +/** + * A relation from one table to another: rows of `target` whose `to` column + * equals this row's `from` column. `"one"` loads the first such row or `null`, + * `"many"` loads all of them. + */ +export interface Relation< + Kind extends "one" | "many" = "one" | "many", + Target extends string = string, + From extends string = string, + To extends string = string, +> { + readonly _tag: "Relation" + readonly kind: Kind + readonly target: Target + readonly from: From + readonly to: To +} + +/** What the `defineRelations` callback gets: `r.one` and `r.many`, checked against the tables. */ +export interface RelationBuilder { + readonly one: >( + target: Target, + on: { readonly from: From; readonly to: To }, + ) => Relation<"one", Target, From, To> + readonly many: >( + target: Target, + on: { readonly from: From; readonly to: To }, + ) => Relation<"many", Target, From, To> +} + +/** + * The relations of each table, by relation name. `from` must be a key column + * of the source table, and a relation name must not be one of its columns: the + * loaded rows carry both. + */ +export type RelationsConfig = { + readonly [S in keyof Tables]?: { readonly [name: string]: Relation<"one" | "many", keyof Tables & string, KeyColumn> } & { + readonly [C in keyof ColumnsOf]?: never + } +} + +type RelationsOf = K extends keyof R ? NonNullable : {} + +/** One table of a relation schema, as `findMany` takes it: `schema.users`. */ +export interface RelationalTable { + readonly _tag: "RelationalTable" + readonly key: K + readonly table: Tables[K] + /** @internal */ + readonly tables: Tables + /** @internal */ + readonly relations: R +} + +/** What `defineRelations` returns: one `RelationalTable` per table key. */ +export type RelationalSchema = { + readonly [K in keyof Tables & string]: RelationalTable +} + +/** + * Relations over a set of tables, Drizzle's `defineRelations`. Relations name + * their target by its key in `tables`, so two tables can point at each other + * and a table at itself. + * + * ```ts + * const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + * users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + * posts: { author: r.one("users", { from: "authorId", to: "id" }) }, + * })) + * ``` + */ +export const defineRelations = >( + tables: Tables, + build: (r: RelationBuilder) => R, +): RelationalSchema => { + const relation = + (kind: "one" | "many") => + (target: string, on: { readonly from: string; readonly to: string }): Relation => ({ + _tag: "Relation", + kind, + target, + from: on.from, + to: on.to, + }) + const relations = build({ one: relation("one"), many: relation("many") } as unknown as RelationBuilder) + return Object.fromEntries( + Object.keys(tables).map((key) => [key, { _tag: "RelationalTable", key, table: tables[key], tables, relations }]), + ) as unknown as RelationalSchema +} + +/** Options for one nested level of `with`. */ +export interface RelationOptions { + /** More conditions on this level, ANDed with the relation's key match. */ + readonly where?: ($: ColumnAccessor>) => ReadonlyArray + /** The order of this level's rows, within each parent. */ + readonly orderBy?: ($: ColumnAccessor>) => ReadonlyArray, "asc" | "desc"]> + /** Relations to load under each row of this level. */ + readonly with?: WithOf +} + +/** `with`: each relation to load, as `true` or with its own options. */ +export type WithOf = { + readonly [N in keyof RelationsOf]?: + | true + | (RelationsOf[N] extends Relation ? RelationOptions : never) +} + +/** Options for `findMany`: the top level also takes `limit`, `offset` and `crossTenant`. */ +export interface FindManyOptions + extends RelationOptions { + readonly limit?: number + readonly offset?: number + /** Declare every level `.crossTenant()`; see docs/tenant-scoping.md. */ + readonly crossTenant?: boolean +} + +type WithIn = O extends { readonly with?: infer W } ? (W extends object ? W : {}) : {} + +type Simplify = { readonly [K in keyof A]: A[K] } & {} + +/** A row `findMany` returns: every column of the table, plus each relation `with` asked for. */ +export type FindManyRow = Simplify< + SelectRowOf & { + readonly [N in keyof W & keyof RelationsOf as W[N] extends undefined ? never : N]: RelationsOf[N] extends Relation< + infer Kind, + infer T extends keyof Tables & string + > + ? Kind extends "many" + ? ReadonlyArray>> + : FindManyRow> | null + : never + } +> + +type FindError = DatabaseError | QueryBuilderError | CompiledQueryDecodeError +type Row = Record +interface Level { + readonly where?: ($: any) => ReadonlyArray + readonly orderBy?: ($: any) => ReadonlyArray, "asc" | "desc"]> + readonly with?: Record + readonly limit?: number + readonly offset?: number +} + +const distinct = (values: ReadonlyArray): Array => [ + ...new Set(values.filter((value) => value !== null && value !== undefined)), +] + +const levelQuery = ( + table: AnyTable, + level: Level, + match: ($: any) => ReadonlyArray, + crossTenant: boolean, +): CHQuery => { + let query: CHQuery = from(table as Table) + .select() + .where(($: any) => [...match($), ...(level.where?.($) ?? [])]) + if (level.orderBy !== undefined) query = query.orderBy(level.orderBy as any) + if (level.limit !== undefined) query = query.limit(level.limit) + if (level.offset !== undefined) query = query.offset(level.offset) + return crossTenant ? query.crossTenant() : query +} + +/** + * Load the relations `level.with` asks for under `rows` of `table`: one query + * per relation, then the next level under its rows. + */ +const attach = ( + tables: RelationTables, + relations: Record | undefined>, + key: string, + rows: ReadonlyArray, + level: Level, + crossTenant: boolean, +): Effect.Effect, FindError, Database> => + Effect.gen(function* () { + const names = Object.entries(level.with ?? {}).filter(([, spec]) => spec !== undefined) + if (names.length === 0) return rows + const db = yield* Database + const parent = tables[key]! + const loaded: Record>> = {} + for (const [name, spec] of names) { + const relation = relations[key]?.[name] + if (relation === undefined) { + return yield* Effect.die(new Error(`findMany: ${key} has no relation named ${name}`)) + } + const target = tables[relation.target]! + const nested: Level = spec === true ? {} : (spec as Level) + const keys = distinct(rows.map((row) => row[relation.from])) + let children: ReadonlyArray = [] + if (keys.length > 0) { + // The child level keeps the tenant its parents were read for: when the + // parent rows all carry one tenant, the child query pins its own tenant + // column to it, so its scope is proved like any other query's. + const tenants = + parent.tenantColumn !== undefined && target.tenantColumn !== undefined + ? distinct(rows.map((row) => row[parent.tenantColumn!])) + : [] + const tenantColumn = target.tenantColumn + const query = levelQuery( + target, + nested, + ($) => [ + $[relation.to].in_(...keys), + tenantColumn !== undefined && tenants.length === 1 ? $[tenantColumn].eq(tenants[0]) : undefined, + ], + crossTenant, + ) + const decoded = (yield* db.run(query as any)) as ReadonlyArray + children = yield* attach(tables, relations, relation.target, decoded, nested, crossTenant) + } + const byKey = new Map>() + for (const child of children) { + const value = child[relation.to] + const group = byKey.get(value) + if (group === undefined) byKey.set(value, [child]) + else group.push(child) + } + loaded[name] = byKey + } + return rows.map((row) => { + const out: Row = { ...row } + for (const [name] of names) { + const relation = relations[key]![name]! + const group = loaded[name]!.get(row[relation.from]) ?? [] + out[name] = relation.kind === "many" ? group : (group[0] ?? null) + } + return out + }) + }) + +/** + * Read rows of one table with the relations `with` names, nested to any depth. + * Each level is one statement, `WHERE IN (...)` over the keys of the + * level above, decoded through its own row codec. A level whose parents carry + * one tenant is pinned to it. Levels are separate statements: run `findMany` + * inside `transaction` for one snapshot across them. + * + * ```ts + * Db.findMany(schema.users, { where: ($) => [$.orgId.eq(orgId)], with: { posts: { with: { author: true } } } }) + * ``` + */ +export const findMany: { + ( + table: RelationalTable, + ): Effect.Effect>, FindError, Database> + >( + table: RelationalTable, + options: O, + ): Effect.Effect>>, FindError, Database> +} = (table: RelationalTable, options?: FindManyOptions) => + Effect.gen(function* () { + const level = (options ?? {}) as Level & { readonly crossTenant?: boolean } + const crossTenant = level.crossTenant === true + const db = yield* Database + const rows = (yield* db.run(levelQuery(table.table, level, () => [], crossTenant) as any)) as ReadonlyArray + return yield* attach(table.tables, table.relations as any, table.key, rows, level, crossTenant) + }) as any diff --git a/tests/relations.clickhouse.test.ts b/tests/relations.clickhouse.test.ts new file mode 100644 index 0000000..98cd43c --- /dev/null +++ b/tests/relations.clickhouse.test.ts @@ -0,0 +1,72 @@ +// `findMany` on a live ClickHouse: relations need no foreign keys, so the same +// loader runs here. Creates two tables in a throwaway database. + +import { ClickhouseClient } from "@effect/sql-clickhouse" +import { Effect } from "effect" +import { describe, expect, it } from "vitest" +import * as CH from "@maple-dev/effect-orm/clickhouse" +import * as Db from "@maple-dev/effect-orm/database" +import { endpoint } from "./clickhouse-support" + +const user = process.env.EFFECT_ORM_CLICKHOUSE_USER ?? "default" +const password = process.env.EFFECT_ORM_CLICKHOUSE_PASSWORD ?? "" + +const Services = CH.table("services", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, Id: CH.uint32, Name: CH.string }, +}) +const Events = CH.table("events", { + external: true, + tenantColumn: "OrgId", + columns: { OrgId: CH.string, ServiceId: CH.uint32, At: CH.dateTime64 }, +}) +const schema = Db.defineRelations({ services: Services, events: Events }, (r) => ({ + services: { events: r.many("events", { from: "Id", to: "ServiceId" }) }, + events: { service: r.one("services", { from: "ServiceId", to: "Id" }) }, +})) + +describe.skipIf(!endpoint)("findMany on live ClickHouse", () => { + it("loads nested relations, pinned to the tenant, one statement per level", async () => { + const { rows, sent } = await Effect.runPromise( + Effect.gen(function* () { + const database = `eo_relations_${Date.now()}_${Math.floor(Math.random() * 1e6)}` + const admin = yield* ClickhouseClient.ClickhouseClient + yield* admin.asCommand(admin.unsafe(`CREATE DATABASE ${database}`)) + return yield* Effect.gen(function* () { + const client = yield* ClickhouseClient.ClickhouseClient + const sent: Array = [] + const db = Db.fromSqlClient(client, { + dialect: CH.clickhouseDialect, + command: client.asCommand, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void sent.push(statement.sql)), + }) + yield* db.execute(Db.sql`CREATE TABLE services (OrgId String, Id UInt32, Name String) ENGINE = MergeTree ORDER BY Id`) + yield* db.execute( + Db.sql`CREATE TABLE events (OrgId String, ServiceId UInt32, At DateTime64(3)) ENGINE = MergeTree ORDER BY At`, + ) + yield* db.execute(Db.sql`INSERT INTO services VALUES ('a', 1, 'api'), ('b', 2, 'web')`) + yield* db.execute( + Db.sql`INSERT INTO events VALUES ('a', 1, '2026-01-01 00:00:00.000'), ('b', 1, '2026-01-01 00:00:01.000')`, + ) + sent.length = 0 + const rows = yield* Db.findMany(schema.services, { + where: ($) => [$.OrgId.eq("a")], + with: { events: { with: { service: true } } }, + }).pipe(Effect.provideService(Db.Database, db)) + return { rows, sent } + }).pipe( + Effect.provide(ClickhouseClient.layer({ url: endpoint!, username: user, password, database })), + Effect.ensuring(Effect.orDie(admin.asCommand(admin.unsafe(`DROP DATABASE IF EXISTS ${database} SYNC`)))), + ) + }).pipe(Effect.provide(ClickhouseClient.layer({ url: endpoint!, username: user, password, database: "default" }))), + ) + expect(sent).toHaveLength(3) + expect(rows).toHaveLength(1) + const [api] = rows + expect(api!.Name).toBe("api") + expect(api!.events).toHaveLength(1) + expect(api!.events[0]!.service?.Name).toBe("api") + }) +}) From d08bef864e1b2c92528ba00bfca9c4111b316a2c Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:27:47 +0200 Subject: [PATCH 2/4] docs: relational reads with defineRelations and findMany --- README.md | 1 + docs/README.md | 1 + docs/reference.md | 1 + docs/relations.md | 129 ++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+) create mode 100644 docs/relations.md diff --git a/README.md b/README.md index 4863efe..afa3ed5 100644 --- a/README.md +++ b/README.md @@ -207,6 +207,7 @@ Full guides live in [`docs/`](./docs/README.md): | [Decoding results](./docs/decoding-results.md) | `rowSchema`, `decodeRows`, decode errors | | [Running a query](./docs/running-queries.md) | Executing the SQL with a real client, wire settings, `SETTINGS` | | [Tenant scoping](./docs/tenant-scoping.md) | `tenantColumn`, what marks a query scoped, `crossTenant()` | +| [Relational reads](./docs/relations.md) | `defineRelations`, `findMany` with nested `with` | | [Postgres](./docs/postgres.md) | The Postgres dialect, its column types and functions | | [Schema and migrations](./docs/migrations.md) | DDL from `table`, `effect-orm generate`, applying migrations | | [Extending the DSL](./docs/extending.md) | `defineFn`, raw escape hatches, handwritten SQL | diff --git a/docs/README.md b/docs/README.md index 7a61d11..a150170 100644 --- a/docs/README.md +++ b/docs/README.md @@ -65,6 +65,7 @@ Roughly in reading order. | [Postgres](./postgres.md) | The Postgres dialect, its column types and functions | | [Schema and migrations](./migrations.md) | DDL from `CH.table` / `PG.table`, `materializedView`, `effect-orm generate`, applying migrations, adopting drizzle-kit | | [Statements and transactions](./database.md) | `Database` over your `SqlClient`: `run`, `execute`, `transaction`, retry | +| [Relational reads](./relations.md) | `defineRelations`, `findMany` with nested `with`, tenant scope per level | ## Reference diff --git a/docs/reference.md b/docs/reference.md index abc2d46..f34bc3c 100644 --- a/docs/reference.md +++ b/docs/reference.md @@ -513,6 +513,7 @@ placeholders, double-quoted identifiers. | `run`, `status`, `verify`, `baseline`, `MigrationDriver` and the migrate errors | `/migrate` | | `Database`, `run`, `sql`, `query`, `execute`, `transaction`, `requireTransaction`, `retryContention`, `Transaction` | `/database` | | `DatabaseError`, `TransactionCommitFailed`, `TransactionRollbackFailed` and the other transaction errors | `/database` | +| `defineRelations`, `findMany`, `Relation`, `RelationBuilder`, `RelationsConfig`, `RelationTables`, `RelationalSchema`, `RelationalTable`, `RelationOptions`, `FindManyOptions`, `FindManyRow`, `WithOf`, `KeyColumn` — relational reads, one statement per level ([Relational reads](./relations.md)) | `/database` | | `withQuerySummary`, `QuerySummary` — the `db.query.summary` of the statements an effect runs, for spans | `/database` | | `ClickhouseHttpClient`, `layer`, `make`, `ClickhouseHttpClientConfig` — a ClickHouse `SqlClient` over `fetch`, for Workers | `/clickhouse-http` | | `defineConfig`, `generate`, `check`, `pull`, `loadSchema`, `readMigrations`, `analyze`, `KitError` | `/kit` | diff --git a/docs/relations.md b/docs/relations.md new file mode 100644 index 0000000..173046e --- /dev/null +++ b/docs/relations.md @@ -0,0 +1,129 @@ +# Relational reads + +`Db.defineRelations` declares how tables relate. `Db.findMany` reads rows of one table with the +related rows under them, nested to any depth, as Drizzle's relational queries and Prisma's +`include` do. Both are in `@maple-dev/effect-orm/database`. + +```ts title="relations-find-many.ts" +import { PgliteClient } from "@effect/sql-pglite" +import { Effect, Layer } from "effect" +import * as Db from "@maple-dev/effect-orm/database" +import * as PG from "@maple-dev/effect-orm/postgres" + +const Users = PG.table("users", { columns: { id: PG.int4, name: PG.text }, primaryKey: ["id"] }) +const Posts = PG.table("posts", { + columns: { id: PG.int4, authorId: PG.column(PG.int4, { name: "author_id" }), title: PG.text }, + primaryKey: ["id"], +}) + +export const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +const DatabaseLive = Db.layerSqlClient({ dialect: PG.postgresDialect }).pipe( + Layer.provideMerge(PgliteClient.layer({ postgresqlconf: "timezone = 'UTC'" })), +) + +export const users = Effect.gen(function* () { + yield* Db.execute(Db.sql`CREATE TABLE users (id int4 PRIMARY KEY, name text NOT NULL)`) + yield* Db.execute(Db.sql`CREATE TABLE posts (id int4 PRIMARY KEY, author_id int4 NOT NULL, title text NOT NULL)`) + yield* Db.execute(Db.sql`INSERT INTO users VALUES (1, 'ada'), (2, 'bob')`) + yield* Db.execute(Db.sql`INSERT INTO posts VALUES (10, 1, 'hello')`) + return yield* Db.findMany(schema.users, { + orderBy: ($) => [[$.id, "asc"]], + with: { posts: { with: { author: true } } }, + }) +}).pipe(Effect.provide(DatabaseLive)) +// [ +// { id: 1, name: "ada", posts: [{ id: 10, authorId: 1, title: "hello", author: { id: 1, name: "ada" } }] }, +// { id: 2, name: "bob", posts: [] }, +// ] +``` + +## Declare relations + +`defineRelations(tables, build)` takes the tables by key, and a callback that returns the +relations of each table by name. A relation names its target by that key, so two tables can point +at each other, and a table can point at itself. + +| Builder | Loads | Result | +| --- | --- | --- | +| `r.many(target, { from, to })` | Every `target` row whose `to` column equals this row's `from` column | `ReadonlyArray` | +| `r.one(target, { from, to })` | The first such row | `Row \| null` | + +The types check each declaration. The target must be a key of `tables`. `from` must be a column of +the source table and `to` a column of the target. Both must decode to a `string`, `number` or +`bigint`, because the loader matches keys by value. A relation name must not be a column name of +its table, because the loaded row carries both. + +A `one` relation is always `| null`. A row can be missing even when a foreign key says it cannot: +another statement can delete it between two levels. + +Relations are metadata only. They do not need a foreign key in the database, so they work on +ClickHouse too. + +_(Backed by `src/database/relations.test-d.ts`, and `tests/relations.clickhouse.test.ts` on ClickHouse.)_ + +## Read with `findMany` + +`findMany(schema.table, options?)` returns every column of the table, plus each relation that +`with` names. `with` takes `true`, or the options of that level: + +| Option | Levels | Meaning | +| --- | --- | --- | +| `where` | All | More conditions, ANDed with the relation's key match | +| `orderBy` | All | The order of the rows; nested rows keep it within each parent | +| `with` | All | Relations to load under each row of this level | +| `limit`, `offset` | Top only | Paging of the top level | +| `crossTenant` | Top only | Declare every level `.crossTenant()` | + +Rows come back decoded through the codecs of their columns at every level. A `timestamptz` in a +nested row is a `DateTime.Utc`, not a string. + +## One statement per level + +Each level of `with` is one ordinary builder query: every column of the target table, +`WHERE IN ()`, plus the level's own `where` and +`orderBy`. The loader groups the rows by key and puts them under their parents. A level with no +keys sends no statement. A tree with three levels sends three statements, whatever the number of +rows. + +The levels are separate statements. Outside a transaction, a write from another connection can +land between two of them. Run `findMany` inside [`Db.transaction`](./database.md#transactions) +with `isolationLevel: "repeatable read"` for one snapshot across all levels. + +All keys of a level go into one `IN` list. Postgres allows at most 65,535 bound values in one +statement, so keep the top level paged with `limit`. + +## Tenant scope on every level + +Each level runs through `run`, so each level has its own +[`tenantScope`](./tenant-scoping.md), and `requireTenantScope` checks each one. + +When the parent rows all carry one tenant, and both tables declare a `tenantColumn`, the nested +query also gets ` = `. That pins the level the same way a hand-written +filter does. It also hides a child row whose tenant differs from its parent's. + +A level gets no pin when: + +- the parent table has no `tenantColumn`; +- the parent rows carry more than one tenant. + +That level is then `"cross-tenant"`, and `requireTenantScope` refuses it. There is one exception: +if the relation's `to` column is the target's tenant column and the level has one key, the +one-value `IN` scopes it already. With `crossTenant: true`, every level is declared +cross-tenant and runs. + +_(Backed by `src/database/relations.test.ts`.)_ + +## Not yet + +- Choosing columns per level (`columns: { id: true }`). Every level reads every column. +- A `limit` per parent on a nested level. That needs a window function or a lateral join. +- Composite keys (`from` and `to` with more than one column). +- `findFirst`. Use `findMany` with `limit: 1`. +- Filters and counts over relations (`_count`, `some`, `every`, `none`). +- `param.*` markers in a `where`: `findMany` takes no params, so write values in. + +See [design/relations.md](../design/relations.md) for the design and the alternatives. From 1311ae81acd3c1145c41c3704a9dd6050cc22496 Mon Sep 17 00:00:00 2001 From: Taras Kornichuk Date: Sun, 25 Oct 2026 17:27:47 +0200 Subject: [PATCH 3/4] docs(design): relational reads, one statement per level --- .changeset/relational-reads.md | 5 ++ design/relations.md | 108 +++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 .changeset/relational-reads.md create mode 100644 design/relations.md diff --git a/.changeset/relational-reads.md b/.changeset/relational-reads.md new file mode 100644 index 0000000..a2858dd --- /dev/null +++ b/.changeset/relational-reads.md @@ -0,0 +1,5 @@ +--- +"@maple-dev/effect-orm": minor +--- + +`Db.defineRelations(tables, (r) => ...)` and `Db.findMany(schema.table, { where, orderBy, limit, with })` in `/database` read rows with their related rows, nested to any depth, as Drizzle's relational queries and Prisma's `include` do. `r.one` and `r.many` are checked against the tables. Each level of `with` is one statement, `WHERE IN (...)` over the keys of the level above, decoded through its own row codec, so a nested `timestamptz` is a `DateTime.Utc`. A level whose parent rows carry one tenant is pinned to it, and `requireTenantScope` checks every level. See docs/relations.md and design/relations.md. diff --git a/design/relations.md b/design/relations.md new file mode 100644 index 0000000..a7f5377 --- /dev/null +++ b/design/relations.md @@ -0,0 +1,108 @@ +# Relational reads: `defineRelations` and `findMany` + +Status: first slice built (issue #79; `docs/relations.md`). `one` and `many` relations over +single-column keys, `with` to any depth, `where` and `orderBy` per level, `limit` and `offset` on +the top level. Section 5 lists what is not built. + +## 1. Goal + +Read a row with its related rows in one call, as Drizzle's relational queries +(`db.query.users.findMany({ with: { posts: true } })`) and Prisma's `include` do. Keep what the +rest of the library gives a query: + +- rows decoded through the column codecs, at every level; +- a tenant scope derived for every statement, which `requireTenantScope` checks; +- a result type derived from the `with` tree. + +## 2. The API + +```ts +const schema = Db.defineRelations({ users: Users, posts: Posts }, (r) => ({ + users: { posts: r.many("posts", { from: "id", to: "authorId" }) }, + posts: { author: r.one("users", { from: "authorId", to: "id" }) }, +})) + +Db.findMany(schema.users, { where: ($) => [$.orgId.eq(orgId)], with: { posts: { with: { author: true } } } }) +``` + +- **One callback over all tables**, as Drizzle v2 `defineRelations`. A relation names its target + by key, not by value, so tables that point at each other need no thunks. +- **Relations live beside the tables, not in them.** `table()` stays as it is, and the DDL, the + diff and the importers do not change. A Prisma importer can write the `defineRelations` call. +- **`findMany` lives in `/database`.** It runs more than one statement, so it needs the + `Database` service. `defineRelations` sits next to it, so the dialect entries do not change. +- **Checked in the types.** The target must be a table key, `to` a column of the target, `from` a + column of the source, and both must decode to `string | number | bigint`. A relation name must + not be a column name. + +## 3. One statement per level, not JSON aggregation + +The issue proposed one statement with JSON aggregation (`jsonb_agg` / `json_object`), as Drizzle +RQB v2 does. This slice runs **one statement per level** instead: +`SELECT FROM WHERE IN () [AND ] +[AND ] [ORDER BY ...]`. The loader then groups the rows by key in memory. Prisma does the +same by default. + +Why: + +1. **Decoding stays correct for free.** Each level is an ordinary builder query, decoded by its + own derived row codec. With JSON aggregation, every column comes back in its JSON form, not + its wire form. An `int8` becomes a JSON number and loses precision past 2^53, a `timestamptz` + becomes an ISO string in a different format, `bytea` becomes hex, and a `numeric` can change. + Each column type would need a second "from JSON" codec per dialect, or a cast to text in the + SQL. That is a large change to the type system for one feature. +2. **Tenant scope needs no new analysis.** Each level is a query the existing analysis already + understands. A nested JSON subquery would need correlated-subquery evidence through + `jsonb_agg`, which the analysis does not have yet. +3. **Every dialect.** The SQL is only `IN (...)`, so the same code runs on Postgres, on + ClickHouse, and later on SQLite / D1 and MySQL. JSON aggregation needs a different function + set per dialect, and ClickHouse has no `jsonb_agg`. +4. **The statement count is fixed.** It is the depth of the `with` tree, not the row count, so + there is no N+1. + +The costs: + +- **More round trips.** Three levels are three statements. On a server close to the app this is + small. JSON aggregation is better on a slow link. +- **No single snapshot.** Outside a transaction, a write can land between two levels. The docs + say to use `Db.transaction({ isolationLevel: "repeatable read" })`. `findMany` does not open a + transaction itself: ClickHouse has none, and the caller may already be in one. +- **Long `IN` lists.** All keys of a level are in one list. Postgres has a limit of 65,535 bound + values. Chunking belongs with #57 (parameter limits). + +JSON aggregation can come later as a second strategy behind the same API, per dialect, once the +column types have JSON codecs. The result type and the tenant rule would not change. + +## 4. Tenant scope on nested levels + +Each level goes through `Database.run`, so it is checked like any other statement. A nested +level filters on the parent keys, which says nothing about the tenant. So the loader adds a pin: + +> When the parent table and the target table both declare a `tenantColumn`, and every parent row +> carries the same tenant value `V`, the nested query also gets ` = V`. + +This is sound. The pin is an ordinary `eq` on a literal, so the existing analysis proves +`"single-tenant"`. `V` comes from rows the parent query read, so a level never reads a tenant its +parent did not read. The pin also hides bad data: a child row whose tenant differs from its +parent's. + +No pin when the parent table is untenanted, or the parent rows carry more than one tenant. The +level is then `"cross-tenant"`, and `requireTenantScope` refuses it, which is the safe default. A +level whose `to` column is the target's tenant column, with one key, is scoped by the one-value +`IN` already. `crossTenant: true` declares every level `.crossTenant()`. + +The pin does not pass through an untenanted level: a grandparent's tenant says nothing about a +grandchild reached through a shared table. + +## 5. Not built + +| What | Why not now | +| --- | --- | +| JSON aggregation strategy | Needs JSON codecs per column type (section 3) | +| `columns` per level | Small, but changes the row type of every level; next slice | +| `limit` per parent on a nested level | Needs `row_number()` over a partition, or a lateral join | +| Composite keys | The loader would match tuples; no issue asks for it yet | +| `findFirst` | `findMany` with `limit: 1`; cardinality helpers are #69 | +| `_count`, `some` / `every` / `none` | Issue #80 | +| Params in nested `where` | `findMany` takes no params; values go in as literals | +| SQLite, MySQL, D1 | Not on main yet; the SQL here is plain `IN`, so no dialect work is expected | From f32db0aa0f355adc2bfe549e67ebac917555b31f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Mon, 2 Nov 2026 16:33:16 +0100 Subject: [PATCH 4/4] fix(database): split findMany IN lists to the dialect's bound-value limit A level with more keys than D1's 100 bindings failed to compile. Each level now sends one IN statement per batch that fits maxParameters next to its other bindings. Adds SQLite and D1 coverage and updates the docs. Co-Authored-By: Claude Opus 5.5 --- .changeset/relational-reads.md | 2 +- design/relations.md | 14 +++-- docs/relations.md | 6 +- src/database/relations.ts | 43 +++++++++---- src/sqlite/relations.test.ts | 110 +++++++++++++++++++++++++++++++++ 5 files changed, 155 insertions(+), 20 deletions(-) create mode 100644 src/sqlite/relations.test.ts diff --git a/.changeset/relational-reads.md b/.changeset/relational-reads.md index a2858dd..b6c3d54 100644 --- a/.changeset/relational-reads.md +++ b/.changeset/relational-reads.md @@ -2,4 +2,4 @@ "@maple-dev/effect-orm": minor --- -`Db.defineRelations(tables, (r) => ...)` and `Db.findMany(schema.table, { where, orderBy, limit, with })` in `/database` read rows with their related rows, nested to any depth, as Drizzle's relational queries and Prisma's `include` do. `r.one` and `r.many` are checked against the tables. Each level of `with` is one statement, `WHERE IN (...)` over the keys of the level above, decoded through its own row codec, so a nested `timestamptz` is a `DateTime.Utc`. A level whose parent rows carry one tenant is pinned to it, and `requireTenantScope` checks every level. See docs/relations.md and design/relations.md. +`Db.defineRelations(tables, (r) => ...)` and `Db.findMany(schema.table, { where, orderBy, limit, with })` in `/database` read rows with their related rows, nested to any depth, as Drizzle's relational queries and Prisma's `include` do. `r.one` and `r.many` are checked against the tables. Each level of `with` is one statement, `WHERE IN (...)` over the keys of the level above, decoded through its own row codec, so a nested `timestamptz` is a `DateTime.Utc`. A level whose parent rows carry one tenant is pinned to it, and `requireTenantScope` checks every level. A level whose keys pass the dialect's bound-value limit (D1: 100) is split into several `IN` statements. Works on Postgres, ClickHouse, SQLite and D1. See docs/relations.md and design/relations.md. diff --git a/design/relations.md b/design/relations.md index a7f5377..8cb9799 100644 --- a/design/relations.md +++ b/design/relations.md @@ -55,10 +55,11 @@ Why: understands. A nested JSON subquery would need correlated-subquery evidence through `jsonb_agg`, which the analysis does not have yet. 3. **Every dialect.** The SQL is only `IN (...)`, so the same code runs on Postgres, on - ClickHouse, and later on SQLite / D1 and MySQL. JSON aggregation needs a different function + ClickHouse, SQLite and D1, and later on MySQL. JSON aggregation needs a different function set per dialect, and ClickHouse has no `jsonb_agg`. 4. **The statement count is fixed.** It is the depth of the `with` tree, not the row count, so - there is no N+1. + there is no N+1. The one exception is a level whose keys pass the dialect's bound-value limit + (see "Long `IN` lists" below). The costs: @@ -66,9 +67,10 @@ The costs: small. JSON aggregation is better on a slow link. - **No single snapshot.** Outside a transaction, a write can land between two levels. The docs say to use `Db.transaction({ isolationLevel: "repeatable read" })`. `findMany` does not open a - transaction itself: ClickHouse has none, and the caller may already be in one. -- **Long `IN` lists.** All keys of a level are in one list. Postgres has a limit of 65,535 bound - values. Chunking belongs with #57 (parameter limits). + transaction itself: ClickHouse and D1 have none, and the caller may already be in one. +- **Long `IN` lists.** A level's keys are split into batches that fit the dialect's + `maxParameters` next to the level's other bindings (D1 allows 100), one statement per batch. + Each parent key is in one batch, so the rows of one parent and their order do not change. JSON aggregation can come later as a second strategy behind the same API, per dialect, once the column types have JSON codecs. The result type and the tenant rule would not change. @@ -105,4 +107,4 @@ grandchild reached through a shared table. | `findFirst` | `findMany` with `limit: 1`; cardinality helpers are #69 | | `_count`, `some` / `every` / `none` | Issue #80 | | Params in nested `where` | `findMany` takes no params; values go in as literals | -| SQLite, MySQL, D1 | Not on main yet; the SQL here is plain `IN`, so no dialect work is expected | +| MySQL | Not on main yet; the SQL here is plain `IN`, so no dialect work is expected | diff --git a/docs/relations.md b/docs/relations.md index 173046e..1eb1ea3 100644 --- a/docs/relations.md +++ b/docs/relations.md @@ -93,8 +93,10 @@ The levels are separate statements. Outside a transaction, a write from another land between two of them. Run `findMany` inside [`Db.transaction`](./database.md#transactions) with `isolationLevel: "repeatable read"` for one snapshot across all levels. -All keys of a level go into one `IN` list. Postgres allows at most 65,535 bound values in one -statement, so keep the top level paged with `limit`. +A level's keys go into one `IN` list when they fit the dialect's bound-value limit (Postgres +65,535, SQLite 32,766, D1 100). Past it the level is split into several `IN` statements, each +within the limit, and their rows are joined before grouping, so the result is the same. On D1 +a level with 250 keys sends three statements. Keep the top level paged with `limit`. ## Tenant scope on every level diff --git a/src/database/relations.ts b/src/database/relations.ts index d5fae05..72dcb6e 100644 --- a/src/database/relations.ts +++ b/src/database/relations.ts @@ -7,7 +7,7 @@ // See design/relations.md for why this and not JSON aggregation. import { Effect } from "effect" -import type { CompiledQueryDecodeError } from "../ch/compile" +import { compileCH, type CompiledQueryDecodeError } from "../ch/compile" import type { QueryBuilderError } from "../ch/errors" import type { Condition, Expr } from "../ch/expr" import { from, type ColumnAccessor, type CHQuery } from "../ch/query" @@ -181,6 +181,12 @@ const distinct = (values: ReadonlyArray): Array => [ ...new Set(values.filter((value) => value !== null && value !== undefined)), ] +const chunks = (values: ReadonlyArray, size: number): Array> => { + const out: Array> = [] + for (let i = 0; i < values.length; i += size) out.push(values.slice(i, i + size)) + return out +} + const levelQuery = ( table: AnyTable, level: Level, @@ -232,16 +238,31 @@ const attach = ( ? distinct(rows.map((row) => row[parent.tenantColumn!])) : [] const tenantColumn = target.tenantColumn - const query = levelQuery( - target, - nested, - ($) => [ - $[relation.to].in_(...keys), - tenantColumn !== undefined && tenants.length === 1 ? $[tenantColumn].eq(tenants[0]) : undefined, - ], - crossTenant, - ) - const decoded = (yield* db.run(query as any)) as ReadonlyArray + const queryFor = (batch: ReadonlyArray) => + levelQuery( + target, + nested, + ($) => [ + $[relation.to].in_(...batch), + tenantColumn !== undefined && tenants.length === 1 ? $[tenantColumn].eq(tenants[0]) : undefined, + ], + crossTenant, + ) + // A dialect with a bound-value limit (D1: 100) would refuse a long + // `IN` list. Split the keys so each statement fits next to the + // level's other bindings; each parent key lands in one batch, so + // per-parent order is kept. + let batches: ReadonlyArray> = [keys] + const limit = db.dialect.params._tag === "bind" ? db.dialect.params.maxParameters : undefined + if (limit !== undefined && keys.length > 1) { + const one = yield* compileCH(queryFor(keys.slice(0, 1)) as any, undefined, { dialect: db.dialect }) + const size = Math.max(1, limit - (one.parameters.length - 1)) + if (keys.length > size) batches = chunks(keys, size) + } + const decoded: Array = [] + for (const batch of batches) { + decoded.push(...((yield* db.run(queryFor(batch) as any)) as ReadonlyArray)) + } children = yield* attach(tables, relations, relation.target, decoded, nested, crossTenant) } const byKey = new Map>() diff --git a/src/sqlite/relations.test.ts b/src/sqlite/relations.test.ts new file mode 100644 index 0000000..bd91e45 --- /dev/null +++ b/src/sqlite/relations.test.ts @@ -0,0 +1,110 @@ +// `findMany` on SQLite (node:sqlite) and on a local D1 binding. D1 binds at +// most 100 values per statement, so a level with more keys than that is split +// into several `IN (...)` statements. +import * as D1Client from "@effect/sql-d1/D1Client" +import * as SqliteClient from "@effect/sql-sqlite-node/SqliteClient" +import { Effect } from "effect" +import * as SqlClient from "effect/sql/SqlClient" +import { Miniflare } from "miniflare" +import { expect, it } from "vitest" +import * as Db from "../database" +import * as SQLite from "../sqlite" + +const Units = SQLite.table("units", { id: SQLite.integer, org_id: SQLite.text, name: SQLite.text }, { tenantColumn: "org_id" }) +const Locks = SQLite.table("locks", { id: SQLite.integer, org_id: SQLite.text, unit_id: SQLite.integer }, { tenantColumn: "org_id" }) +const schema = Db.defineRelations({ units: Units, locks: Locks }, (r) => ({ + units: { locks: r.many("locks", { from: "id", to: "unit_id" }) }, + locks: { unit: r.one("units", { from: "unit_id", to: "id" }) }, +})) + +const setup = (db: Db.DatabaseApi, units: number) => + Effect.gen(function* () { + yield* db.execute({ sql: "CREATE TABLE units (id INTEGER PRIMARY KEY, org_id TEXT NOT NULL, name TEXT NOT NULL)" }) + yield* db.execute({ sql: "CREATE TABLE locks (id INTEGER PRIMARY KEY, org_id TEXT NOT NULL, unit_id INTEGER NOT NULL)" }) + const values = Array.from({ length: units }, (_, i) => `(${i + 1}, 'a', 'u${i + 1}')`).join(", ") + yield* db.execute({ sql: `INSERT INTO units VALUES ${values}, (${units + 1}, 'b', 'other')` }) + // Two locks per unit of org a, and one lock of org b pointing at unit 1. + const locks = Array.from({ length: units }, (_, i) => `(${2 * i + 1}, 'a', ${i + 1}), (${2 * i + 2}, 'a', ${i + 1})`).join(", ") + yield* db.execute({ sql: `INSERT INTO locks VALUES ${locks}, (${2 * units + 1}, 'b', 1)` }) + }) + +const read = Db.findMany(schema.units, { + where: ($) => [$.org_id.eq("a")], + orderBy: ($) => [[$.id, "asc"]], + with: { locks: { orderBy: ($) => [[$.id, "desc"]], with: { unit: true } } }, +}) + +it("loads nested relations on SQLite, pinned to the tenant, one statement per level", async () => { + const { units, statements } = await Effect.runPromise( + Effect.gen(function* () { + const statements: Array = [] + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { + dialect: SQLite.sqliteDialect, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + yield* setup(db, 2) + statements.length = 0 + const units = yield* read.pipe(Effect.provideService(Db.Database, db)) + return { units, statements } + }).pipe(Effect.provide(SqliteClient.layer({ filename: ":memory:" }))), + ) + expect(units).toEqual([ + { + id: 1, + org_id: "a", + name: "u1", + locks: [ + { id: 2, org_id: "a", unit_id: 1, unit: { id: 1, org_id: "a", name: "u1" } }, + { id: 1, org_id: "a", unit_id: 1, unit: { id: 1, org_id: "a", name: "u1" } }, + ], + }, + { + id: 2, + org_id: "a", + name: "u2", + locks: [ + { id: 4, org_id: "a", unit_id: 2, unit: { id: 2, org_id: "a", name: "u2" } }, + { id: 3, org_id: "a", unit_id: 2, unit: { id: 2, org_id: "a", name: "u2" } }, + ], + }, + ]) + expect(statements).toHaveLength(3) + expect(statements[1]).toMatch(/"unit_id" IN \(/) + expect(statements[1]).toMatch(/"org_id" = \?/) +}) + +it("splits a level's IN list to fit D1's 100 bound values", async () => { + const mf = new Miniflare({ + modules: true, + script: "export default { fetch() { return new Response('D1 relations test') } }", + compatibilityDate: "2026-07-01", + d1Databases: ["DB"], + }) + try { + const binding = await mf.getD1Database("DB") + const { units, statements } = await Effect.runPromise( + Effect.gen(function* () { + const statements: Array = [] + const db = Db.fromSqlClient(yield* SqlClient.SqlClient, { + dialect: SQLite.d1Dialect, + requireTenantScope: true, + observe: (statement) => Effect.sync(() => void statements.push(statement.sql)), + }) + yield* setup(db, 250) + statements.length = 0 + const units = yield* read.pipe(Effect.provideService(Db.Database, db)) + return { units, statements } + }).pipe(Effect.provide(D1Client.layer({ db: binding }))), + ) + expect(units).toHaveLength(250) + expect(units.every((unit) => unit.locks.length === 2 && unit.locks.every((lock) => lock.unit?.id === unit.id))).toBe(true) + expect(units[0]!.locks.map((lock) => lock.id)).toEqual([2, 1]) + // 250 unit keys: 99 per statement next to the tenant pin is 3 statements; + // the 250 distinct unit_id keys of the locks level split the same way. + expect(statements).toHaveLength(7) + for (const statement of statements) expect((statement.match(/\?/g) ?? []).length).toBeLessThanOrEqual(100) + } finally { + await mf.dispose() + } +}, 60_000)