From 308c36ef7843c5591cf0d00bc6fd64c875c77a47 Mon Sep 17 00:00:00 2001 From: emremy Date: Sun, 3 May 2026 02:44:50 +0300 Subject: [PATCH 1/2] feat(v0.3.0): add unique indexes, by-key helpers, JS array benchmarks, and mutation fixes - implement unique index support with strict integrity guarantees - add findBy, updateBy, deleteBy helpers for stable-key operations - add JS array migration helpers (fromRows, firstWhere, countWhere, exists) - introduce comprehensive JS Array vs ColQL benchmark suite (1k / 100k / 1M) - fix mutation performance issues: - bulk delete compaction instead of per-row deleteAt - avoid unnecessary unique index rebuilds - correct benchmark timing to exclude setup cost - improve unique index correctness and rebuild behavior - expand test coverage for unique indexes, mutations, and helpers - update docs to clarify index guarantees and usage patterns --- README.md | 14 +- benchmarks/array-comparison.mjs | 234 ++++++++++ docs/doc/06-indexing.md | 2 +- docs/doc/08-mutations.md | 6 + docs/doc/10-error-handling.md | 4 + docs/doc/11-serialization.md | 6 +- docs/doc/12-memory-model.md | 4 +- docs/doc/13-performance-and-benchmarks.md | 4 + docs/doc/14-typescript-type-safety.md | 22 + .../15-limitations-and-design-decisions.md | 3 + docs/doc/16-api-reference.md | 28 +- docs/doc/17-unique-indexes.md | 70 +++ package.json | 1 + src/index.ts | 4 +- src/indexing/index-manager.ts | 168 ++++++- src/indexing/unique-index.ts | 86 ++++ src/storage/boolean-column.ts | 44 ++ src/storage/dictionary-column.ts | 45 ++ src/storage/numeric-column.ts | 45 ++ src/table.ts | 412 +++++++++++++++++- src/types.ts | 4 + tests/array-helpers.test.ts | 61 +++ tests/by-key-helpers.test.ts | 72 +++ tests/type-inference.test-d.ts | 61 ++- tests/unique-index-parity.test.ts | 61 +++ tests/unique-index.test.ts | 164 +++++++ 26 files changed, 1608 insertions(+), 17 deletions(-) create mode 100644 benchmarks/array-comparison.mjs create mode 100644 docs/doc/17-unique-indexes.md create mode 100644 src/indexing/unique-index.ts create mode 100644 tests/array-helpers.test.ts create mode 100644 tests/by-key-helpers.test.ts create mode 100644 tests/unique-index-parity.test.ts create mode 100644 tests/unique-index.test.ts diff --git a/README.md b/README.md index d888f88..538e531 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ It is not a SQL database or persistence layer. ColQL is for data you already wan - Lazy queries with filtering, projection, aggregation, streaming, limit, and offset - Object predicates plus tuple-style `where(column, operator, value)` - Explicit equality indexes and sorted numeric indexes for hot predicates +- Unique indexes for stable ID lookups and duplicate-key protection +- JS Array migration helpers such as `fromRows`, `firstWhere`, `countWhere`, and `exists` - Mutable tables with `updateMany` and `deleteMany` - Runtime validation with structured `ColQLError` codes - Binary serialization for table data @@ -48,6 +50,7 @@ users.insertMany([ users.createIndex("status"); users.createSortedIndex("age"); +users.createUniqueIndex("id"); const activeAdults = users .where({ @@ -64,6 +67,8 @@ const result = users.updateMany( console.log(activeAdults); console.log(result.affectedRows); + +console.log(users.findBy("id", 1)); ``` ## Performance Snapshot @@ -88,6 +93,7 @@ npm run test:large ``` For benchmark scripts and interpretation notes, see [Performance and Benchmarks](./docs/doc/13-performance-and-benchmarks.md). +For JS Array comparisons, run `npm run benchmark:array-comparison`; results are local guidance, not universal promises. ## When To Use ColQL @@ -98,12 +104,14 @@ Use ColQL when: - filters and aggregations should avoid intermediate arrays - a TypeScript schema can describe your columns - explicit indexes are acceptable for hot equality or range predicates +- stable identity can be modeled with an explicit ID column and unique index - runtime validation matters because data may come from untyped sources Avoid ColQL when: - you need durable storage, transactions, joins, or SQL - row indexes must be stable external identifiers +- a small/simple JavaScript array is already clear and fast enough - every query requires arbitrary sorting or grouping - you need concurrent writers or multi-process coordination - you want automatic indexes, compound indexes, or query planning across tables @@ -132,6 +140,7 @@ Recommended reading: - [Querying](./docs/doc/04-querying.md) - [Equality Indexes](./docs/doc/06-indexing.md) - [Sorted Indexes](./docs/doc/07-sorted-indexes.md) +- [Unique Indexes](./docs/doc/17-unique-indexes.md) - [Mutations](./docs/doc/08-mutations.md) - [Serialization](./docs/doc/11-serialization.md) - [Memory Model](./docs/doc/12-memory-model.md) @@ -159,6 +168,8 @@ users.deleteMany({ status: "archived" }); users.createIndex("id"); users.createSortedIndex("age"); +users.createUniqueIndex("id"); +users.findBy("id", 123); const buffer = users.serialize(); const restored = table.deserialize(buffer); @@ -195,6 +206,7 @@ npm run benchmark:range npm run benchmark:optimizer npm run benchmark:serialization npm run benchmark:delete +npm run benchmark:array-comparison ``` ## Status @@ -203,4 +215,4 @@ ColQL v0.2.x aims to keep the public API reasonably stable, but breaking changes ## Limitations -ColQL intentionally does not include SQL parsing, joins, transactions, concurrency control, automatic indexes, compound indexes, or durable storage. Indexes are derived performance structures; query results must be the same whether ColQL uses an index or a full scan. +ColQL intentionally does not include SQL parsing, joins, transactions, concurrency control, automatic indexes, compound indexes, or durable storage. Equality and sorted indexes are derived performance structures; query results must be the same whether ColQL uses an index or a full scan. Unique indexes are derived too, but they also enforce uniqueness while present and are not serialized. diff --git a/benchmarks/array-comparison.mjs b/benchmarks/array-comparison.mjs new file mode 100644 index 0000000..0f1dbda --- /dev/null +++ b/benchmarks/array-comparison.mjs @@ -0,0 +1,234 @@ +import os from "node:os"; +import { column, fromRows, table } from "../dist/index.mjs"; + +const DEFAULT_SIZES = [1_000, 100_000, 1_000_000]; +const RUNS = 3; + +const sizes = process.env.COLQL_ARRAY_BENCH_SIZES + ? process.env.COLQL_ARRAY_BENCH_SIZES.split(",").map((value) => Number.parseInt(value.trim(), 10)) + : DEFAULT_SIZES; +const jsonOutput = process.argv.includes("--json"); + +for (const size of sizes) { + if (!Number.isInteger(size) || size < 1) { + throw new Error(`Invalid benchmark size: ${String(size)}`); + } +} + +const schema = { + id: column.uint32(), + age: column.uint8(), + score: column.uint32(), + status: column.dictionary(["active", "passive", "archived"]), + active: column.boolean(), +}; + +function mb(bytes) { + return bytes / 1024 / 1024; +} + +function memoryTotal() { + const usage = process.memoryUsage(); + return usage.heapUsed + usage.arrayBuffers; +} + +function forceGc() { + if (global.gc) { + global.gc(); + } +} + +function createRows(rowCount) { + return Array.from({ length: rowCount }, (_unused, id) => ({ + id, + age: (id * 7) % 100, + score: (id * 13) % 10_000, + status: id % 3 === 0 ? "active" : id % 3 === 1 ? "passive" : "archived", + active: id % 2 === 0, + })); +} + +function createTable(rows, indexes = "none") { + const users = fromRows(schema, rows); + if (indexes === "equality") { + users.createIndex("id").createIndex("status"); + } + if (indexes === "sorted") { + users.createSortedIndex("age"); + } + if (indexes === "unique") { + users.createUniqueIndex("id"); + } + return users; +} + +function time(fn) { + const start = performance.now(); + const result = fn(); + return { duration: performance.now() - start, result }; +} + +function average(values) { + return values.reduce((total, value) => total + value, 0) / values.length; +} + +function median(values) { + const sorted = [...values].sort((left, right) => left - right); + return sorted[Math.floor(sorted.length / 2)]; +} + +function measureMemory(label, factory) { + forceGc(); + const before = memoryTotal(); + const value = factory(); + forceGc(); + const after = memoryTotal(); + return { label, ms: 0, memoryMB: Math.max(0, mb(after - before)), value }; +} + +function runWorkloads(rows) { + const rowCount = rows.length; + const targetId = rowCount - 10; + const broadAge = 10; + const selectiveAge = 95; + const results = []; + + const memoryArray = measureMemory("memory: JS object array", () => createRows(rowCount)); + results.push({ label: memoryArray.label, ms: 0, memoryMB: memoryArray.memoryMB }); + memoryArray.value.length = 0; + + const memoryColql = measureMemory("memory: ColQL scan table", () => createTable(rows)); + results.push({ label: memoryColql.label, ms: 0, memoryMB: memoryColql.memoryMB }); + + const buildFromRows = time(() => createTable(rows).rowCount); + results.push({ label: "fromRows / insertMany", ms: buildFromRows.duration }); + if (buildFromRows.result !== rowCount) throw new Error("fromRows sanity check failed."); + + const scan = createTable(rows); + const equality = createTable(rows, "equality"); + const sorted = createTable(rows, "sorted"); + const unique = createTable(rows, "unique"); + + const workloads = [ + { label: "filter/count: JS array", fn: () => rows.filter((row) => row.status === "active" && row.age >= 18).length }, + { label: "filter/count: ColQL scan", fn: () => scan.where({ status: "active", age: { gte: 18 } }).count() }, + { label: "filter/count: ColQL equality index", fn: () => equality.where({ status: "active", age: { gte: 18 } }).count() }, + [ + "projection+limit: JS array", + () => + rows + .filter((row) => row.status === "active" && row.age >= 18) + .slice(0, 100) + .map((row) => ({ id: row.id, age: row.age })).length, + ], + [ + "projection+limit: ColQL", + () => scan.where({ status: "active", age: { gte: 18 } }).select(["id", "age"]).limit(100).toArray().length, + ], + { label: "find by id: JS array", fn: () => rows.find((row) => row.id === targetId)?.id }, + { label: "find by id: ColQL equality index", fn: () => equality.where("id", "=", targetId).first()?.id }, + { label: "unique lookup findBy", fn: () => unique.findBy("id", targetId)?.id }, + { label: "exists: JS array", fn: () => rows.some((row) => row.id === targetId) }, + { label: "exists: ColQL helper", fn: () => equality.exists("id", "=", targetId) }, + { label: "countWhere: JS array", fn: () => rows.filter((row) => row.status === "passive").length }, + { label: "countWhere: ColQL helper", fn: () => equality.countWhere({ status: "passive" }) }, + { label: "range query: JS array", fn: () => rows.filter((row) => row.age >= selectiveAge).length }, + { label: "range query: ColQL scan", fn: () => scan.where("age", ">=", selectiveAge).count() }, + { label: "range query: ColQL sorted index", fn: () => sorted.where("age", ">=", selectiveAge).count() }, + { label: "broad scan: JS array", fn: () => rows.filter((row) => row.age >= broadAge).length }, + { label: "broad scan: ColQL", fn: () => sorted.where("age", ">=", broadAge).count() }, + { label: "callback filter(fn): JS array", fn: () => rows.filter((row) => row.active && row.score % 7 === 0).length }, + { label: "callback filter(fn): ColQL", fn: () => scan.filter((row) => row.active && row.score % 7 === 0).count() }, + [ + "update by predicate: JS array", + () => rows.map((row) => (row.status === "archived" ? { ...row, status: "passive" } : row)).filter((row) => row.status === "passive").length, + ], + { + label: "update by predicate: ColQL", + prepare: () => createTable(rows), + fn: (users) => { + users.updateMany({ status: "archived" }, { status: "passive" }); + return users.where("status", "=", "passive").count(); + }, + }, + [ + "delete by predicate: JS array", + () => rows.filter((row) => row.age < selectiveAge).length, + ], + { + label: "delete by predicate: ColQL", + prepare: () => createTable(rows), + fn: (users) => { + users.deleteMany({ age: { gte: selectiveAge } }); + return users.rowCount; + }, + }, + { + label: "updateBy/deleteBy: unique index", + prepare: () => createTable(rows, "unique"), + fn: (users) => { + users.updateBy("id", targetId, { score: 123 }); + users.deleteBy("id", targetId); + return users.findBy("id", targetId); + }, + }, + ]; + + const expected = new Map(); + for (const workload of workloads) { + const normalizedWorkload = Array.isArray(workload) + ? { label: workload[0], fn: workload[1] } + : workload; + const runs = Array.from({ length: RUNS }, () => { + const input = normalizedWorkload.prepare?.(); + return time(() => normalizedWorkload.fn(input)); + }); + const normalized = runs.map((run) => (run.result === undefined ? "undefined" : JSON.stringify(run.result))); + if (new Set(normalized).size !== 1) { + throw new Error(`Benchmark sanity check failed for ${normalizedWorkload.label}.`); + } + + const group = normalizedWorkload.label.replace(/: (JS array|ColQL.*|unique index)$/, ""); + const value = normalized[0]; + if (expected.has(group) && expected.get(group) !== value) { + throw new Error(`Array and ColQL results differ for ${group}.`); + } + expected.set(group, value); + + results.push({ label: normalizedWorkload.label, ms: average(runs.map((run) => run.duration)), medianMs: median(runs.map((run) => run.duration)) }); + } + + return results; +} + +function printHuman(allResults) { + console.log("JS Array vs ColQL comparison benchmark"); + console.log(`Node ${process.version} on ${process.platform} ${process.arch}`); + console.log(`CPU: ${os.cpus()[0]?.model ?? "unknown"} (${os.cpus().length} logical cores)`); + console.log("Caveats: local machine only; results vary with Node version, CPU, memory pressure, data distribution, selectivity, and workload shape."); + console.log("These numbers are not CI requirements or universal guarantees.\n"); + + for (const group of allResults) { + console.log(`${group.rows.toLocaleString()} rows, average over ${RUNS} runs`); + console.log("workload avg ms median ms memory MB"); + console.log("--------------------------------------------------------------------------"); + for (const result of group.results) { + const ms = result.ms.toFixed(3).padStart(8); + const medianMs = (result.medianMs ?? result.ms).toFixed(3).padStart(9); + const memory = result.memoryMB === undefined ? "".padStart(9) : result.memoryMB.toFixed(2).padStart(9); + console.log(`${result.label.padEnd(44)} ${ms} ${medianMs} ${memory}`); + } + console.log(""); + } +} + +const allResults = sizes.map((rows) => { + const sourceRows = createRows(rows); + return { rows, results: runWorkloads(sourceRows) }; +}); + +if (jsonOutput) { + console.log(JSON.stringify({ env: { node: process.version, platform: process.platform, arch: process.arch, cpu: os.cpus()[0]?.model }, results: allResults }, null, 2)); +} else { + printHuman(allResults); +} diff --git a/docs/doc/06-indexing.md b/docs/doc/06-indexing.md index 391f1d8..97209e4 100644 --- a/docs/doc/06-indexing.md +++ b/docs/doc/06-indexing.md @@ -1,6 +1,6 @@ # Equality Indexes -Equality indexes are optional derived performance structures for selective equality and membership queries. A query must return the same result whether ColQL uses an index or a full scan. +Equality indexes are optional derived performance structures for selective equality and membership queries. A query must return the same result whether ColQL uses an index or a full scan. Unique indexes are separate integrity indexes; see [Unique Indexes](./17-unique-indexes.md). ```ts users.createIndex("id"); diff --git a/docs/doc/08-mutations.md b/docs/doc/08-mutations.md index 97840c2..c4ce6ef 100644 --- a/docs/doc/08-mutations.md +++ b/docs/doc/08-mutations.md @@ -14,6 +14,9 @@ users.deleteMany(predicate); users.updateWhere(column, operator, value, partialRow); users.deleteWhere(column, operator, value); + +users.updateBy(uniqueColumn, value, partialRow); +users.deleteBy(uniqueColumn, value); ``` New mutation APIs return: @@ -111,6 +114,7 @@ ColQL applies mutation safety rules internally: - matching row indexes are snapshotted before predicate mutation - update input is validated before writing to storage - predicate updates are all-or-nothing for validation +- unique-index violations are checked before writing and keep bulk updates all-or-nothing - predicate deletes delete matched row indexes from highest to lowest - no-match predicate update/delete returns `{ affectedRows: 0 }` - nonzero update/delete mutations mark existing indexes dirty @@ -118,6 +122,8 @@ ColQL applies mutation safety rules internally: Dirty indexes are rebuilt before an indexed query uses them, so index dirtiness affects rebuild cost, not query correctness. +Unique indexes are stricter than equality and sorted indexes. They enforce uniqueness for indexed columns and reject duplicate-producing inserts or updates with `COLQL_DUPLICATE_KEY`. Deletes free unique keys for reuse. + Snapshotting matters when an update changes the predicate column: ```ts diff --git a/docs/doc/10-error-handling.md b/docs/doc/10-error-handling.md index 41f096e..4b32f4e 100644 --- a/docs/doc/10-error-handling.md +++ b/docs/doc/10-error-handling.md @@ -61,6 +61,10 @@ Index errors: - `COLQL_SORTED_INDEX_EXISTS` - `COLQL_SORTED_INDEX_NOT_FOUND` - `COLQL_SORTED_INDEX_UNSUPPORTED_COLUMN` +- `COLQL_DUPLICATE_KEY` +- `COLQL_UNIQUE_INDEX_EXISTS` +- `COLQL_UNIQUE_INDEX_NOT_FOUND` +- `COLQL_UNIQUE_INDEX_UNSUPPORTED` Serialization errors: diff --git a/docs/doc/11-serialization.md b/docs/doc/11-serialization.md index ad4caa2..c50ec7d 100644 --- a/docs/doc/11-serialization.md +++ b/docs/doc/11-serialization.md @@ -26,16 +26,18 @@ Indexes are not serialized: - equality indexes - sorted indexes +- unique indexes -They are derived performance data and can be rebuilt after deserialization. Recreating indexes after deserialization affects performance only, not query correctness. +They are derived data and can be rebuilt after deserialization. Recreating equality and sorted indexes affects performance only, not query correctness. Recreating unique indexes also restores uniqueness enforcement. ```ts const restored = table.deserialize(buffer); restored.createIndex("id"); restored.createSortedIndex("age"); +restored.createUniqueIndex("id"); ``` -`restored.indexes()` and `restored.sortedIndexes()` are empty until indexes are recreated. +`restored.indexes()`, `restored.sortedIndexes()`, and `restored.uniqueIndexes()` are empty until indexes are recreated. ## After Mutations and Deletes diff --git a/docs/doc/12-memory-model.md b/docs/doc/12-memory-model.md index b922ff2..24aadb3 100644 --- a/docs/doc/12-memory-model.md +++ b/docs/doc/12-memory-model.md @@ -40,12 +40,14 @@ Indexes are separate derived performance structures: - equality indexes store row-position buckets by value - sorted indexes store row positions sorted by numeric value +- unique indexes store one row position per unique key and also enforce uniqueness -Indexes improve selected query shapes but increase memory. They do not change query correctness; the same query must return the same result through an index or a full scan. Drop indexes if memory matters more than indexed lookup speed: +Indexes improve selected query shapes but increase memory. Equality and sorted indexes do not change query correctness; the same query must return the same result through an index or a full scan. Unique indexes are different: they are still derived structures, but they also reject duplicate keys and support by-key helpers. ```ts users.dropIndex("status"); users.dropSortedIndex("age"); +users.dropUniqueIndex("id"); ``` ## Materialization diff --git a/docs/doc/13-performance-and-benchmarks.md b/docs/doc/13-performance-and-benchmarks.md index 7279852..d14a1eb 100644 --- a/docs/doc/13-performance-and-benchmarks.md +++ b/docs/doc/13-performance-and-benchmarks.md @@ -18,6 +18,7 @@ npm run benchmark:range npm run benchmark:optimizer npm run benchmark:serialization npm run benchmark:delete +npm run benchmark:array-comparison ``` Most benchmark scripts accept larger scenarios with: @@ -36,6 +37,7 @@ COLQL_BENCH_LARGE=1 npm run benchmark:indexed - `benchmark:serialization`: serialize/deserialize timing and output size. - `benchmark:delete`: physical delete, update, dirty index rebuild, and memory phases. - `benchmark:physical-delete`: focused physical-delete behavior. +- `benchmark:array-comparison`: JS object arrays versus ColQL scan, equality, sorted, and unique-index paths across common workloads. ## Memory Metrics @@ -103,4 +105,6 @@ Measure the exact workload you care about: - mutation frequency - index lifecycle +For small/simple data, a JavaScript array can be the better tool. ColQL becomes more useful when memory layout, structured predicates, or explicit indexed lookups matter. + See [Memory Model](./12-memory-model.md) for memory tradeoffs. diff --git a/docs/doc/14-typescript-type-safety.md b/docs/doc/14-typescript-type-safety.md index 6758a4a..6c77b34 100644 --- a/docs/doc/14-typescript-type-safety.md +++ b/docs/doc/14-typescript-type-safety.md @@ -134,6 +134,28 @@ const result: MutationResult = users.deleteWhere("status", "=", "passive"); console.log(result.affectedRows); ``` +## Unique and Migration Helper Typing + +Unique indexes accept numeric and dictionary columns. Boolean columns are rejected by the TypeScript surface and by runtime validation: + +```ts +users.createUniqueIndex("id"); +users.findBy("id", 123); +users.updateBy("id", 123, { status: "active" }); +users.deleteBy("id", 123); +``` + +JS Array migration helpers keep the same schema-derived row and predicate typing: + +```ts +const users = fromRows(schema, rows); +users.firstWhere({ status: "active" }); +users.countWhere("age", ">=", 18); +users.exists((row) => row.is_active); +``` + +Structured helper predicates use `where(...)`; callback predicates use `filter(fn)` and are full scans. + ## Type Tests The repository includes `tests/type-inference.test-d.ts` with `@ts-expect-error` examples. These are useful references for the intended type surface. diff --git a/docs/doc/15-limitations-and-design-decisions.md b/docs/doc/15-limitations-and-design-decisions.md index a226396..e446686 100644 --- a/docs/doc/15-limitations-and-design-decisions.md +++ b/docs/doc/15-limitations-and-design-decisions.md @@ -17,6 +17,7 @@ ColQL aims to keep the public API reasonably stable, but breaking changes may st - concurrency control - durable storage - serialized indexes +- compound unique indexes ## Why These Limits Exist @@ -46,6 +47,8 @@ const users = table({ Equality and sorted indexes are optional derived structures and are not serialized. They affect performance only, not correctness. A query must return the same result through an index or a full scan. +Unique indexes are also derived and not serialized, but they are integrity constraints as well as lookup structures. Recreate them after deserialization when uniqueness enforcement or by-key helpers are needed. + Dirty indexes are rebuilt before use or explicitly by the user. This avoids complex incremental row-position maintenance, especially around physical deletes. ## Mutation Semantics Are Safety-Oriented diff --git a/docs/doc/16-api-reference.md b/docs/doc/16-api-reference.md index 51060cf..3c914ee 100644 --- a/docs/doc/16-api-reference.md +++ b/docs/doc/16-api-reference.md @@ -5,7 +5,7 @@ This is a factual summary of the public API. See the topic docs for deeper behav ## Imports ```ts -import { table, column, ColQLError } from "@colql/colql"; +import { table, column, fromRows, ColQLError } from "@colql/colql"; import type { MutationResult, ObjectWherePredicate, @@ -23,11 +23,13 @@ import type { ```ts const users = table(schema); +const loaded = fromRows(schema, rows); const instrumented = table(schema, { onQuery: (info) => console.log(info) }); const restored = table.deserialize(buffer); ``` `table(schema)` returns a `Table` instance. +`fromRows(schema, rows, options?)` creates a table and inserts rows with `insertMany`. `table(schema, options)` accepts compatible table options such as `onQuery`. `table.deserialize(input)` accepts an `ArrayBuffer` or `Uint8Array` and returns a table. @@ -95,6 +97,9 @@ users.where(objectPredicate); users.whereIn(column, values); users.whereNotIn(column, values); users.filter(callback); +users.firstWhere(predicate); +users.countWhere(predicate); +users.exists(predicate); users.select(columns); users.limit(n); users.offset(n); @@ -109,6 +114,7 @@ users.filter((row) => row.age > 25); ``` `where(objectPredicate)` is structured predicate syntax and may use indexes. `filter(callback)` is a full-scan callback escape hatch, runs after structured predicates, and is not index-aware. +`firstWhere`, `countWhere`, and `exists` are table-level wrappers over structured `where(...)` or callback `filter(fn)`. Operators: @@ -247,6 +253,24 @@ users.rebuildIndexes(); // this Sorted indexes are numeric range indexes. They are derived performance structures and are rebuilt before use when dirty. +## Unique Indexes + +```ts +users.createUniqueIndex(column); // this +users.dropUniqueIndex(column); // this +users.hasUniqueIndex(column); // boolean +users.uniqueIndexes(); // string[] +users.uniqueIndexStats(); // UniqueIndexStats[] +users.rebuildUniqueIndex(column); // this +users.rebuildUniqueIndexes(); // this + +users.findBy(column, value); // row | undefined +users.updateBy(column, value, partialRow); // MutationResult +users.deleteBy(column, value); // MutationResult +``` + +Unique indexes support numeric and dictionary columns. They are derived structures, not serialized, and enforce uniqueness while present. By-key helpers require an existing unique index and do not scan when one is missing. + ## Serialization ```ts @@ -254,7 +278,7 @@ const buffer = users.serialize(); // ArrayBuffer const restored = table.deserialize(buffer); ``` -`deserialize` accepts `ArrayBuffer` or `Uint8Array`. Indexes are not serialized; recreate equality and sorted indexes after deserialization when indexed performance is needed. +`deserialize` accepts `ArrayBuffer` or `Uint8Array`. Indexes are not serialized; recreate equality, sorted, and unique indexes after deserialization when indexed performance or uniqueness enforcement is needed. ## Diagnostics diff --git a/docs/doc/17-unique-indexes.md b/docs/doc/17-unique-indexes.md new file mode 100644 index 0000000..454208b --- /dev/null +++ b/docs/doc/17-unique-indexes.md @@ -0,0 +1,70 @@ +# Unique Indexes + +Unique indexes are derived lookup structures that also enforce data integrity. This is the key difference from equality and sorted indexes: equality and sorted indexes affect performance only, while unique indexes reject duplicate keys. + +```ts +users.createUniqueIndex("id"); + +const user = users.findBy("id", 123); +users.updateBy("id", 123, { status: "active" }); +users.deleteBy("id", 123); +``` + +## API + +```ts +users.createUniqueIndex("id"); +users.dropUniqueIndex("id"); +users.hasUniqueIndex("id"); +users.uniqueIndexes(); +users.uniqueIndexStats(); +users.rebuildUniqueIndex("id"); +users.rebuildUniqueIndexes(); + +users.findBy("id", 123); +users.updateBy("id", 123, { status: "active" }); +users.deleteBy("id", 123); +``` + +Unique indexes support numeric and dictionary columns. Boolean columns are not supported and throw `COLQL_UNIQUE_INDEX_UNSUPPORTED`. + +## Guarantees + +Once a unique index exists, duplicate keys for that column are rejected: + +- `createUniqueIndex()` scans existing rows and throws if duplicates already exist. +- `insert()` rejects duplicate keys. +- `insertMany()` rejects duplicates against existing rows and within the input batch. +- `update()` and predicate updates reject duplicate-producing changes. +- failed bulk insert/update operations are all-or-nothing. +- deletes free keys for reuse. +- rebuilds detect duplicates and fail atomically. + +Duplicate violations throw `COLQL_DUPLICATE_KEY` with details such as `columnName`, `encodedValue`, and row/input positions when available. + +## By-Key Helpers + +`findBy`, `updateBy`, and `deleteBy` require a unique index and do not scan when one is missing. Missing unique indexes throw `COLQL_UNIQUE_INDEX_NOT_FOUND`. + +Missing keys are not errors: + +```ts +users.findBy("id", 999); // undefined +users.updateBy("id", 999, { age: 30 }); // { affectedRows: 0 } +users.deleteBy("id", 999); // { affectedRows: 0 } +``` + +Row indexes remain unstable physical positions. Use an explicit ID column plus a unique index for stable identity. + +## Dirty Rebuilds and Serialization + +Unique indexes store row positions internally, so deletes and updates can make them dirty. Dirty unique indexes are rebuilt before by-key lookup or stats so stale row positions are not returned. + +Unique indexes are not serialized: + +```ts +const restored = table.deserialize(users.serialize()); +restored.createUniqueIndex("id"); +``` + +See [Mutations](./08-mutations.md), [Serialization](./11-serialization.md), and [Memory Model](./12-memory-model.md). diff --git a/package.json b/package.json index 6e03a13..b2bcec2 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,7 @@ "benchmark:range": "node benchmarks/range-query.mjs", "benchmark:optimizer": "node benchmarks/query-optimizer.mjs", "benchmark:serialization": "node benchmarks/serialization.mjs", + "benchmark:array-comparison": "node --expose-gc benchmarks/array-comparison.mjs", "prepublishOnly": "npm run build", "benchmark:physical-delete": "node --expose-gc benchmarks/physical-delete.mjs", "benchmark:delete": "node --expose-gc benchmarks/delete.mjs" diff --git a/src/index.ts b/src/index.ts index f1ad9da..ad18c17 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,6 +1,7 @@ export { column } from "./column"; -export { table } from "./table"; +export { fromRows, table } from "./table"; export { ColQLError } from "./errors"; +export type { UniqueIndexStats } from "./indexing/unique-index"; export type { BooleanWherePredicate, DictionaryWherePredicate, @@ -14,4 +15,5 @@ export type { RowForSchema, Schema, TableOptions, + UniqueColumnKey, } from "./types"; diff --git a/src/indexing/index-manager.ts b/src/indexing/index-manager.ts index e6d9ced..93ed214 100644 --- a/src/indexing/index-manager.ts +++ b/src/indexing/index-manager.ts @@ -2,6 +2,7 @@ import { ColQLError } from "../errors"; import type { ColumnDefinition } from "../types"; import { EqualityIndex, type EqualityIndexStats, type IndexableValue } from "./equality-index"; import { SortedIndex, type RangeOperator, type SortedIndexStats } from "./sorted-index"; +import { UniqueIndex, type UniqueIndexStats, type UniqueIndexValue } from "./unique-index"; const DEFAULT_INDEX_SELECTIVITY_THRESHOLD = 0.4; @@ -65,6 +66,7 @@ type CandidateEstimate = EqualityCandidateEstimate | SortedCandidateEstimate; export class IndexManager { private readonly indexesByColumn = new Map(); private readonly sortedIndexesByColumn = new Map(); + private readonly uniqueIndexesByColumn = new Map(); private equalityDirty = false; create( @@ -179,6 +181,99 @@ export class IndexManager { .filter((stats): stats is SortedIndexStats => stats !== undefined); } + createUnique( + columnName: string, + definition: ColumnDefinition, + rowCount: number, + readComparableValue: (rowIndex: number, columnName: string) => number | boolean, + ): void { + if (this.uniqueIndexesByColumn.has(columnName)) { + throw new ColQLError("COLQL_UNIQUE_INDEX_EXISTS", `Unique index already exists for column "${columnName}".`); + } + + this.assertUniqueSupported(columnName, definition); + this.uniqueIndexesByColumn.set(columnName, this.buildUniqueIndex(columnName, rowCount, readComparableValue)); + } + + dropUnique(columnName: string): void { + if (!this.uniqueIndexesByColumn.delete(columnName)) { + throw new ColQLError("COLQL_UNIQUE_INDEX_NOT_FOUND", `Unique index not found for column "${columnName}".`); + } + } + + hasUnique(columnName: string): boolean { + return this.uniqueIndexesByColumn.has(columnName); + } + + listUnique(): string[] { + return [...this.uniqueIndexesByColumn.keys()]; + } + + uniqueStats( + rowCount: number, + readComparableValue: (rowIndex: number, columnName: string) => number | boolean, + ): UniqueIndexStats[] { + this.rebuildUniqueIfDirty(rowCount, readComparableValue); + return this.listUnique() + .map((columnName) => this.uniqueIndexesByColumn.get(columnName)?.stats()) + .filter((stats): stats is UniqueIndexStats => stats !== undefined); + } + + rebuildUnique( + columnName: string, + rowCount: number, + readComparableValue: (rowIndex: number, columnName: string) => number | boolean, + ): void { + if (!this.uniqueIndexesByColumn.has(columnName)) { + throw new ColQLError("COLQL_UNIQUE_INDEX_NOT_FOUND", `Unique index not found for column "${columnName}".`); + } + + this.uniqueIndexesByColumn.set(columnName, this.buildUniqueIndex(columnName, rowCount, readComparableValue)); + } + + rebuildUniqueIndexes( + rowCount: number, + readComparableValue: (rowIndex: number, columnName: string) => number | boolean, + ): void { + const columns = this.listUnique(); + const next = new Map(); + for (const column of columns) { + next.set(column, this.buildUniqueIndex(column, rowCount, readComparableValue)); + } + + this.uniqueIndexesByColumn.clear(); + for (const [column, index] of next) { + this.uniqueIndexesByColumn.set(column, index); + } + } + + addUniqueRow(columnName: string, value: number | boolean, rowIndex: number): void { + const index = this.uniqueIndexesByColumn.get(columnName); + if (index === undefined || index.isDirty()) { + return; + } + + index.add(value as UniqueIndexValue, rowIndex); + } + + uniqueLookup( + columnName: string, + value: number, + rowCount: number, + readComparableValue: (rowIndex: number, columnName: string) => number | boolean, + ): number | undefined { + const index = this.uniqueIndexesByColumn.get(columnName); + if (index === undefined) { + throw new ColQLError("COLQL_UNIQUE_INDEX_NOT_FOUND", `Unique index not found for column "${columnName}".`); + } + + if (index.isDirty()) { + this.rebuildUnique(columnName, rowCount, readComparableValue); + } + + return this.uniqueIndexesByColumn.get(columnName)?.get(value); + } + rebuildSorted( columnName: string, rowCount: number, @@ -199,7 +294,7 @@ export class IndexManager { } } - markDirty(): void { + markPerformanceDirty(): void { if (this.indexesByColumn.size > 0) { this.equalityDirty = true; } @@ -207,6 +302,30 @@ export class IndexManager { this.markSortedDirty(); } + markUniqueDirty(columns?: readonly string[]): void { + const indexes = columns === undefined + ? [...this.uniqueIndexesByColumn.values()] + : columns + .map((column) => this.uniqueIndexesByColumn.get(column)) + .filter((index): index is UniqueIndex => index !== undefined); + + for (const index of indexes) { + index.markDirty(); + } + } + + markDeletedRow(rowIndex: number): void { + this.markPerformanceDirty(); + for (const index of this.uniqueIndexesByColumn.values()) { + index.deleteRow(rowIndex); + } + } + + markDirty(): void { + this.markPerformanceDirty(); + this.markUniqueDirty(); + } + bestCandidate( filters: readonly IndexFilter[], rowCount: number, @@ -374,6 +493,43 @@ export class IndexManager { this.equalityDirty = false; } + private rebuildUniqueIfDirty( + rowCount: number, + readComparableValue: (rowIndex: number, columnName: string) => number | boolean, + ): void { + if (![...this.uniqueIndexesByColumn.values()].some((index) => index.isDirty())) { + return; + } + + this.rebuildUniqueIndexes(rowCount, readComparableValue); + } + + private buildUniqueIndex( + columnName: string, + rowCount: number, + readComparableValue: (rowIndex: number, columnName: string) => number | boolean, + ): UniqueIndex { + const index = new UniqueIndex(columnName); + for (let rowIndex = 0; rowIndex < rowCount; rowIndex += 1) { + try { + index.add(readComparableValue(rowIndex, columnName) as UniqueIndexValue, rowIndex); + } catch (error) { + if (error instanceof ColQLError && error.code === "COLQL_DUPLICATE_KEY") { + throw new ColQLError( + "COLQL_DUPLICATE_KEY", + `Duplicate key found while building unique index for column "${columnName}".`, + { ...(error.details as object), columnName, operation: "rebuildUniqueIndex" }, + ); + } + + throw error; + } + } + + index.markFresh(); + return index; + } + private rebuildSortedIndexes( rowCount: number, readNumericValue: (rowIndex: number, columnName: string) => number, @@ -419,4 +575,14 @@ export class IndexManager { ); } } + + private assertUniqueSupported(columnName: string, definition: ColumnDefinition): void { + if (definition.kind === "boolean") { + throw new ColQLError( + "COLQL_UNIQUE_INDEX_UNSUPPORTED", + `Unique indexing is not supported for boolean column "${columnName}".`, + { columnName, kind: definition.kind }, + ); + } + } } diff --git a/src/indexing/unique-index.ts b/src/indexing/unique-index.ts new file mode 100644 index 0000000..a6bfd0a --- /dev/null +++ b/src/indexing/unique-index.ts @@ -0,0 +1,86 @@ +import { ColQLError } from "../errors"; + +export type UniqueIndexValue = number; + +export type UniqueIndexStats = { + readonly column: string; + readonly uniqueValues: number; + readonly rowCount: number; + readonly memoryBytesApprox: number; + readonly dirty: boolean; +}; + +export class UniqueIndex { + private readonly rowsByValue = new Map(); + private indexedRows = 0; + private dirty = false; + + constructor(readonly column: string) {} + + add(value: UniqueIndexValue, rowIndex: number): void { + const existingRowIndex = this.rowsByValue.get(value); + if (existingRowIndex !== undefined) { + throw new ColQLError( + "COLQL_DUPLICATE_KEY", + `Duplicate key for unique index "${this.column}".`, + { + columnName: this.column, + encodedValue: value, + existingRowIndex, + rowIndex, + }, + ); + } + + this.rowsByValue.set(value, rowIndex); + this.indexedRows += 1; + } + + get(value: UniqueIndexValue): number | undefined { + return this.rowsByValue.get(value); + } + + deleteRow(rowIndex: number): void { + if (this.dirty) { + return; + } + + for (const [value, indexedRow] of this.rowsByValue) { + if (indexedRow === rowIndex) { + this.rowsByValue.delete(value); + this.indexedRows -= 1; + continue; + } + + if (indexedRow > rowIndex) { + this.rowsByValue.set(value, indexedRow - 1); + } + } + } + + markDirty(): void { + this.dirty = true; + } + + markFresh(): void { + this.dirty = false; + } + + isDirty(): boolean { + return this.dirty; + } + + stats(): UniqueIndexStats { + return { + column: this.column, + uniqueValues: this.rowsByValue.size, + rowCount: this.indexedRows, + memoryBytesApprox: this.memoryBytesApprox(), + dirty: this.dirty, + }; + } + + private memoryBytesApprox(): number { + return this.indexedRows * Uint32Array.BYTES_PER_ELEMENT + this.rowsByValue.size * 40; + } +} diff --git a/src/storage/boolean-column.ts b/src/storage/boolean-column.ts index 5a7657c..9640a23 100644 --- a/src/storage/boolean-column.ts +++ b/src/storage/boolean-column.ts @@ -37,6 +37,50 @@ export class BooleanColumnStorage implements ColumnStorage { get(rowIndex: number): boolean { if (rowIndex >= this.currentRowCount && rowIndex < this.logicalCapacity) return false; const { chunkIndex, offset } = this.locate(rowIndex); return this.chunks[chunkIndex].get(offset); } set(rowIndex: number, value: boolean): void { assertBooleanValue("boolean", value); if (!Number.isInteger(rowIndex) || rowIndex < 0 || rowIndex >= this.logicalCapacity) this.assertIndex(rowIndex); while (rowIndex > this.currentRowCount) this.append(false); if (rowIndex === this.currentRowCount) return this.append(value); const { chunkIndex, offset } = this.locate(rowIndex); this.chunks[chunkIndex].set(offset, value); } deleteAt(rowIndex: number): void { const { chunkIndex, offset } = this.locate(rowIndex); this.chunks[chunkIndex].deleteAt(offset, this.lengths[chunkIndex]); this.lengths[chunkIndex] -= 1; this.currentRowCount -= 1; this.removeEmptyChunk(chunkIndex); } + deleteMany(rowIndexes: readonly number[]): void { + if (rowIndexes.length === 0) return; + + let deleteOffset = 0; + let nextDelete = rowIndexes[deleteOffset]; + const nextChunks: BooleanChunk[] = []; + const nextLengths: number[] = []; + let nextRowCount = 0; + let sourceRowIndex = 0; + + const appendRaw = (value: boolean): void => { + let chunk = nextChunks[nextChunks.length - 1]; + if (chunk === undefined || nextLengths[nextLengths.length - 1] >= this.chunkSize) { + chunk = new BooleanChunk(this.chunkSize); + nextChunks.push(chunk); + nextLengths.push(0); + } + + const chunkIndex = nextChunks.length - 1; + chunk.set(nextLengths[chunkIndex], value); + nextLengths[chunkIndex] += 1; + nextRowCount += 1; + }; + + for (let chunkIndex = 0; chunkIndex < this.chunks.length; chunkIndex += 1) { + const chunk = this.chunks[chunkIndex]; + const length = this.lengths[chunkIndex]; + for (let offset = 0; offset < length; offset += 1) { + if (sourceRowIndex === nextDelete) { + deleteOffset += 1; + nextDelete = rowIndexes[deleteOffset]; + } else { + appendRaw(chunk.get(offset)); + } + sourceRowIndex += 1; + } + } + + this.chunks.length = 0; + this.chunks.push(...nextChunks); + this.lengths.length = 0; + this.lengths.push(...nextLengths); + this.currentRowCount = nextRowCount; + } resize(capacity: number): void { assertNonNegativeInteger(capacity, "limit"); this.logicalCapacity = capacity; while (this.chunks.length * this.chunkSize < capacity) { this.chunks.push(new BooleanChunk(this.chunkSize)); this.lengths.push(0); } } toBytes(): Uint8Array { const output = new Uint8Array(Math.ceil(this.logicalCapacity / BITS_PER_BYTE)); let targetBitOffset = 0; for (let chunkIndex = 0; chunkIndex < this.chunks.length; chunkIndex += 1) { const length = this.lengths[chunkIndex]; this.chunks[chunkIndex].copyInto(output, targetBitOffset, length); targetBitOffset += length; } return output; } diff --git a/src/storage/dictionary-column.ts b/src/storage/dictionary-column.ts index 099592d..5aa76f1 100644 --- a/src/storage/dictionary-column.ts +++ b/src/storage/dictionary-column.ts @@ -71,6 +71,51 @@ export class DictionaryColumnStorage implement this.removeEmptyChunk(chunkIndex); } + deleteMany(rowIndexes: readonly number[]): void { + if (rowIndexes.length === 0) return; + + let deleteOffset = 0; + let nextDelete = rowIndexes[deleteOffset]; + const nextChunks: DictionaryCodeArray[] = []; + const nextLengths: number[] = []; + let nextRowCount = 0; + let sourceRowIndex = 0; + + const appendCodeRaw = (code: number): void => { + let chunk = nextChunks[nextChunks.length - 1]; + if (chunk === undefined || nextLengths[nextLengths.length - 1] >= this.chunkSize) { + chunk = new this.ArrayType(this.chunkSize); + nextChunks.push(chunk); + nextLengths.push(0); + } + + const chunkIndex = nextChunks.length - 1; + chunk[nextLengths[chunkIndex]] = code; + nextLengths[chunkIndex] += 1; + nextRowCount += 1; + }; + + for (let chunkIndex = 0; chunkIndex < this.chunks.length; chunkIndex += 1) { + const chunk = this.chunks[chunkIndex]; + const length = this.lengths[chunkIndex]; + for (let offset = 0; offset < length; offset += 1) { + if (sourceRowIndex === nextDelete) { + deleteOffset += 1; + nextDelete = rowIndexes[deleteOffset]; + } else { + appendCodeRaw(chunk[offset]); + } + sourceRowIndex += 1; + } + } + + this.chunks.length = 0; + this.chunks.push(...nextChunks); + this.lengths.length = 0; + this.lengths.push(...nextLengths); + this.currentRowCount = nextRowCount; + } + resize(capacity: number): void { assertNonNegativeInteger(capacity, "limit"); this.logicalCapacity = capacity; diff --git a/src/storage/numeric-column.ts b/src/storage/numeric-column.ts index 5a43ebf..c0faf9c 100644 --- a/src/storage/numeric-column.ts +++ b/src/storage/numeric-column.ts @@ -90,6 +90,51 @@ export class NumericColumnStorage implements ColumnStorage { this.removeEmptyChunk(chunkIndex); } + deleteMany(rowIndexes: readonly number[]): void { + if (rowIndexes.length === 0) return; + + let deleteOffset = 0; + let nextDelete = rowIndexes[deleteOffset]; + const nextChunks: NumericArray[] = []; + const nextLengths: number[] = []; + let nextRowCount = 0; + let sourceRowIndex = 0; + + const appendRaw = (value: number): void => { + let chunk = nextChunks[nextChunks.length - 1]; + if (chunk === undefined || nextLengths[nextLengths.length - 1] >= this.chunkSize) { + chunk = new this.ArrayType(this.chunkSize); + nextChunks.push(chunk); + nextLengths.push(0); + } + + const chunkIndex = nextChunks.length - 1; + chunk[nextLengths[chunkIndex]] = value; + nextLengths[chunkIndex] += 1; + nextRowCount += 1; + }; + + for (let chunkIndex = 0; chunkIndex < this.chunks.length; chunkIndex += 1) { + const chunk = this.chunks[chunkIndex]; + const length = this.lengths[chunkIndex]; + for (let offset = 0; offset < length; offset += 1) { + if (sourceRowIndex === nextDelete) { + deleteOffset += 1; + nextDelete = rowIndexes[deleteOffset]; + } else { + appendRaw(chunk[offset]); + } + sourceRowIndex += 1; + } + } + + this.chunks.length = 0; + this.chunks.push(...nextChunks); + this.lengths.length = 0; + this.lengths.push(...nextLengths); + this.currentRowCount = nextRowCount; + } + resize(capacity: number): void { assertNonNegativeInteger(capacity, "limit"); this.logicalCapacity = capacity; diff --git a/src/table.ts b/src/table.ts index 8c717bb..99c5782 100644 --- a/src/table.ts +++ b/src/table.ts @@ -12,6 +12,7 @@ import { } from "./indexing/index-manager"; import type { EqualityIndexStats } from "./indexing/equality-index"; import type { SortedIndexStats } from "./indexing/sorted-index"; +import type { UniqueIndexStats } from "./indexing/unique-index"; import { assertColumnExists, assertNonNegativeInteger, @@ -38,6 +39,7 @@ import type { Schema, SelectedRow, TableOptions, + UniqueColumnKey, } from "./types"; const DEFAULT_CAPACITY = 1024; @@ -68,6 +70,10 @@ type StorageMap = { [Key in keyof TSchema]: ColumnStorage>; }; +type BulkDeleteStorage = { + deleteMany(rowIndexes: readonly number[]): void; +}; + type InternalFilter = { readonly columnName: string; readonly operator: Operator; @@ -179,6 +185,7 @@ export class Table { insert(row: RowForSchema): this { this.validateRow(row); + this.assertUniqueInsert(row, "insert"); this.ensureCapacity(this.currentRowCount + 1); this.appendRow(row); this.addRowToIndexes(this.currentRowCount - 1); @@ -188,7 +195,7 @@ export class Table { delete(rowIndex: number): this { assertRowIndex(rowIndex, this.currentRowCount); this.deleteRowAt(rowIndex); - this.indexManager.markDirty(); + this.indexManager.markDeletedRow(rowIndex); return this; } @@ -198,8 +205,9 @@ export class Table { ): MutationResult { assertRowIndex(rowIndex, this.currentRowCount); const values = this.validatePartialRow(partialRow, "updated row"); + this.assertUniqueUpdate([rowIndex], values, "update"); this.applyPartialRow(rowIndex, values); - this.indexManager.markDirty(); + this.markIndexesAfterUpdate(values); return { affectedRows: 1 }; } @@ -280,11 +288,12 @@ export class Table { this.assertReadableRow(rowIndex); } + this.assertUniqueUpdate(indexes, values, "updateMany"); for (const rowIndex of indexes) { this.applyPartialRow(rowIndex, values); } - this.indexManager.markDirty(); + this.markIndexesAfterUpdate(values); return { affectedRows: indexes.length }; } @@ -300,9 +309,7 @@ export class Table { this.assertReadableRow(rowIndex); } - for (const rowIndex of indexes) { - this.deleteRowAt(rowIndex); - } + this.deleteRowsAt(indexes); this.indexManager.markDirty(); return { affectedRows: indexes.length }; @@ -320,6 +327,32 @@ export class Table { ); } + private deleteRowsAt(rowIndexesDescending: readonly number[]): void { + const rowIndexes = [...rowIndexesDescending].sort((left, right) => left - right); + if (rowIndexes.length === 1) { + this.deleteRowAt(rowIndexes[0]); + return; + } + + for (const key of this.schemaKeys()) { + const storage = this.storages[key]; + if (this.hasBulkDelete(storage)) { + storage.deleteMany(rowIndexes); + continue; + } + + for (let index = rowIndexes.length - 1; index >= 0; index -= 1) { + storage.deleteAt(rowIndexes[index]); + } + } + + this.currentRowCount -= rowIndexes.length; + this.currentCapacity = Math.max( + 1, + ...this.schemaKeys().map((key) => this.storages[key].capacity), + ); + } + insertMany(rows: readonly RowForSchema[]): this { if (!Array.isArray(rows)) { throw new ColQLError( @@ -347,6 +380,7 @@ export class Table { return this; } + this.assertUniqueInsertMany(rows); const firstRowIndex = this.currentRowCount; this.ensureCapacity(this.currentRowCount + rows.length); for (const row of rows) { @@ -360,6 +394,7 @@ export class Table { rowIndex += 1 ) { this.addRowToEqualityIndexes(rowIndex); + this.addRowToUniqueIndexes(rowIndex); } return this; @@ -556,6 +591,177 @@ export class Table { return this.indexManager.sortedStats(); } + createUniqueIndex>(columnName: Key): this { + assertColumnExists(this.schema, columnName, "createUniqueIndex()"); + this.indexManager.createUnique( + String(columnName), + this.schema[columnName], + this.currentRowCount, + (rowIndex, name) => + this.getComparableValue(rowIndex, name as keyof TSchema), + ); + return this; + } + + dropUniqueIndex(columnName: Key): this { + assertColumnExists(this.schema, columnName, "dropUniqueIndex()"); + this.indexManager.dropUnique(String(columnName)); + return this; + } + + hasUniqueIndex(columnName: Key): boolean { + assertColumnExists(this.schema, columnName, "hasUniqueIndex()"); + return this.indexManager.hasUnique(String(columnName)); + } + + uniqueIndexes(): string[] { + return this.indexManager.listUnique(); + } + + uniqueIndexStats(): UniqueIndexStats[] { + return this.indexManager.uniqueStats( + this.currentRowCount, + (rowIndex, name) => + this.getComparableValue(rowIndex, name as keyof TSchema), + ); + } + + rebuildUniqueIndex>(columnName: Key): this { + assertColumnExists(this.schema, columnName, "rebuildUniqueIndex()"); + this.indexManager.rebuildUnique( + String(columnName), + this.currentRowCount, + (rowIndex, name) => + this.getComparableValue(rowIndex, name as keyof TSchema), + ); + return this; + } + + rebuildUniqueIndexes(): this { + this.indexManager.rebuildUniqueIndexes( + this.currentRowCount, + (rowIndex, name) => + this.getComparableValue(rowIndex, name as keyof TSchema), + ); + return this; + } + + findBy>( + columnName: Key, + value: ColumnValue, + ): RowForSchema | undefined { + const rowIndex = this.uniqueRowIndexByValue(columnName, value, "findBy"); + return rowIndex === undefined ? undefined : this.materializeRow(rowIndex); + } + + updateBy>( + columnName: Key, + value: ColumnValue, + partialRow: PartialRowForSchema, + ): MutationResult { + const rowIndex = this.uniqueRowIndexByValue(columnName, value, "updateBy"); + if (rowIndex === undefined) { + this.validatePartialRow(partialRow, "updated row"); + return { affectedRows: 0 }; + } + + return this.update(rowIndex, partialRow); + } + + deleteBy>( + columnName: Key, + value: ColumnValue, + ): MutationResult { + const rowIndex = this.uniqueRowIndexByValue(columnName, value, "deleteBy"); + if (rowIndex === undefined) { + return { affectedRows: 0 }; + } + + this.delete(rowIndex); + return { affectedRows: 1 }; + } + + firstWhere(predicate: ObjectWherePredicate): RowForSchema | undefined; + firstWhere(predicate: RowPredicate): RowForSchema | undefined; + firstWhere( + columnName: Key, + operator: TOperator, + value: ValueForOperator, TOperator>, + ): RowForSchema | undefined; + firstWhere( + columnNameOrPredicate: Key | ObjectWherePredicate | RowPredicate, + operator?: TOperator, + value?: ValueForOperator, TOperator>, + ): RowForSchema | undefined { + if (typeof columnNameOrPredicate === "function") { + return this.filter(columnNameOrPredicate).first(); + } + + if (arguments.length === 1) { + return this.where(columnNameOrPredicate as ObjectWherePredicate).first(); + } + + return this.where( + columnNameOrPredicate as Key, + operator as TOperator, + value as ValueForOperator, TOperator>, + ).first(); + } + + countWhere(predicate: ObjectWherePredicate): number; + countWhere(predicate: RowPredicate): number; + countWhere( + columnName: Key, + operator: TOperator, + value: ValueForOperator, TOperator>, + ): number; + countWhere( + columnNameOrPredicate: Key | ObjectWherePredicate | RowPredicate, + operator?: TOperator, + value?: ValueForOperator, TOperator>, + ): number { + if (typeof columnNameOrPredicate === "function") { + return this.filter(columnNameOrPredicate).count(); + } + + if (arguments.length === 1) { + return this.where(columnNameOrPredicate as ObjectWherePredicate).count(); + } + + return this.where( + columnNameOrPredicate as Key, + operator as TOperator, + value as ValueForOperator, TOperator>, + ).count(); + } + + exists(predicate: ObjectWherePredicate): boolean; + exists(predicate: RowPredicate): boolean; + exists( + columnName: Key, + operator: TOperator, + value: ValueForOperator, TOperator>, + ): boolean; + exists( + columnNameOrPredicate: Key | ObjectWherePredicate | RowPredicate, + operator?: TOperator, + value?: ValueForOperator, TOperator>, + ): boolean { + if (typeof columnNameOrPredicate === "function") { + return this.filter(columnNameOrPredicate).limit(1).count() > 0; + } + + if (arguments.length === 1) { + return this.where(columnNameOrPredicate as ObjectWherePredicate).limit(1).count() > 0; + } + + return this.where( + columnNameOrPredicate as Key, + operator as TOperator, + value as ValueForOperator, TOperator>, + ).limit(1).count() > 0; + } + forEach(callback: (row: RowForSchema, index: number) => void): void { this.query().forEach(callback); } @@ -981,6 +1187,7 @@ export class Table { private addRowToIndexes(rowIndex: number): void { this.indexManager.markSortedDirty(); this.addRowToEqualityIndexes(rowIndex); + this.addRowToUniqueIndexes(rowIndex); } private addRowToEqualityIndexes(rowIndex: number): void { @@ -998,6 +1205,191 @@ export class Table { } } + private addRowToUniqueIndexes(rowIndex: number): void { + for (const columnName of this.indexManager.listUnique()) { + this.indexManager.addUniqueRow( + columnName, + this.getComparableValue(rowIndex, columnName as keyof TSchema), + rowIndex, + ); + } + } + + private assertUniqueInsert(row: RowForSchema, operation: string): void { + for (const columnName of this.indexManager.listUnique()) { + const encodedValue = this.comparableValueFromRow(row, columnName as keyof TSchema); + const existingRowIndex = this.indexManager.uniqueLookup( + columnName, + encodedValue, + this.currentRowCount, + (rowIndex, name) => + this.getComparableValue(rowIndex, name as keyof TSchema), + ); + + if (existingRowIndex !== undefined) { + throw this.duplicateKeyError(columnName, encodedValue, { + operation, + existingRowIndex, + rowIndex: this.currentRowCount, + }); + } + } + } + + private assertUniqueInsertMany(rows: readonly RowForSchema[]): void { + for (const columnName of this.indexManager.listUnique()) { + const seen = new Map(); + for (let index = 0; index < rows.length; index += 1) { + const encodedValue = this.comparableValueFromRow(rows[index], columnName as keyof TSchema); + const existingRowIndex = this.indexManager.uniqueLookup( + columnName, + encodedValue, + this.currentRowCount, + (rowIndex, name) => + this.getComparableValue(rowIndex, name as keyof TSchema), + ); + + if (existingRowIndex !== undefined) { + throw this.duplicateKeyError(columnName, encodedValue, { + operation: "insertMany", + existingRowIndex, + inputIndex: index, + }); + } + + const conflictingInputIndex = seen.get(encodedValue); + if (conflictingInputIndex !== undefined) { + throw this.duplicateKeyError(columnName, encodedValue, { + operation: "insertMany", + inputIndex: index, + conflictingInputIndex, + }); + } + + seen.set(encodedValue, index); + } + } + } + + private assertUniqueUpdate( + rowIndexes: readonly number[], + values: readonly [keyof TSchema, ColumnValue][], + operation: string, + ): void { + const updatesByColumn = new Map>(); + for (const [key, value] of values) { + updatesByColumn.set(key, value); + } + + const targetRows = new Set(rowIndexes); + for (const columnName of this.indexManager.listUnique()) { + const key = columnName as keyof TSchema; + if (!updatesByColumn.has(key)) { + continue; + } + + const rowsByValue = new Map(); + for (let rowIndex = 0; rowIndex < this.currentRowCount; rowIndex += 1) { + const encodedValue = targetRows.has(rowIndex) + ? this.comparableValueFromValue(key, updatesByColumn.get(key)) + : (this.getComparableValue(rowIndex, key) as number); + const existingRowIndex = rowsByValue.get(encodedValue); + if (existingRowIndex !== undefined) { + throw this.duplicateKeyError(columnName, encodedValue, { + operation, + existingRowIndex, + rowIndex, + }); + } + + rowsByValue.set(encodedValue, rowIndex); + } + } + } + + private markIndexesAfterUpdate( + values: readonly [keyof TSchema, ColumnValue][], + ): void { + this.indexManager.markPerformanceDirty(); + const uniqueColumns = values + .map(([key]) => String(key)) + .filter((columnName) => this.indexManager.hasUnique(columnName)); + if (uniqueColumns.length > 0) { + this.indexManager.markUniqueDirty(uniqueColumns); + } + } + + private uniqueRowIndexByValue>( + columnName: Key, + value: ColumnValue, + context: string, + ): number | undefined { + assertColumnExists(this.schema, columnName, `${context}()`); + if (!this.indexManager.hasUnique(String(columnName))) { + throw new ColQLError( + "COLQL_UNIQUE_INDEX_NOT_FOUND", + `Unique index not found for column "${String(columnName)}".`, + { columnName: String(columnName) }, + ); + } + + validateColumnValue(String(columnName), this.schema[columnName], value); + return this.indexManager.uniqueLookup( + String(columnName), + this.comparableValueFromValue(columnName, value), + this.currentRowCount, + (rowIndex, name) => + this.getComparableValue(rowIndex, name as keyof TSchema), + ); + } + + private comparableValueFromRow( + row: RowForSchema, + columnName: keyof TSchema, + ): number { + return this.comparableValueFromValue(columnName, row[columnName]); + } + + private comparableValueFromValue( + columnName: keyof TSchema, + value: unknown, + ): number { + const storage = this.storages[columnName]; + if (storage instanceof DictionaryColumnStorage) { + return storage.encode(value as string); + } + + if (this.schema[columnName].kind === "boolean") { + throw new ColQLError( + "COLQL_UNIQUE_INDEX_UNSUPPORTED", + `Unique indexing is not supported for boolean column "${String(columnName)}".`, + { columnName: String(columnName), kind: "boolean" }, + ); + } + + return value as number; + } + + private duplicateKeyError( + columnName: string, + encodedValue: number, + details: Record, + ): ColQLError { + return new ColQLError( + "COLQL_DUPLICATE_KEY", + `Duplicate key for unique index "${columnName}".`, + { + columnName, + encodedValue, + ...details, + }, + ); + } + + private hasBulkDelete(storage: ColumnStorage): storage is ColumnStorage & BulkDeleteStorage { + return typeof (storage as Partial).deleteMany === "function"; + } + private createSerializedColumnMeta( name: string, ): Omit { @@ -1390,6 +1782,14 @@ export function table( return new Table(schema, options); } +export function fromRows( + schema: TSchema, + rows: readonly RowForSchema[], + options?: TableOptions, +): Table { + return table(schema, options).insertMany(rows); +} + export namespace table { export function deserialize(input: ArrayBuffer | Uint8Array): Table { return Table.deserialize(input); diff --git a/src/types.ts b/src/types.ts index f4b38bc..1218fd8 100644 --- a/src/types.ts +++ b/src/types.ts @@ -85,6 +85,10 @@ export type NumericColumnKey = { [Key in keyof TSchema]: ColumnValue extends number ? Key : never; }[keyof TSchema]; +export type UniqueColumnKey = { + [Key in keyof TSchema]: TSchema[Key] extends BooleanColumnDefinition ? never : Key; +}[keyof TSchema]; + export type WhereValue = T | readonly T[]; export interface Filter { diff --git a/tests/array-helpers.test.ts b/tests/array-helpers.test.ts new file mode 100644 index 0000000..4a46264 --- /dev/null +++ b/tests/array-helpers.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { column, fromRows, table } from "../src"; + +const schema = { + id: column.uint32(), + age: column.uint8(), + status: column.dictionary(["active", "passive"] as const), +}; + +const rows = [ + { id: 1, age: 20, status: "active" }, + { id: 2, age: 30, status: "passive" }, + { id: 3, age: 40, status: "active" }, +] as const; + +describe("JS Array migration helpers", () => { + it("fromRows creates a table through insertMany", () => { + const users = fromRows(schema, rows); + + expect(users.toArray()).toEqual(rows); + expect(() => fromRows(schema, [{ id: 4, age: 999, status: "active" }])).toThrow(); + }); + + it("firstWhere delegates to structured where and callback filter", () => { + const users = fromRows(schema, rows); + + expect(users.firstWhere({ status: "active" })).toEqual(rows[0]); + expect(users.firstWhere("id", "=", 2)).toEqual(rows[1]); + expect(users.firstWhere((row) => row.age > 35)).toEqual(rows[2]); + expect(users.firstWhere({ id: 999 })).toBeUndefined(); + }); + + it("countWhere delegates to structured where and callback filter", () => { + const users = fromRows(schema, rows); + + expect(users.countWhere({ status: "active" })).toBe(2); + expect(users.countWhere("age", ">=", 30)).toBe(2); + expect(users.countWhere((row) => row.status === "passive")).toBe(1); + }); + + it("exists uses a limited query path", () => { + const users = table(schema).insertMany(rows); + + expect(users.exists({ status: "active" })).toBe(true); + expect(users.exists("id", "=", 999)).toBe(false); + expect(users.exists((row) => row.age === 30)).toBe(true); + }); + + it("preserves index behavior for structured helpers", () => { + const events: boolean[] = []; + const users = table(schema, { + onQuery(info) { + events.push(info.indexUsed); + }, + }).insertMany(rows); + + users.createIndex("id"); + expect(users.firstWhere("id", "=", 3)).toEqual(rows[2]); + expect(events.at(-1)).toBe(true); + }); +}); diff --git a/tests/by-key-helpers.test.ts b/tests/by-key-helpers.test.ts new file mode 100644 index 0000000..70c05e5 --- /dev/null +++ b/tests/by-key-helpers.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, it } from "vitest"; +import { ColQLError, column, table } from "../src"; + +const schema = { + id: column.uint32(), + status: column.dictionary(["active", "passive"] as const), + age: column.uint8(), +}; + +function createUsers() { + return table(schema) + .insertMany([ + { id: 1, status: "active", age: 20 }, + { id: 2, status: "passive", age: 30 }, + { id: 3, status: "active", age: 40 }, + ]) + .createUniqueIndex("id"); +} + +function expectCode(fn: () => unknown, code: string): void { + expect(fn).toThrow(ColQLError); + try { + fn(); + } catch (error) { + expect((error as ColQLError).code).toBe(code); + } +} + +describe("by-key helpers", () => { + it("findBy returns existing rows and undefined for missing keys", () => { + const users = createUsers(); + + expect(users.findBy("id", 2)).toEqual({ id: 2, status: "passive", age: 30 }); + expect(users.findBy("id", 999)).toBeUndefined(); + }); + + it("updateBy updates existing rows and returns zero for missing keys", () => { + const users = createUsers(); + + expect(users.updateBy("id", 2, { age: 31 })).toEqual({ affectedRows: 1 }); + expect(users.findBy("id", 2)).toEqual({ id: 2, status: "passive", age: 31 }); + expect(users.updateBy("id", 999, { age: 10 })).toEqual({ affectedRows: 0 }); + }); + + it("deleteBy deletes existing rows and returns zero for missing keys", () => { + const users = createUsers(); + + expect(users.deleteBy("id", 2)).toEqual({ affectedRows: 1 }); + expect(users.findBy("id", 2)).toBeUndefined(); + expect(users.deleteBy("id", 999)).toEqual({ affectedRows: 0 }); + }); + + it("requires a unique index and does not scan without one", () => { + const users = table(schema).insertMany([ + { id: 1, status: "active", age: 20 }, + { id: 1, status: "passive", age: 30 }, + ]); + + expectCode(() => users.findBy("id", 1), "COLQL_UNIQUE_INDEX_NOT_FOUND"); + expectCode(() => users.updateBy("id", 1, { age: 40 }), "COLQL_UNIQUE_INDEX_NOT_FOUND"); + expectCode(() => users.deleteBy("id", 1), "COLQL_UNIQUE_INDEX_NOT_FOUND"); + }); + + it("does not leak rowIndex stability after deletes shift rows", () => { + const users = createUsers(); + + users.deleteBy("id", 1); + expect(users.findBy("id", 2)).toEqual({ id: 2, status: "passive", age: 30 }); + expect(users.updateBy("id", 3, { age: 41 })).toEqual({ affectedRows: 1 }); + expect(users.findBy("id", 3)).toEqual({ id: 3, status: "active", age: 41 }); + }); +}); diff --git a/tests/type-inference.test-d.ts b/tests/type-inference.test-d.ts index 8ee6daf..3158766 100644 --- a/tests/type-inference.test-d.ts +++ b/tests/type-inference.test-d.ts @@ -1,5 +1,5 @@ -import { column, table } from "../src"; -import type { MutationResult, QueryInfo } from "../src"; +import { column, fromRows, table } from "../src"; +import type { MutationResult, QueryInfo, UniqueIndexStats } from "../src"; const users = table({ id: column.uint32(), @@ -60,6 +60,25 @@ users.hasSortedIndex("age"); users.sortedIndexes(); users.sortedIndexStats(); users.dropSortedIndex("age"); +users.createUniqueIndex("id"); +users.createUniqueIndex("status"); +users.hasUniqueIndex("id"); +users.uniqueIndexes(); +const uniqueStats: UniqueIndexStats[] = users.uniqueIndexStats(); +users.rebuildUniqueIndex("id"); +users.rebuildUniqueIndexes(); +users.dropUniqueIndex("status"); +const foundById: { id: number; age: number; status: "active" | "passive"; is_active: boolean } | undefined = users.findBy("id", 1); +const updateByResult: MutationResult = users.updateBy("id", 1, { age: 30 }); +const deleteByResult: MutationResult = users.deleteBy("id", 1); +const fromRowsUsers = fromRows(users.getSchema(), [ + { id: 10, age: 25, status: "active", is_active: true }, +]); +const fromRowsFirst: { id: number; age: number; status: "active" | "passive"; is_active: boolean } | undefined = fromRowsUsers.firstWhere({ status: "active" }); +const firstWhereTuple: { id: number; age: number; status: "active" | "passive"; is_active: boolean } | undefined = users.firstWhere("id", "=", 1); +const firstWhereCallback: { id: number; age: number; status: "active" | "passive"; is_active: boolean } | undefined = users.firstWhere((item) => item.age > 20); +const countWhereResult: number = users.countWhere({ status: "active" }); +const existsResult: boolean = users.exists((item) => item.is_active); const deleteReturn: typeof users = users.delete(0); const updateResult: MutationResult = users.update(0, { age: 30 }); const updateStatusResult: MutationResult = users.update(0, { status: "active" }); @@ -81,6 +100,14 @@ void deleteWhereResult; void queryDeleteResult; void updateManyResult; void deleteManyResult; +void uniqueStats; +void foundById; +void updateByResult; +void deleteByResult; +void firstWhereTuple; +void firstWhereCallback; +void countWhereResult; +void existsResult; const row: { id: number; age: number; status: "active" | "passive"; is_active: boolean } = users.get(0); const serialized: ArrayBuffer = users.serialize(); const restored = table.deserialize(serialized); @@ -147,6 +174,9 @@ users.createIndex("missing"); // @ts-expect-error sorted indexes require numeric columns users.createSortedIndex("status"); +// @ts-expect-error unique indexes reject boolean columns +users.createUniqueIndex("is_active"); + // @ts-expect-error unknown sorted index column users.createSortedIndex("missing"); @@ -188,3 +218,30 @@ users.rebuildSortedIndex("missing"); // @ts-expect-error sorted rebuild indexes require numeric columns users.rebuildSortedIndex("status"); + +// @ts-expect-error unknown unique rebuild index column +users.rebuildUniqueIndex("missing"); + +// @ts-expect-error unique rebuild rejects boolean columns +users.rebuildUniqueIndex("is_active"); + +// @ts-expect-error findBy rejects wrong value type +users.findBy("id", "active"); + +// @ts-expect-error findBy rejects wrong dictionary value +users.findBy("status", "deleted"); + +// @ts-expect-error updateBy rejects wrong partial type +users.updateBy("id", 1, { age: "old" }); + +// @ts-expect-error fromRows rejects wrong row type +fromRows(users.getSchema(), [{ id: 11, age: "old", status: "active", is_active: true }]); + +// @ts-expect-error firstWhere rejects wrong tuple value +users.firstWhere("age", "=", "old"); + +// @ts-expect-error countWhere rejects wrong object predicate +users.countWhere({ status: "deleted" }); + +// @ts-expect-error exists callback receives typed rows +users.exists((item) => item.missing === 1); diff --git a/tests/unique-index-parity.test.ts b/tests/unique-index-parity.test.ts new file mode 100644 index 0000000..3af4a71 --- /dev/null +++ b/tests/unique-index-parity.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, it } from "vitest"; +import { column, table, type RowForSchema } from "../src"; + +const schema = { + id: column.uint32(), + age: column.uint8(), + score: column.uint32(), + status: column.dictionary(["active", "passive", "archived"] as const), +}; + +type User = RowForSchema; + +function seedRows(count: number): User[] { + return Array.from({ length: count }, (_unused, id) => ({ + id, + age: (id * 7) % 100, + score: (id * 13) % 1_000, + status: id % 3 === 0 ? "active" : id % 3 === 1 ? "passive" : "archived", + })); +} + +describe("unique index parity", () => { + it("matches plain array oracle across mutation and by-key sequences", () => { + const initial = seedRows(120); + const users = table(schema).insertMany(initial).createUniqueIndex("id").createIndex("status").createSortedIndex("age"); + const oracle = initial.map((row) => ({ ...row })); + + users.updateMany({ status: "active", age: { gte: 40 } }, { score: 999 }); + for (const row of oracle) { + if (row.status === "active" && row.age >= 40) { + row.score = 999; + } + } + + users.deleteMany({ status: "archived", age: { lt: 50 } }); + for (let index = oracle.length - 1; index >= 0; index -= 1) { + if (oracle[index].status === "archived" && oracle[index].age < 50) { + oracle.splice(index, 1); + } + } + + users.insert({ id: 500, age: 50, score: 500, status: "passive" }); + oracle.push({ id: 500, age: 50, score: 500, status: "passive" }); + + expect(users.findBy("id", 500)).toEqual(oracle.find((row) => row.id === 500)); + expect(users.updateBy("id", 500, { age: 51 })).toEqual({ affectedRows: 1 }); + oracle.find((row) => row.id === 500)!.age = 51; + + expect(users.deleteBy("id", 1)).toEqual({ affectedRows: oracle.some((row) => row.id === 1) ? 1 : 0 }); + const deleteIndex = oracle.findIndex((row) => row.id === 1); + if (deleteIndex >= 0) { + oracle.splice(deleteIndex, 1); + } + + expect(users.toArray()).toEqual(oracle); + expect(users.where("status", "=", "passive").where("age", ">=", 30).toArray()).toEqual( + oracle.filter((row) => row.status === "passive" && row.age >= 30), + ); + expect(users.countWhere("score", "=", 999)).toBe(oracle.filter((row) => row.score === 999).length); + }); +}); diff --git a/tests/unique-index.test.ts b/tests/unique-index.test.ts new file mode 100644 index 0000000..a29dd29 --- /dev/null +++ b/tests/unique-index.test.ts @@ -0,0 +1,164 @@ +import { describe, expect, it } from "vitest"; +import { ColQLError, column, table } from "../src"; + +const schema = { + id: column.uint32(), + age: column.uint8(), + status: column.dictionary(["active", "passive", "archived"] as const), + active: column.boolean(), +}; + +function createUsers() { + return table(schema).insertMany([ + { id: 1, age: 20, status: "active", active: true }, + { id: 2, age: 30, status: "passive", active: false }, + { id: 3, age: 40, status: "archived", active: true }, + ]); +} + +function expectCode(fn: () => unknown, code: string): ColQLError { + expect(fn).toThrow(ColQLError); + try { + fn(); + } catch (error) { + expect((error as ColQLError).code).toBe(code); + return error as ColQLError; + } + throw new Error("Expected ColQLError"); +} + +describe("unique indexes", () => { + it("creates unique indexes on numeric and dictionary columns", () => { + const users = createUsers(); + + expect(users.createUniqueIndex("id")).toBe(users); + expect(users.createUniqueIndex("status")).toBe(users); + + expect(users.hasUniqueIndex("id")).toBe(true); + expect(users.uniqueIndexes()).toEqual(["id", "status"]); + expect(users.uniqueIndexStats()).toEqual( + expect.arrayContaining([ + expect.objectContaining({ column: "id", rowCount: 3, uniqueValues: 3, dirty: false }), + expect.objectContaining({ column: "status", rowCount: 3, uniqueValues: 3, dirty: false }), + ]), + ); + }); + + it("rejects boolean unique indexes and duplicate lifecycle operations", () => { + const users = createUsers(); + + expectCode(() => users.createUniqueIndex("active" as never), "COLQL_UNIQUE_INDEX_UNSUPPORTED"); + users.createUniqueIndex("id"); + expectCode(() => users.createUniqueIndex("id"), "COLQL_UNIQUE_INDEX_EXISTS"); + expectCode(() => users.dropUniqueIndex("age"), "COLQL_UNIQUE_INDEX_NOT_FOUND"); + expectCode(() => users.rebuildUniqueIndex("age"), "COLQL_UNIQUE_INDEX_NOT_FOUND"); + }); + + it("rejects duplicate existing data on create and rebuild atomically", () => { + const users = table(schema).insertMany([ + { id: 1, age: 20, status: "active", active: true }, + { id: 1, age: 30, status: "passive", active: false }, + ]); + + const error = expectCode(() => users.createUniqueIndex("id"), "COLQL_DUPLICATE_KEY"); + expect(error.details).toEqual(expect.objectContaining({ columnName: "id" })); + expect(users.hasUniqueIndex("id")).toBe(false); + + const clean = createUsers().createUniqueIndex("id"); + clean.dropUniqueIndex("id"); + clean.insert({ id: 2, age: 50, status: "active", active: true }); + expectCode(() => clean.createUniqueIndex("id"), "COLQL_DUPLICATE_KEY"); + expect(clean.hasUniqueIndex("id")).toBe(false); + }); + + it("rejects duplicate insert and insertMany while preserving all-or-nothing", () => { + const users = createUsers().createUniqueIndex("id"); + const before = users.toArray(); + + expectCode(() => users.insert({ id: 2, age: 50, status: "active", active: true }), "COLQL_DUPLICATE_KEY"); + expect(users.toArray()).toEqual(before); + + expectCode( + () => + users.insertMany([ + { id: 4, age: 50, status: "active", active: true }, + { id: 2, age: 51, status: "passive", active: false }, + ]), + "COLQL_DUPLICATE_KEY", + ); + expect(users.toArray()).toEqual(before); + + expectCode( + () => + users.insertMany([ + { id: 4, age: 50, status: "active", active: true }, + { id: 4, age: 51, status: "passive", active: false }, + ]), + "COLQL_DUPLICATE_KEY", + ); + expect(users.toArray()).toEqual(before); + + users.insertMany([ + { id: 4, age: 50, status: "active", active: true }, + { id: 5, age: 51, status: "passive", active: false }, + ]); + expect(users.findBy("id", 5)).toEqual({ id: 5, age: 51, status: "passive", active: false }); + expect(users.uniqueIndexStats()[0]).toEqual(expect.objectContaining({ rowCount: 5, uniqueValues: 5 })); + }); + + it("rejects duplicate-producing update and updateMany all-or-nothing", () => { + const users = createUsers().createUniqueIndex("id"); + const before = users.toArray(); + + expectCode(() => users.update(0, { id: 2 }), "COLQL_DUPLICATE_KEY"); + expect(users.toArray()).toEqual(before); + + expectCode(() => users.updateMany({ id: { in: [1, 2] } }, { id: 9 }), "COLQL_DUPLICATE_KEY"); + expect(users.toArray()).toEqual(before); + + expectCode(() => users.where("id", "in", [1, 2]).update({ id: 9 }), "COLQL_DUPLICATE_KEY"); + expect(users.toArray()).toEqual(before); + }); + + it("allows unchanged unique-key updates and frees keys after delete", () => { + const users = createUsers().createUniqueIndex("id"); + + expect(users.update(0, { id: 1, age: 21 })).toEqual({ affectedRows: 1 }); + users.deleteBy("id", 2); + users.insert({ id: 2, age: 55, status: "passive", active: true }); + + expect(users.findBy("id", 2)).toEqual({ id: 2, age: 55, status: "passive", active: true }); + }); + + it("dropUniqueIndex removes enforcement", () => { + const users = createUsers().createUniqueIndex("id"); + + users.dropUniqueIndex("id"); + users.insert({ id: 1, age: 55, status: "active", active: true }); + + expect(users.where("id", "=", 1).count()).toBe(2); + }); + + it("rebuilds dirty unique indexes before stats and by-key lookup", () => { + const users = createUsers().createUniqueIndex("id"); + + users.delete(0); + expect(users.findBy("id", 2)).toEqual({ id: 2, age: 30, status: "passive", active: false }); + expect(users.uniqueIndexStats()[0]).toEqual(expect.objectContaining({ rowCount: 2, uniqueValues: 2, dirty: false })); + + users.updateBy("id", 2, { id: 20 }); + expect(users.findBy("id", 2)).toBeUndefined(); + expect(users.findBy("id", 20)).toEqual({ id: 20, age: 30, status: "passive", active: false }); + }); + + it("does not serialize unique indexes and can recreate after deserialize", () => { + const users = createUsers().createUniqueIndex("id"); + const restored = table.deserialize(users.serialize()); + + expect(restored.uniqueIndexes()).toEqual([]); + expect(restored.toArray()).toEqual(users.toArray()); + + restored.createUniqueIndex("id"); + expect(restored.hasUniqueIndex("id")).toBe(true); + }); +}); From 134504d34d5a333dcccdb9cc764d602f2b1c99a6 Mon Sep 17 00:00:00 2001 From: emremy Date: Sun, 3 May 2026 03:04:42 +0300 Subject: [PATCH 2/2] test(v0.3.0): add real-world scenario tests and strengthen correctness coverage - add user directory scenario (id/email unique + queries + mutations) - add product catalog scenario (sku uniqueness + range queries) - add session/token registry scenario (token uniqueness + expiry + cleanup) - add feature flag scenario (dictionary + boolean + indexed queries) - add mixed operation sequence tests (insert/update/delete/rebuild/serialize) - add edge case coverage for unique indexes and mutations - extend parity checks against JS array oracle - verify correctness across combined operations, not just isolated units --- benchmarks/array-comparison.mjs | 15 +- docs/doc/13-performance-and-benchmarks.md | 2 + src/query.ts | 55 +++++ src/storage/boolean-column.ts | 8 +- src/storage/dictionary-column.ts | 19 ++ src/storage/numeric-column.ts | 19 ++ src/table.ts | 9 +- tests/application-scenarios.test.ts | 242 ++++++++++++++++++++++ 8 files changed, 358 insertions(+), 11 deletions(-) create mode 100644 tests/application-scenarios.test.ts diff --git a/benchmarks/array-comparison.mjs b/benchmarks/array-comparison.mjs index 0f1dbda..2c21be8 100644 --- a/benchmarks/array-comparison.mjs +++ b/benchmarks/array-comparison.mjs @@ -164,12 +164,19 @@ function runWorkloads(rows) { }, }, { - label: "updateBy/deleteBy: unique index", + label: "updateBy: unique index", prepare: () => createTable(rows, "unique"), fn: (users) => { - users.updateBy("id", targetId, { score: 123 }); - users.deleteBy("id", targetId); - return users.findBy("id", targetId); + const result = users.updateBy("id", targetId, { score: 123 }); + return `${result.affectedRows}:${users.findBy("id", targetId)?.score}`; + }, + }, + { + label: "deleteBy: unique index", + prepare: () => createTable(rows, "unique"), + fn: (users) => { + const result = users.deleteBy("id", targetId); + return `${result.affectedRows}:${users.findBy("id", targetId) === undefined}`; }, }, ]; diff --git a/docs/doc/13-performance-and-benchmarks.md b/docs/doc/13-performance-and-benchmarks.md index d14a1eb..56ca6b7 100644 --- a/docs/doc/13-performance-and-benchmarks.md +++ b/docs/doc/13-performance-and-benchmarks.md @@ -93,6 +93,8 @@ The first indexed query after mutation may include lazy index rebuild cost. Dirt In the local delete/mutation run, the first indexed query after dirtying indexes was much slower than the second indexed query because it paid lazy rebuild cost. The benchmark also shows `toArray()` as a separate memory phase because it materializes row objects. +Broad mutations can still be slower than raw JavaScript array transforms. ColQL validates mutation payloads, snapshots matching row positions for all-or-nothing behavior, updates columnar storage, and marks or maintains derived indexes. This safety work is intentional; compare against arrays for the exact mutation shape you care about. + ## Practical Advice Measure the exact workload you care about: diff --git a/src/query.ts b/src/query.ts index 077db62..901ce21 100644 --- a/src/query.ts +++ b/src/query.ts @@ -153,11 +153,62 @@ export class Query implements Iterable } private firstUninstrumented(): TResult | undefined { + if (this.rowPredicates.length === 0) { + const rowIndex = this.firstStructuredRowIndex(); + return rowIndex === undefined + ? undefined + : this.source.materializeRow(rowIndex, this.selectedColumns) as TResult; + } + const iterator = this[Symbol.iterator](); const next = iterator.next(); return next.done ? undefined : next.value; } + private firstStructuredRowIndex(): number | undefined { + let seen = 0; + let scanned = 0; + + try { + const plan = this.source.getIndexedCandidatePlan(this.filters); + if (plan !== undefined) { + for (const rowIndex of plan.rowIndexes) { + scanned += 1; + if (!this.matchesStructuredFilters(rowIndex)) { + continue; + } + + if (seen < this.offsetValue) { + seen += 1; + continue; + } + + return rowIndex; + } + + return undefined; + } + + for (let rowIndex = 0; rowIndex < this.source.rowCount; rowIndex += 1) { + scanned += 1; + if (!this.matchesStructuredFilters(rowIndex)) { + continue; + } + + if (seen < this.offsetValue) { + seen += 1; + continue; + } + + return rowIndex; + } + + return undefined; + } finally { + this.source.recordRowScans(scanned); + } + } + count(): number { if (this.source.hasQueryHook()) { return this.runTerminal(() => this.countUninstrumented()); @@ -265,6 +316,10 @@ export class Query implements Iterable } private isEmptyUninstrumented(): boolean { + if (this.rowPredicates.length === 0) { + return this.firstStructuredRowIndex() === undefined; + } + for (const _rowIndex of this.matchingRowIndexes()) { return false; } diff --git a/src/storage/boolean-column.ts b/src/storage/boolean-column.ts index 9640a23..56a0ee8 100644 --- a/src/storage/boolean-column.ts +++ b/src/storage/boolean-column.ts @@ -19,6 +19,7 @@ export class BooleanColumnStorage implements ColumnStorage { private readonly lengths: number[] = []; private currentRowCount = 0; private logicalCapacity = 0; + private packedChunks = true; constructor(capacity: number, bytes?: Uint8Array, rowCount = bytes === undefined ? 0 : capacity, private readonly chunkSize = DEFAULT_CHUNK_SIZE) { this.assertChunkSize(chunkSize); @@ -36,7 +37,7 @@ export class BooleanColumnStorage implements ColumnStorage { append(value: boolean): void { assertBooleanValue("boolean", value); this.ensureAppendCapacity(); const chunkIndex = this.ensureWritableChunk(); const offset = this.lengths[chunkIndex]; this.chunks[chunkIndex].set(offset, value); this.lengths[chunkIndex] += 1; this.currentRowCount += 1; } get(rowIndex: number): boolean { if (rowIndex >= this.currentRowCount && rowIndex < this.logicalCapacity) return false; const { chunkIndex, offset } = this.locate(rowIndex); return this.chunks[chunkIndex].get(offset); } set(rowIndex: number, value: boolean): void { assertBooleanValue("boolean", value); if (!Number.isInteger(rowIndex) || rowIndex < 0 || rowIndex >= this.logicalCapacity) this.assertIndex(rowIndex); while (rowIndex > this.currentRowCount) this.append(false); if (rowIndex === this.currentRowCount) return this.append(value); const { chunkIndex, offset } = this.locate(rowIndex); this.chunks[chunkIndex].set(offset, value); } - deleteAt(rowIndex: number): void { const { chunkIndex, offset } = this.locate(rowIndex); this.chunks[chunkIndex].deleteAt(offset, this.lengths[chunkIndex]); this.lengths[chunkIndex] -= 1; this.currentRowCount -= 1; this.removeEmptyChunk(chunkIndex); } + deleteAt(rowIndex: number): void { const { chunkIndex, offset } = this.locate(rowIndex); this.chunks[chunkIndex].deleteAt(offset, this.lengths[chunkIndex]); this.lengths[chunkIndex] -= 1; this.currentRowCount -= 1; if (chunkIndex < this.chunks.length - 1) this.packedChunks = false; this.removeEmptyChunk(chunkIndex); } deleteMany(rowIndexes: readonly number[]): void { if (rowIndexes.length === 0) return; @@ -80,13 +81,14 @@ export class BooleanColumnStorage implements ColumnStorage { this.lengths.length = 0; this.lengths.push(...nextLengths); this.currentRowCount = nextRowCount; + this.packedChunks = true; } resize(capacity: number): void { assertNonNegativeInteger(capacity, "limit"); this.logicalCapacity = capacity; while (this.chunks.length * this.chunkSize < capacity) { this.chunks.push(new BooleanChunk(this.chunkSize)); this.lengths.push(0); } } toBytes(): Uint8Array { const output = new Uint8Array(Math.ceil(this.logicalCapacity / BITS_PER_BYTE)); let targetBitOffset = 0; for (let chunkIndex = 0; chunkIndex < this.chunks.length; chunkIndex += 1) { const length = this.lengths[chunkIndex]; this.chunks[chunkIndex].copyInto(output, targetBitOffset, length); targetBitOffset += length; } return output; } - private locate(rowIndex: number): { chunkIndex: number; offset: number } { this.assertIndex(rowIndex); let remaining = rowIndex; for (let chunkIndex = 0; chunkIndex < this.lengths.length; chunkIndex += 1) { const length = this.lengths[chunkIndex]; if (remaining < length) return { chunkIndex, offset: remaining }; remaining -= length; } throw new ColQLError("COLQL_INVALID_ROW_INDEX", `Invalid row index: could not locate row ${String(rowIndex)}.`); } - private ensureWritableChunk(): number { const lastIndex = this.chunks.length - 1; if (lastIndex >= 0 && this.lengths[lastIndex] < this.chunkSize) return lastIndex; this.chunks.push(new BooleanChunk(this.chunkSize)); this.lengths.push(0); return this.chunks.length - 1; } + private locate(rowIndex: number): { chunkIndex: number; offset: number } { this.assertIndex(rowIndex); if (this.packedChunks) return { chunkIndex: Math.floor(rowIndex / this.chunkSize), offset: rowIndex % this.chunkSize }; let remaining = rowIndex; for (let chunkIndex = 0; chunkIndex < this.lengths.length; chunkIndex += 1) { const length = this.lengths[chunkIndex]; if (remaining < length) return { chunkIndex, offset: remaining }; remaining -= length; } throw new ColQLError("COLQL_INVALID_ROW_INDEX", `Invalid row index: could not locate row ${String(rowIndex)}.`); } + private ensureWritableChunk(): number { if (this.packedChunks) { const packedChunkIndex = Math.floor(this.currentRowCount / this.chunkSize); while (this.chunks.length <= packedChunkIndex) { this.chunks.push(new BooleanChunk(this.chunkSize)); this.lengths.push(0); } return packedChunkIndex; } const lastIndex = this.chunks.length - 1; if (lastIndex >= 0 && this.lengths[lastIndex] < this.chunkSize) return lastIndex; this.chunks.push(new BooleanChunk(this.chunkSize)); this.lengths.push(0); return this.chunks.length - 1; } private ensureAppendCapacity(): void { if (this.currentRowCount >= this.logicalCapacity) this.resize(Math.max(1, this.logicalCapacity * 2, this.currentRowCount + 1)); } private removeEmptyChunk(chunkIndex: number): void { if (this.lengths[chunkIndex] === 0) { this.chunks.splice(chunkIndex, 1); this.lengths.splice(chunkIndex, 1); } } private assertIndex(rowIndex: number): void { if (!Number.isInteger(rowIndex) || rowIndex < 0 || rowIndex >= this.currentRowCount) throw new ColQLError("COLQL_INVALID_ROW_INDEX", `Invalid row index: expected integer between 0 and ${Math.max(this.currentRowCount - 1, 0)}, received ${String(rowIndex)}.`); } diff --git a/src/storage/dictionary-column.ts b/src/storage/dictionary-column.ts index 5aa76f1..e13d552 100644 --- a/src/storage/dictionary-column.ts +++ b/src/storage/dictionary-column.ts @@ -19,6 +19,7 @@ export class DictionaryColumnStorage implement private readonly ArrayType: DictionaryCodeArrayConstructor; private currentRowCount = 0; private logicalCapacity = 0; + private packedChunks = true; constructor(private readonly values: Values, capacity: number, data?: DictionaryCodeArray, rowCount = data?.length ?? 0, private readonly chunkSize = DEFAULT_CHUNK_SIZE) { assertDictionaryValues(values); @@ -68,6 +69,7 @@ export class DictionaryColumnStorage implement if (offset < length - 1) chunk.copyWithin(offset, offset + 1, length); this.lengths[chunkIndex] -= 1; this.currentRowCount -= 1; + if (chunkIndex < this.chunks.length - 1) this.packedChunks = false; this.removeEmptyChunk(chunkIndex); } @@ -114,6 +116,7 @@ export class DictionaryColumnStorage implement this.lengths.length = 0; this.lengths.push(...nextLengths); this.currentRowCount = nextRowCount; + this.packedChunks = true; } resize(capacity: number): void { @@ -147,6 +150,13 @@ export class DictionaryColumnStorage implement private locate(rowIndex: number): { chunkIndex: number; offset: number } { this.assertIndex(rowIndex); + if (this.packedChunks) { + return { + chunkIndex: Math.floor(rowIndex / this.chunkSize), + offset: rowIndex % this.chunkSize, + }; + } + let remaining = rowIndex; for (let chunkIndex = 0; chunkIndex < this.lengths.length; chunkIndex += 1) { const length = this.lengths[chunkIndex]; @@ -157,6 +167,15 @@ export class DictionaryColumnStorage implement } private ensureWritableChunk(): number { + if (this.packedChunks) { + const packedChunkIndex = Math.floor(this.currentRowCount / this.chunkSize); + while (this.chunks.length <= packedChunkIndex) { + this.chunks.push(new this.ArrayType(this.chunkSize)); + this.lengths.push(0); + } + return packedChunkIndex; + } + const lastIndex = this.chunks.length - 1; if (lastIndex >= 0 && this.lengths[lastIndex] < this.chunkSize) return lastIndex; this.chunks.push(new this.ArrayType(this.chunkSize)); diff --git a/src/storage/numeric-column.ts b/src/storage/numeric-column.ts index c0faf9c..decc784 100644 --- a/src/storage/numeric-column.ts +++ b/src/storage/numeric-column.ts @@ -31,6 +31,7 @@ export class NumericColumnStorage implements ColumnStorage { private readonly ArrayType: NumericArrayConstructor; private currentRowCount = 0; private logicalCapacity = 0; + private packedChunks = true; constructor( private readonly columnType: NumericColumnType, @@ -87,6 +88,7 @@ export class NumericColumnStorage implements ColumnStorage { if (offset < length - 1) chunk.copyWithin(offset, offset + 1, length); this.lengths[chunkIndex] -= 1; this.currentRowCount -= 1; + if (chunkIndex < this.chunks.length - 1) this.packedChunks = false; this.removeEmptyChunk(chunkIndex); } @@ -133,6 +135,7 @@ export class NumericColumnStorage implements ColumnStorage { this.lengths.length = 0; this.lengths.push(...nextLengths); this.currentRowCount = nextRowCount; + this.packedChunks = true; } resize(capacity: number): void { @@ -157,6 +160,13 @@ export class NumericColumnStorage implements ColumnStorage { private locate(rowIndex: number): { chunkIndex: number; offset: number } { this.assertIndex(rowIndex); + if (this.packedChunks) { + return { + chunkIndex: Math.floor(rowIndex / this.chunkSize), + offset: rowIndex % this.chunkSize, + }; + } + let remaining = rowIndex; for (let chunkIndex = 0; chunkIndex < this.lengths.length; chunkIndex += 1) { const length = this.lengths[chunkIndex]; @@ -167,6 +177,15 @@ export class NumericColumnStorage implements ColumnStorage { } private ensureWritableChunk(): number { + if (this.packedChunks) { + const packedChunkIndex = Math.floor(this.currentRowCount / this.chunkSize); + while (this.chunks.length <= packedChunkIndex) { + this.chunks.push(new this.ArrayType(this.chunkSize)); + this.lengths.push(0); + } + return packedChunkIndex; + } + const lastIndex = this.chunks.length - 1; if (lastIndex >= 0 && this.lengths[lastIndex] < this.chunkSize) return lastIndex; this.chunks.push(new this.ArrayType(this.chunkSize)); diff --git a/src/table.ts b/src/table.ts index 99c5782..5190a5c 100644 --- a/src/table.ts +++ b/src/table.ts @@ -939,10 +939,11 @@ export class Table { } matchesFilter(rowIndex: number, filter: InternalFilter): boolean { - const left = this.getComparableValue( - rowIndex, - filter.columnName as keyof TSchema, - ); + const key = filter.columnName as keyof TSchema; + const storage = this.storages[key]; + const left = storage instanceof DictionaryColumnStorage + ? storage.getCode(rowIndex) + : (storage.get(rowIndex) as number | boolean); const { operator, value } = filter; if (operator === "in" || operator === "not in") { diff --git a/tests/application-scenarios.test.ts b/tests/application-scenarios.test.ts new file mode 100644 index 0000000..c5e8a2d --- /dev/null +++ b/tests/application-scenarios.test.ts @@ -0,0 +1,242 @@ +import { describe, expect, it } from "vitest"; +import { ColQLError, column, table, type RowForSchema } from "../src"; + +function expectCode(fn: () => unknown, code: string): void { + expect(fn).toThrow(ColQLError); + try { + fn(); + } catch (error) { + expect((error as ColQLError).code).toBe(code); + } +} + +describe("application scenarios", () => { + it("supports a user directory with stable id and email lookups", () => { + const schema = { + id: column.uint32(), + email: column.dictionary(["a@example.com", "b@example.com", "c@example.com", "d@example.com", "e@example.com"] as const), + country: column.dictionary(["US", "TR", "DE"] as const), + age: column.uint8(), + active: column.boolean(), + score: column.uint32(), + }; + type User = RowForSchema; + const rows: User[] = [ + { id: 1, email: "a@example.com", country: "US", age: 21, active: true, score: 90 }, + { id: 2, email: "b@example.com", country: "TR", age: 34, active: true, score: 70 }, + { id: 3, email: "c@example.com", country: "US", age: 42, active: false, score: 50 }, + { id: 4, email: "d@example.com", country: "DE", age: 28, active: true, score: 80 }, + ]; + const users = table(schema) + .insertMany(rows) + .createUniqueIndex("id") + .createUniqueIndex("email") + .createIndex("country") + .createSortedIndex("age"); + const oracle = rows.map((row) => ({ ...row })); + + expect(users.findBy("id", 2)).toEqual(oracle.find((row) => row.id === 2)); + expect(users.findBy("email", "c@example.com")).toEqual(oracle.find((row) => row.email === "c@example.com")); + + expect(users.updateBy("email", "b@example.com", { active: false, score: 75 })).toEqual({ affectedRows: 1 }); + Object.assign(oracle.find((row) => row.email === "b@example.com")!, { active: false, score: 75 }); + + expect(users.deleteBy("id", 3)).toEqual({ affectedRows: 1 }); + oracle.splice(oracle.findIndex((row) => row.id === 3), 1); + + users.insert({ id: 3, email: "c@example.com", country: "TR", age: 37, active: true, score: 95 }); + oracle.push({ id: 3, email: "c@example.com", country: "TR", age: 37, active: true, score: 95 }); + + expectCode(() => users.insert({ id: 5, email: "a@example.com", country: "US", age: 22, active: true, score: 1 }), "COLQL_DUPLICATE_KEY"); + expect(users.where({ country: "TR", age: { gte: 30 } }).select(["id", "email", "score"]).toArray()).toEqual( + oracle + .filter((row) => row.country === "TR" && row.age >= 30) + .map((row) => ({ id: row.id, email: row.email, score: row.score })), + ); + }); + + it("supports a product catalog with SKU uniqueness", () => { + const schema = { + sku: column.dictionary(["SKU-1", "SKU-2", "SKU-3", "SKU-4", "SKU-5"] as const), + category: column.dictionary(["books", "games", "tools"] as const), + price: column.uint32(), + stock: column.uint16(), + active: column.boolean(), + }; + type Product = RowForSchema; + const rows: Product[] = [ + { sku: "SKU-1", category: "books", price: 1500, stock: 10, active: true }, + { sku: "SKU-2", category: "games", price: 6000, stock: 0, active: false }, + { sku: "SKU-3", category: "books", price: 2500, stock: 2, active: true }, + { sku: "SKU-4", category: "tools", price: 4000, stock: 5, active: true }, + ]; + const products = table(schema).insertMany(rows).createUniqueIndex("sku").createIndex("category").createSortedIndex("price"); + const oracle = rows.map((row) => ({ ...row })); + + expect(products.findBy("sku", "SKU-3")).toEqual(oracle.find((row) => row.sku === "SKU-3")); + expect(products.where({ category: "books", price: { gte: 2000, lte: 3000 } }).toArray()).toEqual( + oracle.filter((row) => row.category === "books" && row.price >= 2000 && row.price <= 3000), + ); + + expect(products.updateBy("sku", "SKU-1", { stock: 9 })).toEqual({ affectedRows: 1 }); + oracle.find((row) => row.sku === "SKU-1")!.stock = 9; + + expect(products.deleteMany({ active: false, stock: 0 })).toEqual({ affectedRows: 1 }); + oracle.splice(oracle.findIndex((row) => !row.active && row.stock === 0), 1); + + products.insert({ sku: "SKU-2", category: "games", price: 5500, stock: 3, active: true }); + oracle.push({ sku: "SKU-2", category: "games", price: 5500, stock: 3, active: true }); + + const before = products.toArray(); + expectCode( + () => + products.insertMany([ + { sku: "SKU-5", category: "tools", price: 1000, stock: 1, active: true }, + { sku: "SKU-5", category: "books", price: 1100, stock: 2, active: true }, + ]), + "COLQL_DUPLICATE_KEY", + ); + expect(products.toArray()).toEqual(before); + expect(products.toArray()).toEqual(oracle); + }); + + it("supports a session token registry", () => { + const schema = { + token: column.dictionary(["t1", "t2", "t3", "t4", "t5"] as const), + userId: column.uint32(), + expiresAt: column.uint32(), + revoked: column.boolean(), + }; + const sessions = table(schema) + .insertMany([ + { token: "t1", userId: 1, expiresAt: 100, revoked: false }, + { token: "t2", userId: 1, expiresAt: 200, revoked: false }, + { token: "t3", userId: 2, expiresAt: 50, revoked: false }, + ]) + .createUniqueIndex("token") + .createIndex("userId") + .createSortedIndex("expiresAt"); + + expect(sessions.findBy("token", "t2")).toEqual({ token: "t2", userId: 1, expiresAt: 200, revoked: false }); + expect(sessions.updateBy("token", "t2", { revoked: true })).toEqual({ affectedRows: 1 }); + expect(sessions.findBy("token", "t2")).toEqual({ token: "t2", userId: 1, expiresAt: 200, revoked: true }); + + expect(sessions.deleteMany({ expiresAt: { lt: 100 } })).toEqual({ affectedRows: 1 }); + sessions.insert({ token: "t3", userId: 3, expiresAt: 300, revoked: false }); + expect(sessions.findBy("token", "t3")).toEqual({ token: "t3", userId: 3, expiresAt: 300, revoked: false }); + expectCode(() => sessions.insert({ token: "t1", userId: 9, expiresAt: 999, revoked: false }), "COLQL_DUPLICATE_KEY"); + }); + + it("supports a feature flag rule table", () => { + const schema = { + key: column.dictionary(["checkout", "search", "profile", "billing"] as const), + environment: column.dictionary(["dev", "staging", "prod"] as const), + enabled: column.boolean(), + rollout: column.uint8(), + }; + const rows = [ + { key: "checkout", environment: "prod", enabled: true, rollout: 25 }, + { key: "search", environment: "prod", enabled: false, rollout: 0 }, + { key: "profile", environment: "staging", enabled: true, rollout: 100 }, + ] as const; + const flags = table(schema).insertMany(rows).createUniqueIndex("key").createIndex("environment"); + const oracle = rows.map((row) => ({ ...row })); + + expect(flags.findBy("key", "checkout")).toEqual(oracle.find((row) => row.key === "checkout")); + expect(flags.updateBy("key", "search", { enabled: true, rollout: 10 })).toEqual({ affectedRows: 1 }); + Object.assign(oracle.find((row) => row.key === "search")!, { enabled: true, rollout: 10 }); + + expect(flags.where({ environment: "prod", enabled: true }).toArray()).toEqual( + oracle.filter((row) => row.environment === "prod" && row.enabled), + ); + expectCode(() => flags.createUniqueIndex("enabled" as never), "COLQL_UNIQUE_INDEX_UNSUPPORTED"); + }); + + it("keeps a mixed long operation sequence equal to a JS array oracle", () => { + const schema = { + id: column.uint32(), + age: column.uint8(), + score: column.uint32(), + status: column.dictionary(["active", "passive", "archived"] as const), + active: column.boolean(), + }; + type User = RowForSchema; + const seedRows = (start: number, count: number): User[] => + Array.from({ length: count }, (_unused, offset) => { + const id = start + offset; + return { + id, + age: (id * 7) % 100, + score: (id * 11) % 1_000, + status: id % 3 === 0 ? "active" : id % 3 === 1 ? "passive" : "archived", + active: id % 4 !== 0, + }; + }); + const rows = seedRows(0, 2_000); + let users = table(schema).insertMany(rows).createUniqueIndex("id").createIndex("status").createSortedIndex("age"); + const oracle = rows.map((row) => ({ ...row })); + const expectParity = () => { + expect(users.where({ status: "active", age: { gte: 50 } }).toArray()).toEqual( + oracle.filter((row) => row.status === "active" && row.age >= 50), + ); + expect(users.where("id", "in", [10, 500, 1_500, 2_100]).toArray()).toEqual( + oracle.filter((row) => [10, 500, 1_500, 2_100].includes(row.id)), + ); + expect(users.countWhere({ active: true })).toBe(oracle.filter((row) => row.active).length); + }; + + expectParity(); + users.updateMany({ status: "passive", age: { gte: 40 } }, { status: "active", score: 999 }); + for (const row of oracle) { + if (row.status === "passive" && row.age >= 40) Object.assign(row, { status: "active", score: 999 }); + } + expectParity(); + + users.updateBy("id", 123, { age: 88 }); + oracle.find((row) => row.id === 123)!.age = 88; + expectParity(); + + users.deleteMany({ status: "archived", age: { lt: 30 } }); + for (let index = oracle.length - 1; index >= 0; index -= 1) { + if (oracle[index].status === "archived" && oracle[index].age < 30) oracle.splice(index, 1); + } + expectParity(); + + const inserted = seedRows(2_000, 100); + users.insertMany(inserted); + oracle.push(...inserted.map((row) => ({ ...row }))); + users.rebuildIndexes().rebuildUniqueIndexes(); + expectParity(); + + users = table.deserialize(users.serialize()).createUniqueIndex("id").createIndex("status").createSortedIndex("age") as typeof users; + expect(users.toArray()).toEqual(oracle); + expectParity(); + }); + + it("handles unique-index edge cases", () => { + const schema = { + id: column.uint32(), + status: column.dictionary(["active", "passive"] as const), + }; + + const empty = table(schema).createUniqueIndex("id"); + expect(empty.findBy("id", 1)).toBeUndefined(); + empty.insert({ id: 1, status: "active" }); + expect(empty.updateBy("id", 1, { id: 1 })).toEqual({ affectedRows: 1 }); + expectCode(() => empty.insert({ id: 2, status: "passive" }).updateBy("id", 1, { id: 2 }), "COLQL_DUPLICATE_KEY"); + + empty.deleteMany({ id: { in: [1, 2] } }); + expect(empty.toArray()).toEqual([]); + empty.insert({ id: 1, status: "passive" }); + expect(empty.findBy("id", 1)).toEqual({ id: 1, status: "passive" }); + + const duplicates = table(schema).insertMany([ + { id: 1, status: "active" }, + { id: 2, status: "passive" }, + ]); + duplicates.createUniqueIndex("id").dropUniqueIndex("id"); + duplicates.insert({ id: 1, status: "passive" }); + expect(duplicates.where("id", "=", 1).count()).toBe(2); + expectCode(() => duplicates.createUniqueIndex("id"), "COLQL_DUPLICATE_KEY"); + }); +});