diff --git a/.github/workflows/ingestion.yml b/.github/workflows/ingestion.yml index e0e90c880..6d9edc2c2 100644 --- a/.github/workflows/ingestion.yml +++ b/.github/workflows/ingestion.yml @@ -5,6 +5,8 @@ on: branches: [main] paths: - "lib/ingestion/**" + - "lib/supabase/ingestion_imports/**" + - "types/database.types.ts" - "lib/mcp/tools/ingestion/**" - "lib/mcp/tools/index.ts" - "lib/supabase/storage/*Ingestion*.ts" @@ -18,6 +20,8 @@ on: branches: [main] paths: - "lib/ingestion/**" + - "lib/supabase/ingestion_imports/**" + - "types/database.types.ts" - "lib/mcp/tools/ingestion/**" - "lib/mcp/tools/index.ts" - "lib/supabase/storage/*Ingestion*.ts" diff --git a/lib/ingestion/CATALOG.md b/lib/ingestion/CATALOG.md new file mode 100644 index 000000000..b014fe046 --- /dev/null +++ b/lib/ingestion/CATALOG.md @@ -0,0 +1,62 @@ +# Organization datasets in Recoup + +Organization imports use the existing Recoup Supabase project and its existing organization membership model. No per-organization database, schema, credentials or new tenant identifier is required. External database delivery is an optional separate operation. + +## Storage and ownership + +- Bronze originals, silver records, gold artifacts and manifests remain in the existing private ingestion bucket, under `organization//...`. +- `ingestion_datasets` groups related imports inside one organization. The first import creates a dataset named after the source filename; later imports can explicitly reuse its `dataset_id`. +- `ingestion_imports` records the immutable manifest and each import's processing result. Held and rejected imports retain their evidence without queryable gold rows. +- `ingestion_records` stores ready gold as JSONB records, preserving identifiers and exact decimals as strings. Field types are recorded per import. Shared indexes begin with organization and import identifiers. + +An import is a versioned population, not an automatic append into an org-wide financial ledger. Reading records requires an explicit dataset and job ID. New imports never overwrite earlier versions, and Recoup does not implicitly combine versions or deduplicate economic events across files. + +The database RPC checks current membership inside every transaction and locks the membership through commit. All three tables have RLS enabled and no browser or service-role table grants. Only the service role can invoke the guarded RPC; it receives the actor from verified API/MCP authentication. Service credentials are trusted server credentials and must never reach a browser. A SQL function cannot authenticate a fabricated actor supplied by a compromised service credential. + +## Create and process + +Use the existing `create` operation with `organization_id`, source and contract. The response includes `dataset_id`. No destination configuration is needed for Recoup's own database. + +To add another import to that dataset, pass its `dataset_id` and a new idempotency key: + +```json +{ + "action": "create", + "organization_id": "22222222-2222-4222-8222-222222222222", + "dataset_id": "33333333-3333-4333-8333-333333333333", + "idempotency_key": "income-2026-12-v1", + "source": { "kind": "connector", "connector_id": "report-snapshot" }, + "contract": { + "version": "income-v1", + "fields": [{ "name": "id", "source": "ID", "type": "string" }] + } +} +``` + +Repeating the exact create request can also index a previously completed private import into the shared catalog without recapturing or reprocessing its original; its gold bytes are integrity-checked first. + +The selected dataset is part of the immutable input fingerprint. It must belong to the selected organization. Personal account imports retain their existing private artifact behavior and cannot select an organization dataset. + +Workers store validated ready gold automatically in Recoup. This is internal dataset storage, not business approval or external delivery. Ready rows and their completion receipt commit together in one database transaction. The private completion marker is written afterward. If its write or the database acknowledgement fails, retrying converges on the same rows; database import state can briefly lead the private artifact marker. Infrastructure failures leave retryable queued work rather than a fabricated successful result. + +## Read through HTTP or MCP + +The standard authenticated `ingestion` tool and `POST /api/ingestion` accept: + +```json +{ "action": "datasets", "organization_id": "", "limit": 25 } +{ "action": "imports", "organization_id": "", "dataset_id": "", "limit": 25 } +{ "action": "records", "organization_id": "", "dataset_id": "", "job_id": "", "limit": 25, "after_row": 0 } +``` + +Read-only GET uses the same fields as query parameters, for example `/api/ingestion?action=datasets&organization_id=`. Create and delivery remain POST-only. Authentication is unchanged; never pass `account_id`. + +Dataset/import pages contain `datasets` or `imports` and `next_after`. Supply that cursor as `after` for the next page. Record pages contain `records: [{ row_number, record }]`, field types, the gold digest and `next_after`; supply that number as `after_row`. A null cursor means the page is complete. Lists allow 1–50 items; record pages allow 1–100 with a 2 MiB record-payload budget, so a page can contain fewer rows than requested. A single row larger than that budget returns HTTP 413 rather than being omitted; those records remain stored, but reading them requires a future large-object export path. Each query is scoped again in the database. Per-minute limits are 60 dataset/import list calls and 30 record page calls per actor and organization. + +## Rollout and verification — 2026-11-09 + +Apply Database's `20261110010000_ingestion_catalog.sql` and `20261110010100_ingestion_catalog_page_budget.sql` in order to the existing project before deploying/enabling these API consumers. Keep `INGESTION_ENABLED` off until private storage, the existing rate counter, Redis dispatch storage and hosted Workflow execution are qualified. No new database needs provisioning. Local disposable databases are verification fixtures only. + +Local database tests exercise the actual migration, cross-organization denial, revocation, concurrent retries, exact values, immutable versions, private table/function grants, pagination and rollback. An optional joined test runs this API's real processor against the same disposable database and verifies recovery after a committed write loses its acknowledgement. The fixture schema includes only accounts and memberships plus the new migration; it is not a replay of the full historical Supabase schema or hosted acceptance. + +Do not remove persisted datasets as a routine rollback. Revert the API consumer first and retain the additive tables and historical receipts. No production migration or intake activation is performed by these tests. diff --git a/lib/ingestion/README.md b/lib/ingestion/README.md index 32c276f82..a17e979e3 100644 --- a/lib/ingestion/README.md +++ b/lib/ingestion/README.md @@ -2,6 +2,8 @@ Private, versioned processing from source files or configured JSON APIs through bronze, silver and gold into export artifacts or an external Postgres database. All code lives in this API repository. +Organization-owned imports can also store queryable gold rows in Recoup's existing shared database. See [Organization datasets](./CATALOG.md) for the default internal destination, migration order and list/record operations. No database per organization is required. + ## Operating contract - **Bronze:** exact original bytes, SHA-256, retrieval time and source occurrence. Private originals are never uploaded to public media storage. @@ -19,7 +21,7 @@ Disabled by default. Before enabling a deployment: 1. Provision a dedicated empty **private** Supabase Storage bucket with a 16 MiB `file_size_limit` and set `INGESTION_STORAGE_BUCKET`. Allow `application/octet-stream` and ensure the project upload ceiling permits 16 MiB. Keep its object policies server-only; do not give anonymous/authenticated clients direct reads, writes or bucket updates. The service verifies the bucket is private before every storage operation. Retention and region are operator decisions; this implementation never deletes originals automatically. 2. Configure the API's existing Supabase service credentials and Vercel Workflow runtime. Manifests, originals, results and stage outputs are durable private objects. The existing service-only `take_site_request` primitive from the Sites activity migration is required for per-minute counters. Configure `INGESTION_REDIS_URL` (or the existing `REDIS_URL`) for atomic dispatch leases; acquisition and dispatch fail closed when their controls are unavailable. -3. Optionally configure source connectors and destinations below. Verify access with synthetic fixtures before production data. No source or destination is inferred from client names or request URLs. +3. For organization imports, apply `20261110010000_ingestion_catalog.sql` followed by `20261110010100_ingestion_catalog_page_budget.sql` in the existing database first; see [Organization datasets](./CATALOG.md). Optionally configure source connectors and destinations below. Verify access with synthetic fixtures before production data. No source or destination is inferred from client names or request URLs. 4. Set `INGESTION_ENABLED=true` only after deployment qualification. There is no added billing or recurring schedule. Operations are exposed by `POST /api/ingestion`, read-only `GET /api/ingestion?job_id=...&stage=gold` (omit `stage` for job state) and the standard authenticated MCP tool `ingestion`. Identity comes from `x-api-key` or Bearer authentication; never supply `account_id`. Optional `organization_id` requires current membership. Requests are limited per authenticated actor, scope and operation: 10 creates/deliveries, 30 artifact reads and 120 status reads per minute, for both HTTP and MCP. Personal and organization namespaces are distinct. Organization-scoped jobs are shared with currently authorized organization members. Workers recheck the initiating actor's access before processing and publication; reads and delivery also recheck access. Delegated OAuth does not expose this tool. diff --git a/lib/ingestion/__tests__/catalog.integration.test.ts b/lib/ingestion/__tests__/catalog.integration.test.ts new file mode 100644 index 000000000..d79a8da73 --- /dev/null +++ b/lib/ingestion/__tests__/catalog.integration.test.ts @@ -0,0 +1,126 @@ +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { Client } from "pg"; +import { randomUUID } from "node:crypto"; +import { processIngestionOperation } from "../processIngestionOperation"; +import { executeIngestionJob } from "../executeIngestionJob"; +import type { IngestionDependencies } from "../types"; +const socket = process.env.INGESTION_CATALOG_TEST_SOCKET; +const suite = socket ? describe : describe.skip; +suite("real shared-database ingestion lifecycle", () => { + let admin: Client; + let gateway: Client; + const actor = randomUUID(), + org = randomUUID(), + other = randomUUID(); + beforeAll(async () => { + if (!socket || !/\/recoup-ingestion-[A-Za-z0-9_-]+$/.test(socket)) + throw new Error("Disposable ingestion test socket required"); + admin = new Client({ host: socket, port: 55489, database: "postgres" }); + gateway = new Client({ host: socket, port: 55489, database: "postgres" }); + await admin.connect(); + await admin.query("INSERT INTO public.accounts(id) SELECT unnest($1::uuid[])", [ + [actor, org, other], + ]); + await admin.query( + "INSERT INTO public.account_organization_ids(account_id,organization_id) VALUES ($1,$2)", + [actor, org], + ); + await gateway.connect(); + await gateway.query("SET ROLE service_role"); + }); + afterAll(async () => { + await gateway?.end(); + await admin?.end(); + }); + it("ingests into the shared database, recovers lost acknowledgement and refuses another org", async () => { + const objects = new Map(); + let loseAck = true; + const deps: IngestionDependencies = { + // Deliberately bypass the API membership check here to verify the real database boundary. + authorize: async (_actor, organization) => `organization/${organization}`, + limit: async () => undefined, + store: { + read: async k => objects.get(k) ?? null, + create: async (k, v) => { + if (objects.has(k)) return false; + objects.set(k, v); + return true; + }, + }, + dispatch: async () => undefined, + acquire: async () => { + throw new Error("No external source"); + }, + deliver: async () => { + throw new Error("No external database"); + }, + catalog: async (account, organization, action, input) => { + const result = await gateway.query( + "SELECT public.ingestion_catalog_command($1,$2,$3,$4::jsonb) AS result", + [account, organization, action, JSON.stringify(input)], + ); + if (action === "complete" && loseAck) { + loseAck = false; + throw new Error("Acknowledgement lost"); + } + return result.rows[0].result; + }, + }; + const input = { + action: "create", + organization_id: org, + idempotency_key: "shared-income-1", + source: { + kind: "inline", + filename: "income.csv", + format: "csv", + content_base64: Buffer.from("ID,Amount\n001,0.1\n002,0.2").toString("base64"), + }, + contract: { + version: "1", + fields: [ + { name: "id", source: "ID", type: "string" }, + { name: "amount", source: "Amount", type: "decimal" }, + ], + controls: [{ field: "amount", expected: "0.3" }], + }, + }; + const job = await processIngestionOperation(actor, input, deps); + expect(job.dataset_id).toMatch(/^[a-f0-9-]{36}$/); + await expect(executeIngestionJob(actor, org, job.job_id as string, deps)).rejects.toThrow( + "Acknowledgement lost", + ); + expect([...objects.keys()].some(k => k.endsWith("/result.json"))).toBe(false); + await executeIngestionJob(actor, org, job.job_id as string, deps); + expect((await processIngestionOperation(actor, input, deps)).dataset_id).toBe(job.dataset_id); + const page = await processIngestionOperation( + actor, + { action: "records", organization_id: org, dataset_id: job.dataset_id, job_id: job.job_id }, + deps, + ); + expect(page.records).toEqual([ + { row_number: 1, record: { id: "001", amount: "0.1" } }, + { row_number: 2, record: { id: "002", amount: "0.2" } }, + ]); + await expect( + processIngestionOperation( + actor, + { + action: "records", + organization_id: other, + dataset_id: job.dataset_id, + job_id: job.job_id, + }, + deps, + ), + ).rejects.toThrow("Ingestion access denied"); + await admin.query("DELETE FROM public.account_organization_ids WHERE account_id=$1", [actor]); + await expect( + processIngestionOperation( + actor, + { action: "records", organization_id: org, dataset_id: job.dataset_id, job_id: job.job_id }, + deps, + ), + ).rejects.toThrow("Ingestion access denied"); + }); +}); diff --git a/lib/ingestion/__tests__/catalog.test.ts b/lib/ingestion/__tests__/catalog.test.ts new file mode 100644 index 000000000..2823aa370 --- /dev/null +++ b/lib/ingestion/__tests__/catalog.test.ts @@ -0,0 +1,242 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { processIngestionOperation } from "../processIngestionOperation"; +import { executeIngestionJob } from "../executeIngestionJob"; +import { readIngestionQuery } from "../readIngestionQuery"; +import type { IngestionDependencies } from "../types"; +const actor = "11111111-1111-4111-8111-111111111111"; +const organization = "22222222-2222-4222-8222-222222222222"; +const dataset = "33333333-3333-4333-8333-333333333333"; +const create = { + action: "create", + organization_id: organization, + idempotency_key: "test-import", + source: { + kind: "inline", + filename: "income.csv", + format: "csv", + content_base64: Buffer.from("ID,Amount\n001,0.3").toString("base64"), + }, + contract: { + version: "1", + fields: [ + { name: "id", source: "ID", type: "string" }, + { name: "amount", source: "Amount", type: "decimal" }, + ], + }, +}; +function harness() { + const objects = new Map(); + const catalog = vi.fn().mockResolvedValue({ dataset_id: dataset }); + const dispatch = vi.fn(); + const deps: IngestionDependencies = { + authorize: async (account, org) => { + if (account !== actor || (org && org !== organization)) throw new Error("Access denied"); + return org ? `organization/${org}` : `account/${actor}`; + }, + limit: async () => undefined, + store: { + read: async k => objects.get(k) ?? null, + create: async (k, v) => { + if (objects.has(k)) return false; + objects.set(k, v); + return true; + }, + }, + dispatch, + catalog, + acquire: async () => { + throw new Error("Not configured"); + }, + deliver: async () => ({ committed: true, rows: 1 }), + }; + return { objects, catalog, dispatch, deps }; +} +describe("organization ingestion catalog", () => { + beforeEach(() => vi.clearAllMocks()); + it("registers imports in the shared database before dispatch and returns their dataset", async () => { + const h = harness(); + const job = await processIngestionOperation(actor, create, h.deps); + expect(job.dataset_id).toBe(dataset); + expect(h.catalog).toHaveBeenCalledWith( + actor, + organization, + "register", + expect.objectContaining({ + manifest: expect.objectContaining({ + scope: `organization/${organization}`, + filename: "income.csv", + }), + }), + ); + expect(h.catalog.mock.invocationCallOrder[0]).toBeLessThan( + h.dispatch.mock.invocationCallOrder[0], + ); + }); + it("publishes exact gold rows into Recoup automatically when processing succeeds", async () => { + const h = harness(); + const job = await processIngestionOperation(actor, create, h.deps); + await executeIngestionJob(actor, organization, job.job_id, h.deps); + const complete = h.catalog.mock.calls.find(c => c[2] === "complete"); + expect(complete?.slice(0, 3)).toEqual([actor, organization, "complete"]); + expect(complete?.[3]).toMatchObject({ + job_id: job.job_id, + result: { status: "ready", rows_output: 1 }, + gold: { records: [{ id: "001", amount: "0.3" }] }, + }); + }); + it("records held imports without publishing any gold rows", async () => { + const h = harness(); + const job = await processIngestionOperation( + actor, + { + ...create, + contract: { ...create.contract, controls: [{ field: "amount", expected: "1" }] }, + }, + h.deps, + ); + await executeIngestionJob(actor, organization, job.job_id, h.deps); + expect(h.catalog.mock.calls.find(c => c[2] === "complete")?.[3]).toMatchObject({ + result: { status: "held", rows_output: 0 }, + gold: null, + }); + }); + it("keeps an import retryable when shared-database completion is unavailable", async () => { + const h = harness(); + const job = await processIngestionOperation(actor, create, h.deps); + h.catalog.mockImplementation(async (_actor, _org, action) => { + if (action === "complete") throw new Error("Catalog unavailable"); + return { dataset_id: dataset }; + }); + await expect(executeIngestionJob(actor, organization, job.job_id, h.deps)).rejects.toThrow( + "Catalog unavailable", + ); + expect([...h.objects.keys()].some(k => k.endsWith("/result.json"))).toBe(false); + h.catalog.mockResolvedValue({ dataset_id: dataset }); + await executeIngestionJob(actor, organization, job.job_id, h.deps); + expect([...h.objects.keys()].some(k => k.endsWith("/result.json"))).toBe(true); + }); + it("repairs the shared catalog when replaying an already completed private import", async () => { + const h = harness(); + const job = await processIngestionOperation(actor, create, h.deps); + await executeIngestionJob(actor, organization, job.job_id, h.deps); + h.catalog.mockClear(); + h.dispatch.mockClear(); + await processIngestionOperation(actor, create, h.deps); + expect(h.catalog.mock.calls.some(c => c[2] === "complete")).toBe(true); + expect(h.dispatch).not.toHaveBeenCalled(); + }); + it.each(["missing", "corrupt"])( + "does not register a replay whose saved gold is %s", + async kind => { + const h = harness(); + const job = await processIngestionOperation(actor, create, h.deps); + await executeIngestionJob(actor, organization, job.job_id, h.deps); + h.catalog.mockClear(); + const goldKey = `organization/${organization}/${job.job_id}/gold.json`; + if (kind === "missing") h.objects.delete(goldKey); + else h.objects.set(goldKey, Buffer.from("corrupt")); + await expect(processIngestionOperation(actor, create, h.deps)).rejects.toThrow( + "Gold integrity check failed", + ); + expect(h.catalog).not.toHaveBeenCalled(); + }, + ); + it("rejects malformed stored completion metadata before registering a replay", async () => { + const h = harness(); + const job = await processIngestionOperation(actor, create, h.deps); + h.catalog.mockClear(); + h.objects.set( + `organization/${organization}/${job.job_id}/result.json`, + Buffer.from(JSON.stringify({ status: "ready", gold_sha256: 123 })), + ); + await expect(processIngestionOperation(actor, create, h.deps)).rejects.toThrow( + "Invalid ingestion completion", + ); + expect(h.catalog).not.toHaveBeenCalled(); + }); + it.each([ + { file: "result", token: '\"expected\":\"', error: "Invalid ingestion completion" }, + { file: "manifest", token: '\"filename\":\"', error: "Invalid ingestion manifest" }, + ])("rejects malformed UTF-8 in saved $file metadata", async ({ file, token, error }) => { + const h = harness(); + const input = { + ...create, + contract: { ...create.contract, controls: [{ field: "amount", expected: "0.3" }] }, + }; + const job = await processIngestionOperation(actor, input, h.deps); + await executeIngestionJob(actor, organization, job.job_id, h.deps); + const key = `organization/${organization}/${job.job_id}/${file}.json`; + const bytes = Buffer.from(h.objects.get(key)!); + const offset = bytes.indexOf(token); + expect(offset).toBeGreaterThanOrEqual(0); + bytes[offset + Buffer.byteLength(token)] = 255; + h.objects.set(key, bytes); + h.catalog.mockClear(); + await expect(processIngestionOperation(actor, input, h.deps)).rejects.toThrow(error); + expect(h.catalog).not.toHaveBeenCalled(); + }); + it("binds the selected dataset to the immutable import contract", async () => { + const h = harness(); + const job = await processIngestionOperation(actor, { ...create, dataset_id: dataset }, h.deps); + const manifest = JSON.parse( + Buffer.from( + h.objects.get(`organization/${organization}/${job.job_id}/manifest.json`)!, + ).toString(), + ); + expect(manifest.dataset_id).toBe(dataset); + await expect( + processIngestionOperation(actor, { ...create, dataset_id: actor }, h.deps), + ).rejects.toThrow("Idempotency conflict"); + await expect( + processIngestionOperation( + actor, + { ...create, organization_id: undefined, dataset_id: dataset }, + h.deps, + ), + ).rejects.toThrow(); + }); + it("lists datasets and immutable import rows only through the authenticated organization", async () => { + const h = harness(); + h.catalog.mockResolvedValue({ + records: [{ row_number: 1, record: { id: "001" } }], + next_after: null, + }); + const result = await processIngestionOperation( + actor, + { + action: "records", + organization_id: organization, + dataset_id: dataset, + job_id: "a".repeat(64), + limit: 25, + after_row: 0, + }, + h.deps, + ); + expect(result.records).toHaveLength(1); + expect(h.catalog).toHaveBeenCalledWith(actor, organization, "records", { + dataset_id: dataset, + job_id: "a".repeat(64), + limit: 25, + after_row: 0, + }); + await expect( + processIngestionOperation(actor, { action: "datasets" }, h.deps), + ).rejects.toThrow(); + }); + it("rejects catalog reads before querying when organization membership is denied", async () => { + const h = harness(); + await expect( + processIngestionOperation(actor, { action: "datasets", organization_id: dataset }, h.deps), + ).rejects.toThrow("Access denied"); + expect(h.catalog).not.toHaveBeenCalled(); + }); + it("adapts read-only catalog queries without allowing GET mutations", () => { + expect( + readIngestionQuery( + `https://test/api/ingestion?action=datasets&organization_id=${organization}&limit=10`, + ), + ).toEqual({ action: "datasets", organization_id: organization, limit: 10 }); + expect(() => readIngestionQuery("https://test/api/ingestion?action=create")).toThrow(); + }); +}); diff --git a/lib/ingestion/__tests__/catalogDatabase.test.ts b/lib/ingestion/__tests__/catalogDatabase.test.ts new file mode 100644 index 000000000..efc613f8b --- /dev/null +++ b/lib/ingestion/__tests__/catalogDatabase.test.ts @@ -0,0 +1,44 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { getIngestionCatalogResult } from "@/lib/supabase/ingestion_imports/getIngestionCatalogResult"; +const rpc = vi.hoisted(() => vi.fn()); +vi.mock("@/lib/supabase/serverClient", () => ({ default: { rpc } })); +describe("shared ingestion database gateway", () => { + beforeEach(() => vi.clearAllMocks()); + it("passes the authenticated actor and organization to the guarded RPC", async () => { + rpc.mockResolvedValue({ data: { datasets: [], next_after: null }, error: null }); + expect( + await getIngestionCatalogResult("actor", "organization", "datasets", { limit: 10 }), + ).toEqual({ datasets: [], next_after: null }); + expect(rpc).toHaveBeenCalledWith("ingestion_catalog_command", { + p_actor: "actor", + p_org: "organization", + p_action: "datasets", + p_input: { limit: 10 }, + }); + }); + it("classifies denied database membership without exposing database errors", async () => { + rpc.mockResolvedValue({ data: null, error: { code: "42501", message: "secret details" } }); + await expect( + getIngestionCatalogResult("actor", "organization", "datasets", {}), + ).rejects.toEqual(new Error("Access denied")); + rpc.mockResolvedValue({ + data: null, + error: { code: "P0001", message: "Ingestion catalog conflict" }, + }); + await expect( + getIngestionCatalogResult("actor", "organization", "datasets", {}), + ).rejects.toThrow("Ingestion catalog conflict"); + }); + it("fails closed on unavailable or malformed database responses", async () => { + for (const response of [ + { data: null, error: { code: "42P01", message: "private table" } }, + { data: null, error: null }, + { data: [], error: null }, + ]) { + rpc.mockResolvedValue(response); + await expect( + getIngestionCatalogResult("actor", "organization", "datasets", {}), + ).rejects.toThrow("Ingestion catalog unavailable"); + } + }); +}); diff --git a/lib/ingestion/__tests__/failure.test.ts b/lib/ingestion/__tests__/failure.test.ts index 42a97d1c2..26d37e8af 100644 --- a/lib/ingestion/__tests__/failure.test.ts +++ b/lib/ingestion/__tests__/failure.test.ts @@ -8,6 +8,12 @@ describe("safe ingestion failures", () => { reason: "operation_unavailable", }); }); + it("reports an oversized shared-database record as a size error", () => { + expect(getIngestionFailure(new Error("Ingestion record size limit"))).toMatchObject({ + status: 413, + reason: "record_size_limit", + }); + }); it("explains the size limit for oversized requests", () => { expect(getIngestionFailure(new Error("Request too large"))).toMatchObject({ status: 413, diff --git a/lib/ingestion/__tests__/operations.test.ts b/lib/ingestion/__tests__/operations.test.ts index 3506fc8ef..291896deb 100644 --- a/lib/ingestion/__tests__/operations.test.ts +++ b/lib/ingestion/__tests__/operations.test.ts @@ -21,6 +21,7 @@ function harness() { let allowed = true; const deps: IngestionDependencies = { limit: async () => undefined, + catalog: async () => ({ dataset_id: other }), authorize: async (actor, organization) => { if (!allowed) throw new Error("Access denied"); return organization ? `organization/${organization}` : `account/${actor}`; diff --git a/lib/ingestion/createIngestionDependencies.ts b/lib/ingestion/createIngestionDependencies.ts index ac646d833..d3ddf1271 100644 --- a/lib/ingestion/createIngestionDependencies.ts +++ b/lib/ingestion/createIngestionDependencies.ts @@ -1,3 +1,4 @@ +import { getIngestionCatalogResult } from "@/lib/supabase/ingestion_imports/getIngestionCatalogResult"; import { authorizeIngestionScope } from "./authorizeIngestionScope"; import { readIngestionObject } from "@/lib/supabase/storage/readIngestionObject"; import { createIngestionObject } from "@/lib/supabase/storage/createIngestionObject"; @@ -13,6 +14,7 @@ export function createIngestionDependencies( const store = { read: readIngestionObject, create: createIngestionObject }; return { authorize: authorizeIngestionScope, + catalog: getIngestionCatalogResult, limit: limitIngestionRequest, store, acquire: acquireIngestionSource, diff --git a/lib/ingestion/executeIngestionJob.ts b/lib/ingestion/executeIngestionJob.ts index 81ab0469b..deca9a61a 100644 --- a/lib/ingestion/executeIngestionJob.ts +++ b/lib/ingestion/executeIngestionJob.ts @@ -16,6 +16,7 @@ export async function executeIngestionJob( const manifest = await readIngestionManifest(deps.store, scope, jobId); const prefix = `${scope}/${jobId}`; if (await deps.store.read(`${prefix}/result.json`)) return; + if (organization) await deps.catalog(actor, scope.split("/")[1], "register", { manifest }); if (manifest.engine_version !== INGESTION_ENGINE_VERSION) throw new Error("Engine version unavailable"); const bronze = await deps.store.read(manifest.bronze_key); @@ -45,22 +46,27 @@ export async function executeIngestionJob( await writeIngestionArtifact(deps.store, `${prefix}/silver.json`, silver); if (gold) await writeIngestionArtifact(deps.store, `${prefix}/gold.json`, gold); if ((await deps.authorize(actor, organization)) !== scope) throw new Error("Scope changed"); - await writeIngestionArtifact( - deps.store, - `${prefix}/result.json`, - encode({ + const completion = { + job_id: jobId, + status: processed.status, + engine_version: manifest.engine_version, + source_sha256: manifest.bronze_sha256, + rows_read: rowsRead, + rows_output: processed.gold?.records.length ?? 0, + issues: processed.issues, + issue_count: processed.issue_count, + issues_truncated: Math.max(0, processed.issue_count - processed.issues.length), + controls: processed.controls, + silver_sha256: hashIngestionBytes(silver), + gold_sha256: gold ? hashIngestionBytes(gold) : null, + }; + if (organization) { + await deps.catalog(actor, scope.split("/")[1], "complete", { job_id: jobId, - status: processed.status, - engine_version: manifest.engine_version, - source_sha256: manifest.bronze_sha256, - rows_read: rowsRead, - rows_output: processed.gold?.records.length ?? 0, - issues: processed.issues, - issue_count: processed.issue_count, - issues_truncated: Math.max(0, processed.issue_count - processed.issues.length), - controls: processed.controls, - silver_sha256: hashIngestionBytes(silver), - gold_sha256: gold ? hashIngestionBytes(gold) : null, - }), - ); + result: completion, + gold: processed.gold, + }); + if ((await deps.authorize(actor, organization)) !== scope) throw new Error("Scope changed"); + } + await writeIngestionArtifact(deps.store, `${prefix}/result.json`, encode(completion)); } diff --git a/lib/ingestion/getIngestionFailure.ts b/lib/ingestion/getIngestionFailure.ts index 2eff54b67..f2a1223a1 100644 --- a/lib/ingestion/getIngestionFailure.ts +++ b/lib/ingestion/getIngestionFailure.ts @@ -14,6 +14,8 @@ export function getIngestionFailure(error: unknown) { "Request too large": [413, "request_size_limit"], "Access denied": [403, "access_denied"], "Scope changed": [409, "scope_changed"], + "Ingestion record size limit": [413, "record_size_limit"], + "Ingestion catalog conflict": [409, "catalog_conflict"], "Idempotency conflict": [409, "idempotency_conflict"], "Gold selection mismatch": [409, "gold_selection_mismatch"], "Artifact not ready": [409, "artifact_not_ready"], diff --git a/lib/ingestion/ingestionCompletionSchema.ts b/lib/ingestion/ingestionCompletionSchema.ts new file mode 100644 index 000000000..ad12d332a --- /dev/null +++ b/lib/ingestion/ingestionCompletionSchema.ts @@ -0,0 +1,44 @@ +import { z } from "zod"; +import { ingestionColumnNameSchema } from "./ingestionFieldSchema"; +const digest = z.string().regex(/^[a-f0-9]{64}$/); +const evidence = z.strictObject({ + job_id: digest, + engine_version: z.string().min(1).max(100), + source_sha256: digest, + rows_read: z.number().int().min(0).max(10000), + rows_output: z.number().int().min(0).max(10000), + issues: z + .array( + z.strictObject({ + code: z.string().min(1).max(64), + row: z.number().int().positive().optional(), + field: ingestionColumnNameSchema.optional(), + }), + ) + .max(1000), + issue_count: z.number().int().min(0).max(10000000), + issues_truncated: z.number().int().min(0).max(10000000), + controls: z + .array( + z.strictObject({ + field: ingestionColumnNameSchema, + expected: z.string().max(100), + actual: z.string().max(1000), + passed: z.boolean(), + }), + ) + .max(100), + silver_sha256: digest, +}); +export const ingestionCompletionSchema = z + .discriminatedUnion("status", [ + evidence.extend({ status: z.literal("ready"), gold_sha256: digest }), + evidence.extend({ status: z.enum(["held", "rejected"]), gold_sha256: z.null() }), + ]) + .refine( + v => + v.rows_output <= v.rows_read && + v.issue_count - v.issues.length === v.issues_truncated && + (v.status !== "ready" || (v.issue_count === 0 && v.controls.every(c => c.passed))), + "Invalid completion evidence", + ); diff --git a/lib/ingestion/ingestionContractSchema.ts b/lib/ingestion/ingestionContractSchema.ts new file mode 100644 index 000000000..8cf9e021e --- /dev/null +++ b/lib/ingestion/ingestionContractSchema.ts @@ -0,0 +1,76 @@ +import { z } from "zod"; +import { + ingestionColumnNameSchema as name, + ingestionDecimalSchema as decimal, + ingestionFieldSchema as field, +} from "./ingestionFieldSchema"; +export const ingestionContractSchema = z + .strictObject({ + version: z.string().min(1).max(100), + fields: z.array(field).min(1).max(100), + business_keys: z.array(name).max(20).default([]), + sheet: z.string().min(1).max(200).optional(), + header_row: z.number().int().min(1).max(1000).default(1), + records_path: z.array(z.string().min(1).max(100)).max(10).default([]), + delimiter: z.enum([",", ";", "\t", "|"]).default(","), + ignored_columns: z.array(z.string().min(1).max(200)).max(100).default([]), + controls: z + .array( + z.strictObject({ + field: name, + expected: decimal, + tolerance: decimal.default("0"), + where: z.record(name, z.string().max(2000)).default({}), + }), + ) + .max(100) + .default([]), + aggregate: z + .strictObject({ group_by: z.array(name).max(20), sums: z.array(name).min(1).max(20) }) + .optional(), + text_table: z + .strictObject({ + review_ref: z.string().min(1).max(200), + separator: z + .string() + .min(1) + .max(10) + .refine(s => !/[\r\n]/.test(s), "A separator cannot contain line breaks"), + }) + .optional(), + }) + .superRefine((c, ctx) => { + const names = c.fields.map(f => f.name); + const sources = c.fields.flatMap(f => (f.source === undefined ? [] : [f.source])); + if (new Set(names).size !== names.length || new Set(sources).size !== sources.length) + ctx.addIssue({ code: "custom", message: "Field names and source columns must be unique" }); + if ( + c.ignored_columns.some(n => sources.includes(n)) || + new Set(c.ignored_columns).size !== c.ignored_columns.length + ) + ctx.addIssue({ code: "custom", message: "Ignored columns cannot be mapped or duplicated" }); + const references = [ + ...c.business_keys, + ...c.controls.flatMap(r => [r.field, ...Object.keys(r.where)]), + ...(c.aggregate?.group_by ?? []), + ...(c.aggregate?.sums ?? []), + ]; + if (references.some(n => !names.includes(n))) + ctx.addIssue({ code: "custom", message: "Unknown field reference" }); + const numeric = (n: string) => + c.fields.some(f => f.name === n && ["decimal", "integer"].includes(f.type)); + if ( + c.controls.some(r => !numeric(r.field) || r.tolerance.startsWith("-")) || + c.aggregate?.sums.some(n => !numeric(n)) + ) + ctx.addIssue({ + code: "custom", + message: "Controls and sums need numeric fields and nonnegative tolerances", + }); + const grouped = [...(c.aggregate?.group_by ?? []), ...(c.aggregate?.sums ?? [])]; + if ( + new Set(grouped).size !== grouped.length || + new Set(c.business_keys).size !== c.business_keys.length + ) + ctx.addIssue({ code: "custom", message: "Duplicate key or aggregate reference" }); + }); diff --git a/lib/ingestion/ingestionFieldSchema.ts b/lib/ingestion/ingestionFieldSchema.ts new file mode 100644 index 000000000..6b87abe6e --- /dev/null +++ b/lib/ingestion/ingestionFieldSchema.ts @@ -0,0 +1,20 @@ +import { z } from "zod"; +export const ingestionFieldTypeSchema = z.enum(["string", "decimal", "integer", "date", "boolean"]); +export const ingestionColumnNameSchema = z.string().regex(/^[a-z][a-z0-9_]{0,62}$/); +export const ingestionDecimalSchema = z + .string() + .max(100) + .regex(/^[+-]?\d+(?:\.\d+)?$/); +export const ingestionFieldSchema = z + .strictObject({ + name: ingestionColumnNameSchema, + source: z.string().min(1).max(200).optional(), + constant: z.string().max(2000).optional(), + type: ingestionFieldTypeSchema, + nullable: z.boolean().default(false), + trim: z.boolean().default(false), + decimal_separator: z.enum([".", ","]).default("."), + thousands_separator: z.enum(["", ",", ".", " "]).default(""), + }) + .refine(f => (f.source !== undefined) !== (f.constant !== undefined), "Choose source or constant") + .refine(f => f.decimal_separator !== f.thousands_separator, "Separators must differ"); diff --git a/lib/ingestion/ingestionFormatSchema.ts b/lib/ingestion/ingestionFormatSchema.ts new file mode 100644 index 000000000..2a1146607 --- /dev/null +++ b/lib/ingestion/ingestionFormatSchema.ts @@ -0,0 +1,2 @@ +import { z } from "zod"; +export const ingestionFormatSchema = z.enum(["csv", "tsv", "json", "jsonl", "xlsx", "pdf"]); diff --git a/lib/ingestion/ingestionManifestSchema.ts b/lib/ingestion/ingestionManifestSchema.ts index fa7706c87..9f8f78dab 100644 --- a/lib/ingestion/ingestionManifestSchema.ts +++ b/lib/ingestion/ingestionManifestSchema.ts @@ -4,6 +4,7 @@ const digest = z.string().regex(/^[a-f0-9]{64}$/); export const ingestionManifestSchema = z .strictObject({ job_id: digest, + dataset_id: z.uuid().optional(), input_sha256: digest, engine_version: z.string().min(1).max(100), scope: z.string().regex(/^(account|organization)\/[a-f0-9-]{36}$/), diff --git a/lib/ingestion/ingestionOperationSchema.ts b/lib/ingestion/ingestionOperationSchema.ts new file mode 100644 index 000000000..5401268a9 --- /dev/null +++ b/lib/ingestion/ingestionOperationSchema.ts @@ -0,0 +1,74 @@ +import { z } from "zod"; +import { ingestionContractSchema } from "./ingestionContractSchema"; +import { ingestionFormatSchema } from "./ingestionFormatSchema"; +const key = z + .string() + .min(1) + .max(128) + .regex(/^[A-Za-z0-9._:-]+$/); +export const ingestionOperationSchema = z.discriminatedUnion("action", [ + z + .strictObject({ + action: z.literal("create"), + organization_id: z.uuid().optional(), + dataset_id: z.uuid().optional(), + idempotency_key: key, + source: z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("inline"), + filename: z.string().min(1).max(200), + format: ingestionFormatSchema, + content_base64: z + .string() + .min(4) + .max(2796204) + .regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/), + }), + z.strictObject({ kind: z.literal("connector"), connector_id: key }), + ]), + contract: ingestionContractSchema, + }) + .refine(v => !v.dataset_id || !!v.organization_id, "Datasets require an organization"), + z.strictObject({ + action: z.literal("datasets"), + organization_id: z.uuid(), + limit: z.number().int().min(1).max(50).default(25), + after: z.uuid().optional(), + }), + z.strictObject({ + action: z.literal("imports"), + organization_id: z.uuid(), + dataset_id: z.uuid(), + limit: z.number().int().min(1).max(50).default(25), + after: z + .string() + .regex(/^[a-f0-9]{64}$/) + .optional(), + }), + z.strictObject({ + action: z.literal("records"), + organization_id: z.uuid(), + dataset_id: z.uuid(), + job_id: z.string().regex(/^[a-f0-9]{64}$/), + limit: z.number().int().min(1).max(100).default(25), + after_row: z.number().int().min(0).max(10000).default(0), + }), + z.strictObject({ + action: z.literal("read"), + organization_id: z.uuid().optional(), + job_id: z.string().regex(/^[a-f0-9]{64}$/), + }), + z.strictObject({ + action: z.literal("artifact"), + organization_id: z.uuid().optional(), + job_id: z.string().regex(/^[a-f0-9]{64}$/), + stage: z.enum(["bronze", "silver", "gold"]), + }), + z.strictObject({ + action: z.literal("deliver"), + organization_id: z.uuid().optional(), + job_id: z.string().regex(/^[a-f0-9]{64}$/), + destination_id: key, + gold_sha256: z.string().regex(/^[a-f0-9]{64}$/), + }), +]); diff --git a/lib/ingestion/limitIngestionRequest.ts b/lib/ingestion/limitIngestionRequest.ts index 37392cf97..2282bf2af 100644 --- a/lib/ingestion/limitIngestionRequest.ts +++ b/lib/ingestion/limitIngestionRequest.ts @@ -7,7 +7,15 @@ export async function limitIngestionRequest( scope: string, action: Parameters[2], ) { - const limits = { create: 10, deliver: 10, read: 120, artifact: 30 }; + const limits = { + create: 10, + deliver: 10, + read: 120, + artifact: 30, + datasets: 60, + imports: 60, + records: 30, + }; if ( !(await takeSiteRequest( hashIngestionValue({ domain: "ingestion", actor, scope, action }), diff --git a/lib/ingestion/processIngestionOperation.ts b/lib/ingestion/processIngestionOperation.ts index 1888326f2..e7e0279ab 100644 --- a/lib/ingestion/processIngestionOperation.ts +++ b/lib/ingestion/processIngestionOperation.ts @@ -1,6 +1,7 @@ import { ingestionOperationSchema } from "./schemas"; import { hashIngestionValue } from "./hashIngestionValue"; import { hashIngestionBytes } from "./hashIngestionBytes"; +import { readIngestionCompletion } from "./readIngestionCompletion"; import { readIngestionManifest } from "./readIngestionManifest"; import { writeIngestionArtifact } from "./writeIngestionArtifact"; import { INGESTION_ENGINE_VERSION, INGESTION_MAX_BYTES } from "./const"; @@ -19,6 +20,12 @@ export async function processIngestionOperation( if ((await deps.authorize(actor, args.organization_id)) !== scope) throw new Error("Scope changed"); }; + if (args.action === "datasets" || args.action === "imports" || args.action === "records") { + const { action, organization_id: _organization, ...query } = args; + const result = await deps.catalog(actor, scope.split("/")[1], action, query); + await recheck(); + return result; + } const encode = (value: unknown) => Buffer.from(JSON.stringify(value)); if (args.action === "create") { const jobId = hashIngestionValue({ scope, idempotency_key: args.idempotency_key }); @@ -27,6 +34,7 @@ export async function processIngestionOperation( source: args.source, contract: args.contract, engine_version: INGESTION_ENGINE_VERSION, + ...(args.dataset_id ? { dataset_id: args.dataset_id.toLowerCase() } : {}), }); let saved = await deps.store.read(`${prefix}/manifest.json`); if (!saved) { @@ -51,6 +59,7 @@ export async function processIngestionOperation( await writeIngestionArtifact(deps.store, bronzeKey, source.bytes); const manifest: IngestionManifest = { job_id: jobId, + ...(args.dataset_id ? { dataset_id: args.dataset_id.toLowerCase() } : {}), input_sha256: fingerprint, engine_version: INGESTION_ENGINE_VERSION, scope, @@ -78,19 +87,33 @@ export async function processIngestionOperation( ) throw new Error("Idempotency conflict"); const completed = await deps.store.read(`${prefix}/result.json`); + const result = completed ? readIngestionCompletion(completed, manifest) : null; + let gold: GoldArtifact | null = null; + if (result?.status === "ready" && args.organization_id) { + const bytes = await deps.store.read(`${prefix}/gold.json`); + if (!bytes || hashIngestionBytes(bytes) !== result.gold_sha256) + throw new Error("Gold integrity check failed"); + gold = JSON.parse(new TextDecoder().decode(bytes)); + } + const catalog = scope.startsWith("organization/") + ? await deps.catalog(actor, scope.split("/")[1], "register", { manifest }) + : {}; + if (result && args.organization_id) + await deps.catalog(actor, scope.split("/")[1], "complete", { job_id: jobId, result, gold }); await recheck(); if (!completed) await deps.dispatch(actor, args.organization_id, jobId); await recheck(); return { + ...catalog, + ...(result ?? { status: "queued" as const }), job_id: jobId, - ...(completed ? JSON.parse(new TextDecoder().decode(completed)) : { status: "queued" }), }; } const manifest = await readIngestionManifest(deps.store, scope, args.job_id); const prefix = `${scope}/${args.job_id}`; const marker = await deps.store.read(`${prefix}/result.json`); const result = marker - ? JSON.parse(new TextDecoder().decode(marker)) + ? readIngestionCompletion(marker, manifest) : { job_id: args.job_id, status: "queued", gold_sha256: null }; if (args.action === "read") { await recheck(); @@ -113,7 +136,12 @@ export async function processIngestionOperation( }, }; } - const digest = result[`${args.stage}_sha256`]; + const digest = + args.stage === "gold" + ? result.gold_sha256 + : "silver_sha256" in result + ? result.silver_sha256 + : null; if (!digest || (args.stage === "gold" && result.status !== "ready")) throw new Error("Artifact not ready"); const bytes = await deps.store.read(`${prefix}/${args.stage}.json`); diff --git a/lib/ingestion/readIngestionCompletion.ts b/lib/ingestion/readIngestionCompletion.ts new file mode 100644 index 000000000..17d181764 --- /dev/null +++ b/lib/ingestion/readIngestionCompletion.ts @@ -0,0 +1,19 @@ +import { ingestionCompletionSchema } from "./ingestionCompletionSchema"; +import type { IngestionManifest } from "./types"; +/** Validate private stored metadata before it can drive shared publication or artifact selection. */ +export function readIngestionCompletion(bytes: Uint8Array, manifest: IngestionManifest) { + try { + const result = ingestionCompletionSchema.parse( + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)), + ); + if ( + result.job_id !== manifest.job_id || + result.source_sha256 !== manifest.bronze_sha256 || + result.engine_version !== manifest.engine_version + ) + throw new Error("Completion does not match manifest"); + return result; + } catch { + throw new Error("Invalid ingestion completion"); + } +} diff --git a/lib/ingestion/readIngestionManifest.ts b/lib/ingestion/readIngestionManifest.ts index 628600add..b8577f77c 100644 --- a/lib/ingestion/readIngestionManifest.ts +++ b/lib/ingestion/readIngestionManifest.ts @@ -13,7 +13,9 @@ export async function readIngestionManifest( if (!bytes) throw new Error("Ingestion job not found"); let manifest: IngestionManifest; try { - manifest = ingestionManifestSchema.parse(JSON.parse(new TextDecoder().decode(bytes))); + manifest = ingestionManifestSchema.parse( + JSON.parse(new TextDecoder("utf-8", { fatal: true }).decode(bytes)), + ); } catch { throw new Error("Invalid ingestion manifest"); } diff --git a/lib/ingestion/readIngestionQuery.ts b/lib/ingestion/readIngestionQuery.ts index b18c4b0c3..8891885f9 100644 --- a/lib/ingestion/readIngestionQuery.ts +++ b/lib/ingestion/readIngestionQuery.ts @@ -1,9 +1,24 @@ import { z } from "zod"; +import { ingestionOperationSchema } from "./schemas"; /** GET is read-only; do not let query arguments select a mutation action. */ export function readIngestionQuery(url: string) { const params = new URL(url).searchParams; for (const name of params.keys()) if (params.getAll(name).length !== 1) throw new Error("Invalid ingestion query"); + const action = params.get("action"); + if (action !== null) { + if (!["datasets", "imports", "records"].includes(action)) + throw new Error("Invalid ingestion query"); + const input: Record = Object.fromEntries(params); + for (const key of ["limit", "after_row"]) { + if (params.has(key)) { + const text = params.get(key)!; + if (!/^\d{1,5}$/.test(text)) throw new Error("Invalid ingestion query"); + input[key] = Number(text); + } + } + return ingestionOperationSchema.parse(input); + } const query = z .strictObject({ job_id: z.string().regex(/^[a-f0-9]{64}$/), diff --git a/lib/ingestion/schemas.ts b/lib/ingestion/schemas.ts index f2c64f049..0df255440 100644 --- a/lib/ingestion/schemas.ts +++ b/lib/ingestion/schemas.ts @@ -1,137 +1,3 @@ -import { z } from "zod"; - -const name = z.string().regex(/^[a-z][a-z0-9_]{0,62}$/); -const decimal = z - .string() - .max(100) - .regex(/^[+-]?\d+(?:\.\d+)?$/); -const field = z - .strictObject({ - name, - source: z.string().min(1).max(200).optional(), - constant: z.string().max(2000).optional(), - type: z.enum(["string", "decimal", "integer", "date", "boolean"]), - nullable: z.boolean().default(false), - trim: z.boolean().default(false), - decimal_separator: z.enum([".", ","]).default("."), - thousands_separator: z.enum(["", ",", ".", " "]).default(""), - }) - .refine(f => (f.source !== undefined) !== (f.constant !== undefined), "Choose source or constant") - .refine(f => f.decimal_separator !== f.thousands_separator, "Separators must differ"); - -export const ingestionContractSchema = z - .strictObject({ - version: z.string().min(1).max(100), - fields: z.array(field).min(1).max(100), - business_keys: z.array(name).max(20).default([]), - sheet: z.string().min(1).max(200).optional(), - header_row: z.number().int().min(1).max(1000).default(1), - records_path: z.array(z.string().min(1).max(100)).max(10).default([]), - delimiter: z.enum([",", ";", "\t", "|"]).default(","), - ignored_columns: z.array(z.string().min(1).max(200)).max(100).default([]), - controls: z - .array( - z.strictObject({ - field: name, - expected: decimal, - tolerance: decimal.default("0"), - where: z.record(name, z.string().max(2000)).default({}), - }), - ) - .max(100) - .default([]), - aggregate: z - .strictObject({ group_by: z.array(name).max(20), sums: z.array(name).min(1).max(20) }) - .optional(), - text_table: z - .strictObject({ - review_ref: z.string().min(1).max(200), - separator: z - .string() - .min(1) - .max(10) - .refine(s => !/[\r\n]/.test(s), "A separator cannot contain line breaks"), - }) - .optional(), - }) - .superRefine((c, ctx) => { - const names = c.fields.map(f => f.name); - const sources = c.fields.flatMap(f => (f.source === undefined ? [] : [f.source])); - if (new Set(names).size !== names.length || new Set(sources).size !== sources.length) - ctx.addIssue({ code: "custom", message: "Field names and source columns must be unique" }); - if ( - c.ignored_columns.some(n => sources.includes(n)) || - new Set(c.ignored_columns).size !== c.ignored_columns.length - ) - ctx.addIssue({ code: "custom", message: "Ignored columns cannot be mapped or duplicated" }); - const references = [ - ...c.business_keys, - ...c.controls.flatMap(r => [r.field, ...Object.keys(r.where)]), - ...(c.aggregate?.group_by ?? []), - ...(c.aggregate?.sums ?? []), - ]; - if (references.some(n => !names.includes(n))) - ctx.addIssue({ code: "custom", message: "Unknown field reference" }); - const numeric = (n: string) => - c.fields.some(f => f.name === n && ["decimal", "integer"].includes(f.type)); - if ( - c.controls.some(r => !numeric(r.field) || r.tolerance.startsWith("-")) || - c.aggregate?.sums.some(n => !numeric(n)) - ) - ctx.addIssue({ - code: "custom", - message: "Controls and sums need numeric fields and nonnegative tolerances", - }); - const grouped = [...(c.aggregate?.group_by ?? []), ...(c.aggregate?.sums ?? [])]; - if ( - new Set(grouped).size !== grouped.length || - new Set(c.business_keys).size !== c.business_keys.length - ) - ctx.addIssue({ code: "custom", message: "Duplicate key or aggregate reference" }); - }); - -export const ingestionFormatSchema = z.enum(["csv", "tsv", "json", "jsonl", "xlsx", "pdf"]); -const key = z - .string() - .min(1) - .max(128) - .regex(/^[A-Za-z0-9._:-]+$/); -export const ingestionOperationSchema = z.discriminatedUnion("action", [ - z.strictObject({ - action: z.literal("create"), - organization_id: z.uuid().optional(), - idempotency_key: key, - source: z.discriminatedUnion("kind", [ - z.strictObject({ - kind: z.literal("inline"), - filename: z.string().min(1).max(200), - format: ingestionFormatSchema, - content_base64: z - .string() - .min(4) - .max(2796204) - .regex(/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/), - }), - z.strictObject({ kind: z.literal("connector"), connector_id: key }), - ]), - contract: ingestionContractSchema, - }), - z.strictObject({ - action: z.literal("read"), - organization_id: z.uuid().optional(), - job_id: z.string().regex(/^[a-f0-9]{64}$/), - }), - z.strictObject({ - action: z.literal("artifact"), - organization_id: z.uuid().optional(), - job_id: z.string().regex(/^[a-f0-9]{64}$/), - stage: z.enum(["bronze", "silver", "gold"]), - }), - z.strictObject({ - action: z.literal("deliver"), - organization_id: z.uuid().optional(), - job_id: z.string().regex(/^[a-f0-9]{64}$/), - destination_id: key, - gold_sha256: z.string().regex(/^[a-f0-9]{64}$/), - }), -]); +export { ingestionContractSchema } from "./ingestionContractSchema"; +export { ingestionFormatSchema } from "./ingestionFormatSchema"; +export { ingestionOperationSchema } from "./ingestionOperationSchema"; diff --git a/lib/ingestion/types.ts b/lib/ingestion/types.ts index f8e9674b9..48fbcade9 100644 --- a/lib/ingestion/types.ts +++ b/lib/ingestion/types.ts @@ -43,8 +43,14 @@ export interface IngestionDependencies { limit( actor: string, scope: string, - action: "create" | "read" | "artifact" | "deliver", + action: "create" | "read" | "artifact" | "deliver" | "datasets" | "imports" | "records", ): Promise; + catalog( + actor: string, + organization: string, + action: "register" | "complete" | "datasets" | "imports" | "records", + input: Record, + ): Promise>; dispatch(actor: string, organization: string | undefined, jobId: string): Promise; acquire( scope: string, diff --git a/lib/mcp/tools/ingestion/registerIngestionTool.ts b/lib/mcp/tools/ingestion/registerIngestionTool.ts index c0a7ea2b1..d9234417a 100644 --- a/lib/mcp/tools/ingestion/registerIngestionTool.ts +++ b/lib/mcp/tools/ingestion/registerIngestionTool.ts @@ -9,7 +9,7 @@ export function registerIngestionTool(server: McpServer) { "ingestion", { description: - "Create private data ingestion jobs from CSV, TSV, JSON/JSONL, XLSX or reviewed PDF text layouts; read job validation and bronze/silver/gold artifacts. Supply explicit field mappings and controls. Held or rejected jobs have no publishable gold. Deliver only an explicitly approved exact gold_sha256 to an operator-configured destination_id. Never infer financial definitions or approve delivery from a filename. Private file bytes use base64; API sources use configured connector IDs. Large artifacts should be read through HTTP. This tool is unavailable for delegated OAuth.", + "Create private data ingestion jobs from CSV, TSV, JSON/JSONL, XLSX or reviewed PDF text layouts; read job validation and bronze/silver/gold artifacts. Organization imports that reach ready status store their gold records in Recoup's existing database; optionally supply dataset_id to add a new immutable import to a dataset. Use datasets, imports and records to browse one organization and explicit import version. Supply explicit field mappings and controls. Held or rejected jobs have no publishable gold. Deliver only an explicitly approved exact gold_sha256 to an operator-configured destination_id. Never infer financial definitions or approve delivery from a filename. Private file bytes use base64; API sources use configured connector IDs. Large artifacts should be read through HTTP. This tool is unavailable for delegated OAuth.", inputSchema: ingestionOperationSchema, annotations: { readOnlyHint: false, destructiveHint: true, openWorldHint: true }, }, diff --git a/lib/supabase/ingestion_imports/getIngestionCatalogResult.ts b/lib/supabase/ingestion_imports/getIngestionCatalogResult.ts new file mode 100644 index 000000000..08d827f7a --- /dev/null +++ b/lib/supabase/ingestion_imports/getIngestionCatalogResult.ts @@ -0,0 +1,35 @@ +import supabase from "@/lib/supabase/serverClient"; +import type { Json } from "@/types/database.types"; +import type { IngestionDependencies } from "@/lib/ingestion/types"; + +/** Service-only gateway; the database rechecks and locks organization membership per transaction. */ +export const getIngestionCatalogResult: IngestionDependencies["catalog"] = async ( + actor, + organization, + action, + input, +) => { + const { data, error } = await supabase.rpc("ingestion_catalog_command", { + p_actor: actor, + p_org: organization, + p_action: action, + p_input: input as Json, + }); + if (error) { + console.error("[ingestion] catalog command failed", { + code: /^[A-Z0-9]{5}$/.test(error.code ?? "") ? error.code : "unknown", + }); + if (error.code === "42501") throw new Error("Access denied"); + if ( + error.code === "P0001" && + ["Ingestion catalog conflict", "Artifact not ready", "Ingestion record size limit"].includes( + error.message, + ) + ) + throw new Error(error.message); + throw new Error("Ingestion catalog unavailable"); + } + if (!data || typeof data !== "object" || Array.isArray(data)) + throw new Error("Ingestion catalog unavailable"); + return data; +}; diff --git a/types/database.types.ts b/types/database.types.ts index 878c3d0b7..be17469f6 100644 --- a/types/database.types.ts +++ b/types/database.types.ts @@ -6219,6 +6219,10 @@ export type Database = { [_ in never]: never; }; Functions: { + ingestion_catalog_command: { + Args: { p_actor: string; p_org: string; p_action: string; p_input?: Json }; + Returns: Json; + }; accept_context_result: { Args: { p_document: string;