From 5f7ad4ff4882bda1fb5a661e7f44bf54ca159357 Mon Sep 17 00:00:00 2001 From: chitcommit <208086304+chitcommit@users.noreply.github.com> Date: Fri, 10 Apr 2026 23:29:26 +0000 Subject: [PATCH] feat: Mercury webhook real-time COA classification + ChittySchema integration MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mercury webhook (/api/webhooks/mercury) now persists transactions to the DB with auto-classified suggestedCoaCode populated at ingest time. This completes the L0 ingest path of the trust-path model for the real-time Mercury flow (previously the webhook only deduped and logged, never persisted). Flow: 1. Service-auth check (Bearer token) 2. Zod envelope validation 3. KV idempotency (7-day TTL) 4. Advisory ChittySchema validation (never blocks) 5. DB dedup via externalId (retries-safe) 6. createTransaction with suggestedCoaCode + classificationConfidence 7. ledgerLog audit entry Auto-classification uses findAccountCode() keyword matcher (same path as TurboTenant import). Suspense (9010) gets 0.100 confidence; matched codes get 0.700. ChittySchema integration (server/lib/chittyschema.ts): - New Workers-native client — no process.env at module load - Speaks the real ChittySchema API (/api/health, /api/validate, /api/tables) - Fall-open on 5xx / network error / timeout / malformed body — never blocks financial writes - Advisory pass when table is not registered (surfaces availableTables so operators can see what's in the registry) - Normalizes error shapes (string[] and object[] both supported) - Three functions: validateRow, listTables, checkHealth Cleanup: - Deleted server/lib/chittyschema-validation.ts — stale, used wrong API paths (/api/v1/validate), had Express middleware (dead code), relied on process.env which doesn't work in Workers. Never imported anywhere. - Removed ChittySchemaClient class and getChittySchema factory from chittyos-client.ts — same issues, never imported. Added CHITTYSCHEMA_URL? to the env type for optional override. Tests (25 new, 246 total): - server/__tests__/chittyschema.test.ts (16): valid/invalid results, fall-open on 5xx/network/timeout/malformed-body, unregistered-table advisory pass, default base URL, listTables, checkHealth - server/__tests__/webhooks-mercury.test.ts (10): auth rejection, invalid envelope, successful persistence with auto-classification, suspense fallback, KV dedup, externalId dedup, non-blocking schema validation failure, unreachable schema, envelope-only ack Co-Authored-By: Claude Opus 4.6 (1M context) --- server/__tests__/chittyschema.test.ts | 179 ++++++++++++ server/__tests__/webhooks-mercury.test.ts | 339 ++++++++++++++++++++++ server/env.ts | 3 + server/lib/chittyos-client.ts | 45 +-- server/lib/chittyschema-validation.ts | 254 ---------------- server/lib/chittyschema.ts | 156 ++++++++++ server/routes/webhooks.ts | 159 +++++++++- 7 files changed, 837 insertions(+), 298 deletions(-) create mode 100644 server/__tests__/chittyschema.test.ts create mode 100644 server/__tests__/webhooks-mercury.test.ts delete mode 100644 server/lib/chittyschema-validation.ts create mode 100644 server/lib/chittyschema.ts diff --git a/server/__tests__/chittyschema.test.ts b/server/__tests__/chittyschema.test.ts new file mode 100644 index 0000000..1c35462 --- /dev/null +++ b/server/__tests__/chittyschema.test.ts @@ -0,0 +1,179 @@ +/** + * Tests for server/lib/chittyschema.ts + * + * Focus: validation result shape, fall-open behavior on errors, + * unknown-table handling. Network is mocked — no calls to schema.chitty.cc. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { validateRow, listTables, checkHealth } from '../lib/chittyschema'; + +const env = { CHITTYSCHEMA_URL: 'https://test.schema' }; + +const originalFetch = global.fetch; + +function mockFetch(handler: (url: string, init?: RequestInit) => Promise | Response) { + global.fetch = vi.fn(handler as any) as any; +} + +beforeEach(() => { + global.fetch = originalFetch; +}); + +afterEach(() => { + global.fetch = originalFetch; +}); + +describe('validateRow', () => { + it('returns ok:true when service confirms valid', async () => { + mockFetch(async () => new Response(JSON.stringify({ valid: true }), { status: 200 })); + const result = await validateRow(env, 'FinancialTransactionsInsertSchema', { amount: '10.00' }); + expect(result.ok).toBe(true); + expect(result.advisory).toBe(false); + expect(result.errors).toBeUndefined(); + }); + + it('returns ok:false with normalized errors on validation failure', async () => { + mockFetch( + async () => + new Response( + JSON.stringify({ + valid: false, + errors: [{ path: 'amount', message: 'required', code: 'required' }], + }), + { status: 200 }, + ), + ); + const result = await validateRow(env, 'FinancialTransactionsInsertSchema', {}); + expect(result.ok).toBe(false); + expect(result.advisory).toBe(false); + expect(result.errors).toHaveLength(1); + expect(result.errors![0].path).toBe('amount'); + }); + + it('normalizes string-array errors (unregistered table response)', async () => { + mockFetch( + async () => + new Response( + JSON.stringify({ valid: false, errors: ['something broke'] }), + { status: 200 }, + ), + ); + const result = await validateRow(env, 'SomeTable', {}); + expect(result.ok).toBe(false); + expect(result.errors![0].message).toBe('something broke'); + }); + + it('returns advisory pass with availableTables when table is not registered', async () => { + mockFetch( + async () => + new Response( + JSON.stringify({ + valid: false, + errors: ['Schema not found for table: chart_of_accounts'], + availableTables: ['Foo', 'Bar'], + }), + { status: 200 }, + ), + ); + const result = await validateRow(env, 'chart_of_accounts', {}); + expect(result.ok).toBe(true); + expect(result.advisory).toBe(true); + expect(result.availableTables).toEqual(['Foo', 'Bar']); + }); + + it('falls open (advisory pass) on 5xx', async () => { + mockFetch(async () => new Response('boom', { status: 503 })); + const result = await validateRow(env, 'Any', {}); + expect(result.ok).toBe(true); + expect(result.advisory).toBe(true); + }); + + it('falls open on network error', async () => { + mockFetch(async () => { + throw new TypeError('fetch failed: ECONNREFUSED'); + }); + const result = await validateRow(env, 'Any', {}); + expect(result.ok).toBe(true); + expect(result.advisory).toBe(true); + }); + + it('falls open on timeout', async () => { + mockFetch(async () => { + const err = new Error('The operation was aborted'); + err.name = 'AbortError'; + throw err; + }); + const result = await validateRow(env, 'Any', {}, { timeoutMs: 10 }); + expect(result.ok).toBe(true); + expect(result.advisory).toBe(true); + }); + + it('falls open when body is not valid JSON', async () => { + mockFetch(async () => new Response('not-json-at-all', { status: 200 })); + const result = await validateRow(env, 'Any', {}); + expect(result.ok).toBe(true); + expect(result.advisory).toBe(true); + }); + + it('uses the default base URL when CHITTYSCHEMA_URL is not set', async () => { + let capturedUrl = ''; + mockFetch(async (url: string) => { + capturedUrl = url; + return new Response(JSON.stringify({ valid: true }), { status: 200 }); + }); + await validateRow({}, 'X', {}); + expect(capturedUrl).toBe('https://schema.chitty.cc/api/validate'); + }); +}); + +describe('listTables', () => { + it('returns parsed tables array', async () => { + mockFetch( + async () => + new Response( + JSON.stringify({ + tables: [ + { name: 'identities', database: 'chittyos-core', owner: 'chittyid' }, + ], + }), + { status: 200 }, + ), + ); + const tables = await listTables(env); + expect(tables).toHaveLength(1); + expect(tables[0].name).toBe('identities'); + }); + + it('returns empty array on fetch error', async () => { + mockFetch(async () => { + throw new Error('network'); + }); + const tables = await listTables(env); + expect(tables).toEqual([]); + }); + + it('returns empty array on non-ok status', async () => { + mockFetch(async () => new Response('x', { status: 500 })); + const tables = await listTables(env); + expect(tables).toEqual([]); + }); +}); + +describe('checkHealth', () => { + it('returns true on 2xx', async () => { + mockFetch(async () => new Response('ok', { status: 200 })); + expect(await checkHealth(env)).toBe(true); + }); + + it('returns false on non-ok status', async () => { + mockFetch(async () => new Response('x', { status: 500 })); + expect(await checkHealth(env)).toBe(false); + }); + + it('returns false on network error', async () => { + mockFetch(async () => { + throw new Error('network'); + }); + expect(await checkHealth(env)).toBe(false); + }); +}); diff --git a/server/__tests__/webhooks-mercury.test.ts b/server/__tests__/webhooks-mercury.test.ts new file mode 100644 index 0000000..266eccb --- /dev/null +++ b/server/__tests__/webhooks-mercury.test.ts @@ -0,0 +1,339 @@ +/** + * Tests for POST /api/webhooks/mercury + * + * Exercises the real-time classification flow end-to-end: + * - service auth + * - envelope validation (zod) + * - KV dedup + * - auto-classification via findAccountCode() + * - ChittySchema advisory validation (never blocks) + * - DB persistence with suggestedCoaCode populated + */ +import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest'; +import { Hono } from 'hono'; +import type { HonoEnv } from '../env'; + +// Module-level mocks — the webhook handler imports these directly, so we +// intercept before the route module loads. vi.mock() is hoisted above +// variable declarations, so we use vi.hoisted() for any fn we reference +// from inside the mock factory. +const { mockCreateTransaction, mockGetByExternalId } = vi.hoisted(() => ({ + mockCreateTransaction: vi.fn(), + mockGetByExternalId: vi.fn(), +})); + +vi.mock('../db/connection', () => ({ + createDb: vi.fn(() => ({ /* drizzle stub */ })), +})); + +vi.mock('../storage/system', () => ({ + SystemStorage: class MockSystemStorage { + getTransactionByExternalId = mockGetByExternalId; + createTransaction = mockCreateTransaction; + }, +})); + +vi.mock('../lib/ledger-client', () => ({ + ledgerLog: vi.fn(), +})); + +const originalFetch = global.fetch; + +// Fake KV implementation — in-memory Map with ttl ignored +function makeKv() { + const store = new Map(); + return { + get: vi.fn(async (k: string) => store.get(k) ?? null), + put: vi.fn(async (k: string, v: string) => { + store.set(k, v); + }), + } as any; +} + +const baseEnv = { + CHITTY_AUTH_SERVICE_TOKEN: 'svc-token', + DATABASE_URL: 'fake://db', + FINANCE_KV: makeKv(), + FINANCE_R2: {} as any, + ASSETS: {} as any, +} as any; + +// Lazy-import the webhook routes so module mocks above take effect +async function getApp() { + const { webhookRoutes } = await import('../routes/webhooks'); + const app = new Hono(); + app.route('/', webhookRoutes); + return app; +} + +const TENANT_ID = '11111111-1111-1111-1111-111111111111'; +const ACCOUNT_ID = '22222222-2222-2222-2222-222222222222'; + +function buildEnvelope(partial: Partial = {}): any { + return { + id: 'evt-1', + type: 'transaction.created', + data: { + transaction: { + tenantId: TENANT_ID, + accountId: ACCOUNT_ID, + mercuryTransactionId: 'mtx-abc', + description: 'Home Depot #1234', + amount: -125.4, + category: 'Repairs', + postedAt: '2026-04-10T12:00:00Z', + ...partial, + }, + }, + }; +} + +beforeEach(() => { + mockCreateTransaction.mockReset(); + mockGetByExternalId.mockReset(); + global.fetch = originalFetch; + baseEnv.FINANCE_KV = makeKv(); +}); + +afterEach(() => { + global.fetch = originalFetch; +}); + +describe('POST /api/webhooks/mercury', () => { + it('rejects missing Bearer token with 401', async () => { + const app = await getApp(); + const res = await app.request( + '/api/webhooks/mercury', + { method: 'POST', body: JSON.stringify(buildEnvelope()) }, + baseEnv, + ); + expect(res.status).toBe(401); + }); + + it('rejects wrong token with 401', async () => { + const app = await getApp(); + const res = await app.request( + '/api/webhooks/mercury', + { + method: 'POST', + headers: { authorization: 'Bearer wrong', 'content-type': 'application/json' }, + body: JSON.stringify(buildEnvelope()), + }, + baseEnv, + ); + expect(res.status).toBe(401); + }); + + it('rejects invalid envelope with 400', async () => { + const app = await getApp(); + const res = await app.request( + '/api/webhooks/mercury', + { + method: 'POST', + headers: { authorization: 'Bearer svc-token', 'content-type': 'application/json' }, + body: JSON.stringify({ data: { transaction: { bad: 'shape' } } }), + }, + baseEnv, + ); + expect(res.status).toBe(400); + const body = (await res.json()) as any; + expect(body.error).toBe('invalid_envelope'); + }); + + it('persists transaction with auto-classified suggestedCoaCode', async () => { + mockGetByExternalId.mockResolvedValue(null); + mockCreateTransaction.mockResolvedValue({ id: 'new-tx-id' }); + // ChittySchema returns valid + global.fetch = vi.fn( + async () => new Response(JSON.stringify({ valid: true }), { status: 200 }), + ) as any; + + const app = await getApp(); + const res = await app.request( + '/api/webhooks/mercury', + { + method: 'POST', + headers: { authorization: 'Bearer svc-token', 'content-type': 'application/json' }, + body: JSON.stringify(buildEnvelope()), + }, + baseEnv, + ); + + expect(res.status).toBe(201); + const body = (await res.json()) as any; + expect(body.received).toBe(true); + expect(body.transactionId).toBe('new-tx-id'); + // "Home Depot" maps to supplies (5080) via TURBOTENANT_CATEGORY_MAP + // and "Repairs" category also matches 5070 — category takes precedence + expect(body.suggestedCoaCode).toBe('5070'); + expect(body.classificationConfidence).toBe('0.700'); + expect(body.schemaAdvisory).toBe(false); + + expect(mockCreateTransaction).toHaveBeenCalledOnce(); + const call = mockCreateTransaction.mock.calls[0][0]; + expect(call.tenantId).toBe(TENANT_ID); + expect(call.accountId).toBe(ACCOUNT_ID); + expect(call.externalId).toBe('mercury:mtx-abc'); + expect(call.suggestedCoaCode).toBe('5070'); + expect(call.type).toBe('expense'); // negative amount + expect(call.metadata).toEqual({ + source: 'mercury_webhook', + mercuryTransactionId: 'mtx-abc', + eventId: 'evt-1', + }); + }); + + it('assigns suspense 9010 with low confidence for unmatchable descriptions', async () => { + mockGetByExternalId.mockResolvedValue(null); + mockCreateTransaction.mockResolvedValue({ id: 'new-tx-id' }); + global.fetch = vi.fn( + async () => new Response(JSON.stringify({ valid: true }), { status: 200 }), + ) as any; + + const app = await getApp(); + const res = await app.request( + '/api/webhooks/mercury', + { + method: 'POST', + headers: { authorization: 'Bearer svc-token', 'content-type': 'application/json' }, + body: JSON.stringify( + buildEnvelope({ description: 'xyz unknown merchant', category: null }), + ), + }, + baseEnv, + ); + + expect(res.status).toBe(201); + const body = (await res.json()) as any; + expect(body.suggestedCoaCode).toBe('9010'); + expect(body.classificationConfidence).toBe('0.100'); + }); + + it('dedups via KV on repeat event id', async () => { + mockGetByExternalId.mockResolvedValue(null); + mockCreateTransaction.mockResolvedValue({ id: 'new-tx-id' }); + global.fetch = vi.fn( + async () => new Response(JSON.stringify({ valid: true }), { status: 200 }), + ) as any; + + const app = await getApp(); + const env = { ...baseEnv, FINANCE_KV: makeKv() }; + + // First request creates + const res1 = await app.request( + '/api/webhooks/mercury', + { + method: 'POST', + headers: { authorization: 'Bearer svc-token', 'content-type': 'application/json' }, + body: JSON.stringify(buildEnvelope()), + }, + env, + ); + expect(res1.status).toBe(201); + + // Second request with same event id is deduped + const res2 = await app.request( + '/api/webhooks/mercury', + { + method: 'POST', + headers: { authorization: 'Bearer svc-token', 'content-type': 'application/json' }, + body: JSON.stringify(buildEnvelope()), + }, + env, + ); + expect(res2.status).toBe(202); + const body = (await res2.json()) as any; + expect(body.duplicate).toBe(true); + expect(mockCreateTransaction).toHaveBeenCalledOnce(); // only the first one persisted + }); + + it('dedups via externalId if KV write was lost between retries', async () => { + mockGetByExternalId.mockResolvedValue({ id: 'existing-tx-id' }); + global.fetch = vi.fn( + async () => new Response(JSON.stringify({ valid: true }), { status: 200 }), + ) as any; + + const app = await getApp(); + const res = await app.request( + '/api/webhooks/mercury', + { + method: 'POST', + headers: { authorization: 'Bearer svc-token', 'content-type': 'application/json' }, + body: JSON.stringify(buildEnvelope()), + }, + baseEnv, + ); + + expect(res.status).toBe(202); + const body = (await res.json()) as any; + expect(body.duplicate).toBe(true); + expect(body.transactionId).toBe('existing-tx-id'); + expect(mockCreateTransaction).not.toHaveBeenCalled(); + }); + + it('persists even when ChittySchema validation fails (advisory, not blocking)', async () => { + mockGetByExternalId.mockResolvedValue(null); + mockCreateTransaction.mockResolvedValue({ id: 'new-tx-id' }); + global.fetch = vi.fn( + async () => + new Response( + JSON.stringify({ valid: false, errors: [{ path: 'amount', message: 'bad' }] }), + { status: 200 }, + ), + ) as any; + + const app = await getApp(); + const res = await app.request( + '/api/webhooks/mercury', + { + method: 'POST', + headers: { authorization: 'Bearer svc-token', 'content-type': 'application/json' }, + body: JSON.stringify(buildEnvelope()), + }, + baseEnv, + ); + + // Still persisted despite schema failure — advisory, not blocking + expect(res.status).toBe(201); + expect(mockCreateTransaction).toHaveBeenCalledOnce(); + }); + + it('persists even when ChittySchema is unreachable', async () => { + mockGetByExternalId.mockResolvedValue(null); + mockCreateTransaction.mockResolvedValue({ id: 'new-tx-id' }); + global.fetch = vi.fn(async () => { + throw new TypeError('fetch failed'); + }) as any; + + const app = await getApp(); + const res = await app.request( + '/api/webhooks/mercury', + { + method: 'POST', + headers: { authorization: 'Bearer svc-token', 'content-type': 'application/json' }, + body: JSON.stringify(buildEnvelope()), + }, + baseEnv, + ); + + expect(res.status).toBe(201); + const body = (await res.json()) as any; + expect(body.schemaAdvisory).toBe(true); + expect(mockCreateTransaction).toHaveBeenCalledOnce(); + }); + + it('acks envelope-only events (no transaction) with 202', async () => { + const app = await getApp(); + const res = await app.request( + '/api/webhooks/mercury', + { + method: 'POST', + headers: { authorization: 'Bearer svc-token', 'content-type': 'application/json' }, + body: JSON.stringify({ id: 'evt-noop', type: 'account.created' }), + }, + baseEnv, + ); + expect(res.status).toBe(202); + expect(mockCreateTransaction).not.toHaveBeenCalled(); + }); +}); diff --git a/server/env.ts b/server/env.ts index 76f0ae6..fe22a36 100644 --- a/server/env.ts +++ b/server/env.ts @@ -14,6 +14,9 @@ export interface Env { CHITTYAGENT_API_BASE?: string; CHITTYAGENT_API_TOKEN?: string; CHITTY_LEDGER_BASE?: string; + // ChittySchema — optional override of https://schema.chitty.cc for the + // centralized schema validation service. Leave unset in production. + CHITTYSCHEMA_URL?: string; AI_GATEWAY_ENDPOINT?: string; // CF AI Gateway proxy URL, e.g. https://gateway.ai.cloudflare.com/v1/{acct}/{gw}/openai CHITTYCONNECT_API_TOKEN?: string; // Valuation API keys (optional — each provider only fetched if key is set) diff --git a/server/lib/chittyos-client.ts b/server/lib/chittyos-client.ts index 4f16716..c33c939 100755 --- a/server/lib/chittyos-client.ts +++ b/server/lib/chittyos-client.ts @@ -257,36 +257,10 @@ export class ChittyConnectClient extends ChittyOSClient { } } -/** - * ChittySchema Client - */ -export class ChittySchemaClient extends ChittyOSClient { - constructor(config?: Partial) { - super({ - baseUrl: config?.baseUrl || process.env.CHITTYSCHEMA_URL || 'https://schema.chitty.cc', - ...config, - }); - } - - async validate(type: string, data: Record): Promise<{ - valid: boolean; - errors?: Array<{ path: string; message: string; code: string }>; - }> { - return this.post('/api/v1/validate', { type, data }); - } - - async getEntityTypes(): Promise> { - const response = await this.get<{ types: any[] }>('/api/v1/entity-types'); - return response.types; - } - - async getSchema(type: string): Promise> { - return this.get(`/api/v1/schema/${type}`); - } -} +// NOTE: ChittySchemaClient removed — it used the wrong API paths (/api/v1/*) +// and process.env which doesn't work in Cloudflare Workers at module load. +// Use `server/lib/chittyschema.ts` instead — it's Workers-native and speaks +// the real ChittySchema API (/api/health, /api/validate, /api/tables). /** * ChittyChronicle Client @@ -414,12 +388,9 @@ export class ChittyOSClientFactory { return this.instances.get('chittyconnect'); } - static getChittySchema(config?: Partial): ChittySchemaClient { - if (!this.instances.has('chittyschema') || config) { - this.instances.set('chittyschema', new ChittySchemaClient(config)); - } - return this.instances.get('chittyschema'); - } + // getChittySchema removed — use `server/lib/chittyschema.ts` functions + // (validateRow, listTables, checkHealth) which are Workers-native and + // speak the real ChittySchema API. static getChittyChronicle(config?: Partial): ChittyChronicleClient { if (!this.instances.has('chittychronicle') || config) { @@ -460,7 +431,6 @@ export class ChittyOSClientFactory { chittyid: this.getChittyID(), chittyauth: this.getChittyAuth(), chittyconnect: this.getChittyConnect(), - chittyschema: this.getChittySchema(), chittychronicle: this.getChittyChronicle(), chittyregistry: this.getChittyRegistry(), chittyrental: this.getChittyRental(), @@ -494,7 +464,6 @@ export class ChittyOSClientFactory { export const chittyID = () => ChittyOSClientFactory.getChittyID(); export const chittyAuth = () => ChittyOSClientFactory.getChittyAuth(); export const chittyConnect = () => ChittyOSClientFactory.getChittyConnect(); -export const chittySchema = () => ChittyOSClientFactory.getChittySchema(); export const chittyChronicle = () => ChittyOSClientFactory.getChittyChronicle(); export const chittyRegistry = () => ChittyOSClientFactory.getChittyRegistry(); export const chittyRental = () => ChittyOSClientFactory.getChittyRental(); diff --git a/server/lib/chittyschema-validation.ts b/server/lib/chittyschema-validation.ts deleted file mode 100644 index 24559fe..0000000 --- a/server/lib/chittyschema-validation.ts +++ /dev/null @@ -1,254 +0,0 @@ - -/** - * ChittySchema Integration for ChittyFinance - * Validates financial data against centralized ChittyOS schema service - */ - -import { fetchWithRetry, IntegrationError } from './error-handling'; - -const CHITTYSCHEMA_BASE_URL = process.env.CHITTYSCHEMA_URL || 'https://schema.chitty.cc'; - -export interface ValidationResult { - valid: boolean; - errors?: Array<{ - path: string; - message: string; - code: string; - }>; - warnings?: Array<{ - path: string; - message: string; - }>; -} - -export interface EntityTypeInfo { - type: string; - description: string; - schema: Record; - version: string; -} - -/** - * Validate data against ChittySchema - */ -export async function validateWithChittySchema( - entityType: string, - data: Record -): Promise { - try { - const response = await fetchWithRetry( - `${CHITTYSCHEMA_BASE_URL}/api/v1/validate`, - { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - }, - body: JSON.stringify({ - type: entityType, - data, - }), - }, - { - maxRetries: 2, - baseDelay: 500, - } - ); - - const result = await response.json() as ValidationResult; - return result; - } catch (error) { - console.error('ChittySchema validation error:', error); - throw new IntegrationError( - 'Schema validation service unavailable', - 'chittyschema', - true - ); - } -} - -/** - * Get available entity types from ChittySchema - */ -export async function getEntityTypes(): Promise { - try { - const response = await fetchWithRetry( - `${CHITTYSCHEMA_BASE_URL}/api/v1/entity-types`, - { - headers: { - 'Content-Type': 'application/json', - }, - }, - { - maxRetries: 2, - baseDelay: 500, - } - ); - - const result = await response.json() as { types?: EntityTypeInfo[] }; - return result.types || []; - } catch (error) { - console.error('ChittySchema entity types fetch error:', error); - return []; - } -} - -/** - * Get schema details for a specific entity type - */ -export async function getSchemaDetails(entityType: string): Promise { - try { - const response = await fetchWithRetry( - `${CHITTYSCHEMA_BASE_URL}/api/v1/schema/${entityType}`, - { - headers: { - 'Content-Type': 'application/json', - }, - }, - { - maxRetries: 2, - baseDelay: 500, - } - ); - - if (!response.ok) { - return null; - } - - const result = await response.json() as EntityTypeInfo; - return result; - } catch (error) { - console.error(`ChittySchema details fetch error for ${entityType}:`, error); - return null; - } -} - -/** - * Validate transaction before creation/update - */ -export async function validateTransaction(transaction: Record): Promise { - return validateWithChittySchema('transaction', { - amount: transaction.amount, - type: transaction.type, - description: transaction.description, - date: transaction.date, - category: transaction.category, - tenantId: transaction.tenantId, - accountId: transaction.accountId, - }); -} - -/** - * Validate tenant/entity before creation - */ -export async function validateTenant(tenant: Record): Promise { - return validateWithChittySchema('tenant', { - name: tenant.name, - type: tenant.type, - taxId: tenant.taxId, - parentId: tenant.parentId, - }); -} - -/** - * Validate account before creation - */ -export async function validateAccount(account: Record): Promise { - return validateWithChittySchema('account', { - name: account.name, - type: account.type, - institution: account.institution, - currency: account.currency, - tenantId: account.tenantId, - }); -} - -/** - * Validate property before creation - */ -export async function validateProperty(property: Record): Promise { - return validateWithChittySchema('property', { - name: property.name, - address: property.address, - city: property.city, - state: property.state, - zip: property.zip, - propertyType: property.propertyType, - tenantId: property.tenantId, - }); -} - -/** - * Middleware for validating requests against ChittySchema - */ -export function schemaValidationMiddleware(entityType: string) { - return async (req: any, res: any, next: any) => { - // Skip validation if disabled - if (process.env.SKIP_SCHEMA_VALIDATION === 'true') { - return next(); - } - - try { - const result = await validateWithChittySchema(entityType, req.body); - - if (!result.valid) { - return res.status(400).json({ - error: 'Schema validation failed', - code: 'SCHEMA_VALIDATION_ERROR', - errors: result.errors, - warnings: result.warnings, - }); - } - - // Attach warnings to request for logging - if (result.warnings && result.warnings.length > 0) { - req.schemaWarnings = result.warnings; - } - - next(); - } catch (error) { - // Log error but don't block request if schema service is down - console.error('Schema validation service error:', error); - - if (process.env.NODE_ENV === 'production') { - // In production, continue without validation if service is down - console.warn('Continuing without schema validation due to service unavailability'); - next(); - } else { - // In development, return error - res.status(503).json({ - error: 'Schema validation service unavailable', - code: 'SCHEMA_SERVICE_UNAVAILABLE', - }); - } - } - }; -} - -/** - * Batch validate multiple entities - */ -export async function batchValidate( - items: Array<{ type: string; data: Record }> -): Promise> { - const results = await Promise.all( - items.map(item => validateWithChittySchema(item.type, item.data)) - ); - - return results; -} - -/** - * Health check for ChittySchema service - */ -export async function checkSchemaServiceHealth(): Promise { - try { - const response = await fetch(`${CHITTYSCHEMA_BASE_URL}/health`, { - method: 'GET', - signal: AbortSignal.timeout(5000), - }); - - return response.ok; - } catch (error) { - return false; - } -} diff --git a/server/lib/chittyschema.ts b/server/lib/chittyschema.ts new file mode 100644 index 0000000..01d2e2a --- /dev/null +++ b/server/lib/chittyschema.ts @@ -0,0 +1,156 @@ +/** + * ChittySchema client for ChittyFinance. + * + * Talks to https://schema.chitty.cc using the real API contract: + * GET /api/health — service health + * POST /api/validate — { table, data } → { valid, errors, availableTables? } + * GET /api/tables — list registered tables + * + * Workers-native — all calls take `env` (or an explicit baseUrl) instead of + * reading `process.env` at module load time. Safe to import from any worker + * route or handler. + * + * Philosophy: validation is advisory. The schema service is an external + * dependency and we never want it to block financial writes. If it's down, + * unreachable, or returns a 5xx, callers receive `{ ok: true, advisory: true }` + * and the write proceeds. If the schema returns `valid: false`, callers + * decide whether to block (strict mode) or warn (default). + */ + +export interface ChittySchemaError { + path?: string; + message: string; + code?: string; +} + +export interface ChittySchemaResult { + /** True if validation passed OR the service was unreachable and we fell open. */ + ok: boolean; + /** True if the schema service was down/unreachable and we returned an advisory pass. */ + advisory: boolean; + /** Validation errors from the service (only set when ok=false). */ + errors?: ChittySchemaError[]; + /** When the requested table is not in the registry, list of known tables. */ + availableTables?: string[]; +} + +export interface ChittySchemaTable { + name: string; + database: string; + owner: string; +} + +const DEFAULT_BASE_URL = 'https://schema.chitty.cc'; +const DEFAULT_TIMEOUT_MS = 3000; + +function baseUrlFromEnv(env: { CHITTYSCHEMA_URL?: string }): string { + return env.CHITTYSCHEMA_URL || DEFAULT_BASE_URL; +} + +/** + * Validate a row of data against a ChittySchema-registered table. + * + * Always returns a result object — never throws. If the schema service is + * unreachable, returns `{ ok: true, advisory: true }` so callers can log and + * proceed. If the table is not registered, returns `{ ok: true, advisory: true }` + * with `availableTables` populated so operators can see what's registered. + */ +export async function validateRow( + env: { CHITTYSCHEMA_URL?: string }, + table: string, + data: Record, + opts?: { timeoutMs?: number }, +): Promise { + const url = `${baseUrlFromEnv(env)}/api/validate`; + const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS; + + try { + const response = await fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ table, data }), + signal: AbortSignal.timeout(timeoutMs), + }); + + // 5xx → advisory pass (service is down, don't block writes) + if (response.status >= 500) { + return { ok: true, advisory: true }; + } + + const body = (await response.json().catch(() => null)) as + | { valid?: boolean; errors?: unknown; availableTables?: string[] } + | null; + + if (!body) { + return { ok: true, advisory: true }; + } + + // Table not registered → advisory pass but surface availableTables so + // operators can see what's registered and decide whether to add theirs + if (body.availableTables && !body.valid) { + return { + ok: true, + advisory: true, + availableTables: body.availableTables, + }; + } + + if (body.valid === true) { + return { ok: true, advisory: false }; + } + + // Normalize error shapes — the API returns `errors` as string[] for missing + // table cases and as object[] for real validation failures + const errors: ChittySchemaError[] = Array.isArray(body.errors) + ? body.errors.map((e) => + typeof e === 'string' + ? { message: e } + : (e as ChittySchemaError), + ) + : [{ message: 'Unknown validation error' }]; + + return { ok: false, advisory: false, errors }; + } catch (err) { + // Network error, timeout, DNS fail → advisory pass. Financial writes + // must not be blocked by an external schema service being unreachable. + console.warn('[chittyschema] validateRow failed, falling open:', (err as Error).message); + return { ok: true, advisory: true }; + } +} + +/** + * List registered ChittySchema tables. Returns `[]` on any error. + */ +export async function listTables( + env: { CHITTYSCHEMA_URL?: string }, + opts?: { timeoutMs?: number }, +): Promise { + const url = `${baseUrlFromEnv(env)}/api/tables`; + const timeoutMs = opts?.timeoutMs ?? DEFAULT_TIMEOUT_MS; + + try { + const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) }); + if (!response.ok) return []; + const body = (await response.json()) as { tables?: ChittySchemaTable[] }; + return body.tables ?? []; + } catch { + return []; + } +} + +/** + * Quick health probe. Returns true iff GET /api/health returns 2xx within the timeout. + */ +export async function checkHealth( + env: { CHITTYSCHEMA_URL?: string }, + opts?: { timeoutMs?: number }, +): Promise { + const url = `${baseUrlFromEnv(env)}/api/health`; + const timeoutMs = opts?.timeoutMs ?? 2000; + try { + const response = await fetch(url, { signal: AbortSignal.timeout(timeoutMs) }); + return response.ok; + } catch { + return false; + } +} diff --git a/server/routes/webhooks.ts b/server/routes/webhooks.ts index e5a3a37..cda9b1b 100644 --- a/server/routes/webhooks.ts +++ b/server/routes/webhooks.ts @@ -1,8 +1,46 @@ import { Hono } from 'hono'; +import { z } from 'zod'; import type { HonoEnv } from '../env'; +import { createDb } from '../db/connection'; +import { SystemStorage } from '../storage/system'; +import { findAccountCode } from '../../database/chart-of-accounts'; +import { validateRow } from '../lib/chittyschema'; +import { ledgerLog } from '../lib/ledger-client'; export const webhookRoutes = new Hono(); +/** + * Mercury webhook transaction payload shape. + * + * ChittyConnect normalizes Mercury's native payload to this shape before + * forwarding to us, so we can trust the field names here. + * + * `tenantId` and `accountId` must be resolved by ChittyConnect before it + * POSTs to us — we don't have tenant middleware on webhook routes (they + * use service auth, not session/role auth). + */ +const mercuryTransactionSchema = z.object({ + tenantId: z.string().uuid(), + accountId: z.string().uuid(), + mercuryTransactionId: z.string(), + description: z.string(), + amount: z.number(), // signed: negative = expense + category: z.string().optional().nullable(), + postedAt: z.string(), // ISO 8601 + payee: z.string().optional().nullable(), +}); + +const mercuryWebhookEnvelopeSchema = z.object({ + id: z.string().optional(), + eventId: z.string().optional(), + type: z.string().optional(), // e.g. 'transaction.created' + data: z + .object({ + transaction: mercuryTransactionSchema.optional(), + }) + .optional(), +}); + // POST /api/webhooks/stripe — Stripe webhook endpoint webhookRoutes.post('/api/webhooks/stripe', async (c) => { const secret = c.env.STRIPE_WEBHOOK_SECRET; @@ -45,7 +83,21 @@ webhookRoutes.post('/api/webhooks/stripe', async (c) => { return c.json({ received: true }); }); -// POST /api/webhooks/mercury — Mercury webhook endpoint +// POST /api/webhooks/mercury — Mercury webhook endpoint with real-time classification +// +// Flow: +// 1. Service-auth check (Bearer token == CHITTY_AUTH_SERVICE_TOKEN) +// 2. KV-based idempotency (7-day TTL dedup window) +// 3. Parse + validate envelope with zod +// 4. If payload carries a transaction, persist it to the DB with an +// auto-suggested COA code (L0 → L1 keyword match at ingest) +// 5. Advisory validation against ChittySchema's FinancialTransactionsSchema +// — never blocks the write, only logs warnings +// +// Returns: +// 200 { received, duplicate? } — envelope-only events (no transaction) +// 201 { received, transactionId, suggestedCoaCode, schemaAdvisory? } — persisted tx +// 400 on validation failure (envelope or payload) webhookRoutes.post('/api/webhooks/mercury', async (c) => { // Service auth check const expected = c.env.CHITTY_AUTH_SERVICE_TOKEN; @@ -59,19 +111,114 @@ webhookRoutes.post('/api/webhooks/mercury', async (c) => { return c.json({ error: 'unauthorized' }, 401); } - const body = await c.req.json().catch(() => null); - const eventId = c.req.header('x-event-id') || (body && (body.id || body.eventId)); + const rawBody = await c.req.json().catch(() => null); + const envelope = mercuryWebhookEnvelopeSchema.safeParse(rawBody); + if (!envelope.success) { + return c.json({ error: 'invalid_envelope', details: envelope.error.flatten() }, 400); + } + + const eventId = c.req.header('x-event-id') || envelope.data.id || envelope.data.eventId; if (!eventId) { return c.json({ error: 'missing_event_id' }, 400); } + // KV idempotency — 7-day dedup window const kv = c.env.FINANCE_KV; - const existing = await kv.get(`webhook:mercury:${eventId}`); + const dedupKey = `webhook:mercury:${eventId}`; + const existing = await kv.get(dedupKey); if (existing) { return c.json({ received: true, duplicate: true }, 202); } + await kv.put(dedupKey, JSON.stringify(rawBody || {}), { expirationTtl: 604800 }); - await kv.put(`webhook:mercury:${eventId}`, JSON.stringify(body || {}), { expirationTtl: 604800 }); + const tx = envelope.data.data?.transaction; + if (!tx) { + // Envelope-only event (e.g. account.created, webhook.ping) — just ack + return c.json({ received: true }, 202); + } + + // Auto-classify via keyword match at ingest (L0 → L1 suggestion). + // Suspense (9010) gets low confidence, matched codes get 0.700. + const suggestedCoaCode = findAccountCode(tx.description, tx.category ?? undefined); + const isSuspense = suggestedCoaCode === '9010'; + const classificationConfidence = isSuspense ? '0.100' : '0.700'; + const externalId = `mercury:${tx.mercuryTransactionId}`; + + // Advisory ChittySchema validation — never blocks the write, only logs. + // Uses FinancialTransactionsSchema from the chittyledger database. + const schemaResult = await validateRow(c.env, 'FinancialTransactionsInsertSchema', { + tenantId: tx.tenantId, + accountId: tx.accountId, + amount: String(tx.amount), + type: tx.amount >= 0 ? 'income' : 'expense', + description: tx.description, + date: tx.postedAt, + externalId, + }); + + if (!schemaResult.ok && schemaResult.errors) { + console.warn('[webhook:mercury] ChittySchema validation failed (advisory)', { + eventId, + errors: schemaResult.errors, + }); + } + + // Persist via storage abstraction. Use a fresh DB/storage since webhook + // routes don't go through the storageMiddleware (no tenant context). + const db = createDb(c.env.DATABASE_URL); + const storage = new SystemStorage(db); + + // Dedup at the DB level too — if ChittyConnect retries before the KV + // entry was written, the externalId unique lookup catches it + const dupRow = await storage.getTransactionByExternalId(externalId, tx.tenantId); + if (dupRow) { + return c.json( + { received: true, duplicate: true, transactionId: dupRow.id }, + 202, + ); + } - return c.json({ received: true }, 202); + const created = await storage.createTransaction({ + tenantId: tx.tenantId, + accountId: tx.accountId, + amount: String(tx.amount), + type: tx.amount >= 0 ? 'income' : 'expense', + category: tx.category ?? null, + description: tx.description, + date: new Date(tx.postedAt), + payee: tx.payee ?? null, + externalId, + suggestedCoaCode, + classificationConfidence, + metadata: { source: 'mercury_webhook', mercuryTransactionId: tx.mercuryTransactionId, eventId }, + }); + + ledgerLog( + c, + { + entityType: 'audit', + action: 'webhook.mercury.transaction_ingested', + metadata: { + tenantId: tx.tenantId, + accountId: tx.accountId, + transactionId: created.id, + suggestedCoaCode, + confidence: classificationConfidence, + schemaAdvisory: schemaResult.advisory, + schemaValid: schemaResult.ok, + }, + }, + c.env, + ); + + return c.json( + { + received: true, + transactionId: created.id, + suggestedCoaCode, + classificationConfidence, + schemaAdvisory: schemaResult.advisory, + }, + 201, + ); });