diff --git a/apps/sim/app/api/files/export/[id]/route.test.ts b/apps/sim/app/api/files/export/[id]/route.test.ts index 1d83cdd02f9..7d796a624e8 100644 --- a/apps/sim/app/api/files/export/[id]/route.test.ts +++ b/apps/sim/app/api/files/export/[id]/route.test.ts @@ -51,14 +51,14 @@ function request() { return createMockRequest('GET', undefined, {}, `http://localhost:3000/api/files/export/${DOC_ID}`) } -function assetRecord(id: string, size: number) { +function assetRecord(id: string, size: number | null) { return { id, key: `workspace/ws-1/${id}`, originalName: `${id}.png`, contentType: 'image/png', context: 'workspace', - size, + sizeBytes: size, workspaceId: 'ws-1', } } @@ -69,7 +69,7 @@ const DOC_RECORD = { originalName: 'doc.md', contentType: 'text/markdown', context: 'workspace', - size: 1024, + sizeBytes: 1024, workspaceId: 'ws-1', } @@ -161,6 +161,21 @@ describe('markdown export bundling', () => { expect(zip.file('assets/bad.png')).toBeNull() }) + it('drops an asset with missing canonical size metadata', async () => { + embeds('good', 'missing-size') + assetsResolveTo((id) => assetRecord(id, id === 'missing-size' ? null : 1 * MB)) + + const response = await GET(request(), context) + + expect(response.status).toBe(200) + const zip = await JSZip.loadAsync(Buffer.from(await response.arrayBuffer())) + expect(zip.file('assets/good.png')).not.toBeNull() + expect(zip.file('assets/missing-size.png')).toBeNull() + expect( + mockDownloadFile.mock.calls.some(([options]) => options.key.endsWith('missing-size')) + ).toBe(false) + }) + /** * The two id representations have to stay distinct: metadata resolves by the stored id, while the * rewrite finds the embed by the spelling the document used. Collapsing them either drops the diff --git a/apps/sim/app/api/files/export/[id]/route.ts b/apps/sim/app/api/files/export/[id]/route.ts index 4f67b7b8353..f36104475aa 100644 --- a/apps/sim/app/api/files/export/[id]/route.ts +++ b/apps/sim/app/api/files/export/[id]/route.ts @@ -17,6 +17,7 @@ import { getServeStoragePrefix } from '@/lib/uploads/config' import { downloadFile } from '@/lib/uploads/core/storage-service' import { extractEmbeddedFileRefs } from '@/lib/uploads/server/embedded-image-refs' import { getFileMetadataById } from '@/lib/uploads/server/metadata' +import { getWorkspaceFileSize } from '@/lib/uploads/shared/types' import { storedFileId } from '@/lib/uploads/utils/embedded-image-ref' import { formatFileSize } from '@/lib/uploads/utils/file-utils' import { verifyFileAccess } from '@/app/api/files/authorization' @@ -103,7 +104,7 @@ export const GET = withRouteHandler( metadata: { fileId: record.id, fileName: record.originalName, - bytes: record.size, + bytes: getWorkspaceFileSize(record), format, assetCount, }, @@ -164,7 +165,7 @@ export const GET = withRouteHandler( const imgRecord = await getFileMetadataById(storedFileId(imageId)) if (!imgRecord) return null if (!(await verifyFileAccess(imgRecord.key, userId))) return null - return { imageId, record: imgRecord } + return { imageId, record: imgRecord, size: getWorkspaceFileSize(imgRecord) } } catch (error) { logger.warn('Failed to resolve asset for export', { imageId, @@ -177,8 +178,7 @@ export const GET = withRouteHandler( // The body counts against the same budget as its assets — the zip holds both, so a // limit that measured only the attachments would not describe the archive produced. - const bundleBytes = - mdBuffer.length + assetTargets.reduce((sum, target) => sum + target.record.size, 0) + const bundleBytes = mdBuffer.length + assetTargets.reduce((sum, target) => sum + target.size, 0) if (bundleBytes > MAX_EXPORT_TOTAL_BYTES) { return NextResponse.json( { diff --git a/apps/sim/app/api/files/public/[token]/content/route.test.ts b/apps/sim/app/api/files/public/[token]/content/route.test.ts index 251f96f83e3..54d7ce0d3ad 100644 --- a/apps/sim/app/api/files/public/[token]/content/route.test.ts +++ b/apps/sim/app/api/files/public/[token]/content/route.test.ts @@ -52,7 +52,7 @@ const passwordShare = { workspaceId: 'ws-1', originalName: 'report.pdf', contentType: 'application/pdf', - size: 4, + sizeBytes: 4, }, workspaceName: 'Acme', ownerName: 'Jane', diff --git a/apps/sim/app/api/files/public/[token]/route.test.ts b/apps/sim/app/api/files/public/[token]/route.test.ts index aa32176c87c..3d48d974589 100644 --- a/apps/sim/app/api/files/public/[token]/route.test.ts +++ b/apps/sim/app/api/files/public/[token]/route.test.ts @@ -52,7 +52,7 @@ const publicShare = { workspaceId: 'ws-secret', originalName: 'report.pdf', contentType: 'application/pdf', - size: 2048, + sizeBytes: 2048, }, workspaceName: 'Acme Workspace', ownerName: 'Jane Doe', diff --git a/apps/sim/app/api/files/public/[token]/route.ts b/apps/sim/app/api/files/public/[token]/route.ts index 5c4482b22a9..be95a9b5964 100644 --- a/apps/sim/app/api/files/public/[token]/route.ts +++ b/apps/sim/app/api/files/public/[token]/route.ts @@ -13,6 +13,7 @@ import { generateRequestId } from '@/lib/core/utils/request' import { withRouteHandler } from '@/lib/core/utils/with-route-handler' import { enforcePublicFileRateLimit } from '@/lib/public-shares/rate-limit' import { resolveActiveShareByToken } from '@/lib/public-shares/share-manager' +import { getWorkspaceFileSize } from '@/lib/uploads/shared/types' export const dynamic = 'force-dynamic' @@ -58,7 +59,7 @@ export const GET = withRouteHandler( token, name: file.originalName, type: file.contentType, - size: file.size, + size: getWorkspaceFileSize(file), workspaceName, ownerName, }) diff --git a/apps/sim/app/api/files/uploads/finalizers.ts b/apps/sim/app/api/files/uploads/finalizers.ts index a885f574a11..bc51d3a034a 100644 --- a/apps/sim/app/api/files/uploads/finalizers.ts +++ b/apps/sim/app/api/files/uploads/finalizers.ts @@ -1,7 +1,7 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import { type Principal, resolvePrincipalAuditAttribution } from '@sim/auth/principal' import { db } from '@sim/db' -import { workspaceFiles } from '@sim/db/schema' +import { type WorkspaceFileRow, workspaceFileColumns, workspaceFiles } from '@sim/db/schema' import { generateId } from '@sim/utils/id' import { eq, sql } from 'drizzle-orm' import type { V2File } from '@/lib/api/contracts/v2/files' @@ -80,7 +80,7 @@ interface FinalizedMetadataInput { size: number } -type FileMetadataRecord = typeof workspaceFiles.$inferSelect +type FileMetadataRecord = WorkspaceFileRow /** * Finalizes the domain resource represented by a verified upload object. @@ -371,7 +371,7 @@ async function insertOrLoadFileMetadata( contentUpdatedAt: now, }) .onConflictDoNothing() - .returning() + .returning(workspaceFileColumns) if (inserted) return { file: inserted, created: true } @@ -386,7 +386,7 @@ async function insertOrLoadFileMetadata( async function findFileMetadataByKey(key: string): Promise { const [file] = await db - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where(eq(workspaceFiles.key, key)) .orderBy(sql`${workspaceFiles.deletedAt} IS NULL DESC`) diff --git a/apps/sim/app/f/[token]/page.tsx b/apps/sim/app/f/[token]/page.tsx index 3d5506b6919..642b7975bb4 100644 --- a/apps/sim/app/f/[token]/page.tsx +++ b/apps/sim/app/f/[token]/page.tsx @@ -9,6 +9,7 @@ import { validateAuthToken, } from '@/lib/core/security/deployment' import { resolveActiveShareByToken } from '@/lib/public-shares/share-manager' +import { getWorkspaceFileSize } from '@/lib/uploads/shared/types' import { PublicFileAuth } from '@/app/f/[token]/public-file-auth' import { PublicFileEmailAuth } from '@/app/f/[token]/public-file-email-auth' import { PublicFileSSOAuth } from '@/app/f/[token]/public-file-sso-auth' @@ -117,7 +118,7 @@ export default async function PublicFilePage({ params }: PublicFilePageProps) { token={token} name={file.originalName} type={file.contentType} - size={file.size} + size={getWorkspaceFileSize(file)} version={file.updatedAt.getTime()} workspaceName={workspaceName} ownerName={ownerName} diff --git a/apps/sim/background/cleanup-soft-deletes.test.ts b/apps/sim/background/cleanup-soft-deletes.test.ts index 4cdcd726237..98d17859594 100644 --- a/apps/sim/background/cleanup-soft-deletes.test.ts +++ b/apps/sim/background/cleanup-soft-deletes.test.ts @@ -159,7 +159,7 @@ describe('cleanup soft deletes', () => { }, ]) mockDeleteFiles.mockResolvedValueOnce({ deleted: 2, failed: [] }) - dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'file-deleted', size: 7 }]) + dbChainMockFns.returning.mockResolvedValueOnce([{ id: 'file-deleted', sizeBytes: 7 }]) await runCleanupSoftDeletes(basePayload) diff --git a/apps/sim/background/cleanup-soft-deletes.ts b/apps/sim/background/cleanup-soft-deletes.ts index 076c69cbec5..7fda5bcdaf8 100644 --- a/apps/sim/background/cleanup-soft-deletes.ts +++ b/apps/sim/background/cleanup-soft-deletes.ts @@ -328,12 +328,12 @@ async function deleteExpiredBillableWorkspaceFileRows( ) .returning({ id: workspaceFiles.id, - size: sql`${workspaceFiles.sizeBytes}`.mapWith(Number), + sizeBytes: workspaceFiles.sizeBytes, }) - if (deletedRows.some(({ size }) => size < 0)) { - throw new Error('Cannot delete workspace files with negative stored-byte metadata') - } - const deletedBytes = deletedRows.reduce((total, { size }) => total + size, 0) + const deletedBytes = deletedRows.reduce( + (total, row) => total + getWorkspaceFileSize(row), + 0 + ) await decrementStorageUsageForBillingContextInTx(tx, billingContext, deletedBytes) return deletedRows.length }) diff --git a/apps/sim/ee/workspace-forking/lib/copy/copy-files.ts b/apps/sim/ee/workspace-forking/lib/copy/copy-files.ts index 20bb3840ee2..6485e563c3f 100644 --- a/apps/sim/ee/workspace-forking/lib/copy/copy-files.ts +++ b/apps/sim/ee/workspace-forking/lib/copy/copy-files.ts @@ -1,5 +1,5 @@ import { db } from '@sim/db' -import { workspaceFiles } from '@sim/db/schema' +import { workspaceFileColumns, workspaceFiles } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateId } from '@sim/utils/id' @@ -180,7 +180,7 @@ export async function planForkFileCopies(params: { fileKeys.length > 0 ? inArray(workspaceFiles.key, fileKeys) : undefined, ].filter((clause): clause is NonNullable => clause !== undefined) const metas = await tx - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where( and( diff --git a/apps/sim/lib/billing/storage/payer-transfer.test.ts b/apps/sim/lib/billing/storage/payer-transfer.test.ts index caf8e0dc22f..8113e236e5f 100644 --- a/apps/sim/lib/billing/storage/payer-transfer.test.ts +++ b/apps/sim/lib/billing/storage/payer-transfer.test.ts @@ -102,6 +102,7 @@ interface FakeTransferState { users?: Record workspace: FakeWorkspace workspaceFileBytes?: number + workspaceFileMissingSizeCount?: number } function createFakeTx(state: FakeTransferState) { @@ -165,6 +166,7 @@ function createFakeTx(state: FakeTransferState) { { document_bytes: state.documentBytes ?? 0, workspace_file_bytes: state.workspaceFileBytes ?? 0, + workspace_file_missing_size_count: state.workspaceFileMissingSizeCount ?? 0, }, ] }) @@ -186,6 +188,7 @@ function updateFor( interface FakeBatchTransferState { exactBytes: Record + missingSizeCounts?: Record organizations?: Record users?: Record workspaces: FakeWorkspace[] @@ -214,6 +217,7 @@ function createFakeBatchTx(state: FakeBatchTransferState) { workspace_id: workspaceId, document_bytes: 0, workspace_file_bytes: bytes, + workspace_file_missing_size_count: state.missingSizeCounts?.[workspaceId] ?? 0, })) ) @@ -531,7 +535,31 @@ describe('changeWorkspaceStoragePayerInTx', () => { expect(query.values).not.toContain('workspaceFiles.deletedAt') expect(query.values).toContain('document.connectorId') expect(query.values).toContain('document.deletedAt') - expect(query.values.filter((value) => value === 'workspace-1')).toHaveLength(2) + expect(query.values.filter((value) => value === 'workspace-1')).toHaveLength(3) + }) + + it('fails closed when a billable file is missing canonical size metadata', async () => { + const fake = createFakeTx({ + workspace: { + id: 'workspace-1', + billedAccountUserId: 'user-1', + organizationId: null, + storageUsedBytes: 10, + }, + workspaceFileBytes: 10, + workspaceFileMissingSizeCount: 1, + users: { 'user-1': 10, 'user-2': 0 }, + }) + + await expect( + changeWorkspaceStoragePayerInTx(fake.tx, { + workspaceId: 'workspace-1', + organizationId: null, + billedAccountUserId: 'user-2', + }) + ).rejects.toThrow('Workspace workspace-1 has files missing canonical size_bytes metadata') + + expect(fake.updates).toEqual([]) }) }) @@ -684,6 +712,34 @@ describe('changeWorkspaceStoragePayersInTx', () => { expect(fake.updates).toEqual([]) expect(fake.locks).toEqual([{ ids: ['workspace-a'], table: 'workspace' }]) }) + + it('fails the batch before payer writes when canonical size metadata is missing', async () => { + const fake = createFakeBatchTx({ + exactBytes: { 'workspace-a': 10 }, + missingSizeCounts: { 'workspace-a': 1 }, + users: { current: 10, destination: 0 }, + workspaces: [ + { + id: 'workspace-a', + billedAccountUserId: 'current', + organizationId: null, + storageUsedBytes: 10, + }, + ], + }) + + await expect( + changeWorkspaceStoragePayersInTx(fake.tx, [ + { + workspaceId: 'workspace-a', + organizationId: null, + billedAccountUserId: 'destination', + }, + ]) + ).rejects.toThrow('Workspace workspace-a has files missing canonical size_bytes metadata') + + expect(fake.updates).toEqual([]) + }) }) describe('changeOrganizationWorkspaceBilledAccountsInTx', () => { diff --git a/apps/sim/lib/billing/storage/payer-transfer.ts b/apps/sim/lib/billing/storage/payer-transfer.ts index d641a960a8e..14687a4df56 100644 --- a/apps/sim/lib/billing/storage/payer-transfer.ts +++ b/apps/sim/lib/billing/storage/payer-transfer.ts @@ -17,6 +17,7 @@ interface ExactWorkspaceStorageRow { [key: string]: unknown document_bytes: number | string workspace_file_bytes: number | string + workspace_file_missing_size_count: number | string } interface BatchExactWorkspaceStorageRow extends ExactWorkspaceStorageRow { @@ -87,6 +88,13 @@ async function getExactWorkspaceStorageBytes(tx: DbOrTx, workspaceId: string): P WHERE ${workspaceFiles.workspaceId} = ${workspaceId} AND ${workspaceFiles.context} = 'workspace' ), 0)::bigint AS workspace_file_bytes, + ( + SELECT COUNT(*) + FROM ${workspaceFiles} + WHERE ${workspaceFiles.workspaceId} = ${workspaceId} + AND ${workspaceFiles.context} = 'workspace' + AND ${workspaceFiles.sizeBytes} IS NULL + )::bigint AS workspace_file_missing_size_count, COALESCE(( SELECT SUM(${document.fileSize}::bigint) FROM ${document} @@ -101,6 +109,9 @@ async function getExactWorkspaceStorageBytes(tx: DbOrTx, workspaceId: string): P if (!row) { throw new Error(`Could not recompute storage for workspace ${workspaceId}`) } + if (parseExactBytes(row.workspace_file_missing_size_count, 'missing workspace file size') > 0) { + throw new Error(`Workspace ${workspaceId} has files missing canonical size_bytes metadata`) + } const workspaceFileBytes = parseExactBytes(row.workspace_file_bytes, 'workspace file') const documentBytes = parseExactBytes(row.document_bytes, 'knowledge document') @@ -167,12 +178,16 @@ async function getExactWorkspaceStorageBytesBatch( COALESCE(SUM(storage_by_workspace.workspace_file_bytes), 0)::bigint AS workspace_file_bytes, COALESCE(SUM(storage_by_workspace.document_bytes), 0)::bigint - AS document_bytes + AS document_bytes, + COALESCE(SUM(storage_by_workspace.workspace_file_missing_size_count), 0)::bigint + AS workspace_file_missing_size_count FROM ( SELECT ${workspaceFiles.workspaceId} AS workspace_id, SUM(${workspaceFiles.sizeBytes}) AS workspace_file_bytes, - 0::bigint AS document_bytes + 0::bigint AS document_bytes, + COUNT(*) FILTER (WHERE ${workspaceFiles.sizeBytes} IS NULL)::bigint + AS workspace_file_missing_size_count FROM ${workspaceFiles} WHERE ${inArray(workspaceFiles.workspaceId, workspaceIds)} AND ${workspaceFiles.context} = 'workspace' @@ -183,7 +198,8 @@ async function getExactWorkspaceStorageBytesBatch( SELECT ${knowledgeBase.workspaceId} AS workspace_id, 0::bigint AS workspace_file_bytes, - SUM(${document.fileSize}::bigint) AS document_bytes + SUM(${document.fileSize}::bigint) AS document_bytes, + 0::bigint AS workspace_file_missing_size_count FROM ${document} INNER JOIN ${knowledgeBase} ON ${knowledgeBase.id} = ${document.knowledgeBaseId} @@ -197,6 +213,11 @@ async function getExactWorkspaceStorageBytesBatch( `) for (const row of rows) { + if (parseExactBytes(row.workspace_file_missing_size_count, 'missing workspace file size') > 0) { + throw new Error( + `Workspace ${row.workspace_id} has files missing canonical size_bytes metadata` + ) + } const workspaceFileBytes = parseExactBytes(row.workspace_file_bytes, 'workspace file') const documentBytes = parseExactBytes(row.document_bytes, 'knowledge document') const total = workspaceFileBytes + documentBytes diff --git a/apps/sim/lib/copilot/chat/fork-chat-files.ts b/apps/sim/lib/copilot/chat/fork-chat-files.ts index b0426b6c958..719b77d867e 100644 --- a/apps/sim/lib/copilot/chat/fork-chat-files.ts +++ b/apps/sim/lib/copilot/chat/fork-chat-files.ts @@ -1,8 +1,7 @@ -import { workspaceFiles } from '@sim/db/schema' +import { type WorkspaceFileRow, workspaceFileColumns, workspaceFiles } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage } from '@sim/utils/errors' import { generateShortId } from '@sim/utils/id' -import { omit } from '@sim/utils/object' import { and, eq, isNull } from 'drizzle-orm' import { mapWithConcurrency } from '@/lib/core/utils/concurrency' import type { DbOrTx, DbTransaction } from '@/lib/db/types' @@ -29,7 +28,7 @@ export const FORKABLE_CHAT_FILE_CONTEXT: StorageContext = 'mothership' /** Max concurrent blob byte-copies during a chat fork. */ const CHAT_BLOB_COPY_CONCURRENCY = 4 -export type ForkableChatFileRow = typeof workspaceFiles.$inferSelect +export type ForkableChatFileRow = WorkspaceFileRow /** One blob byte-copy to run after the fork transaction commits. */ export interface ChatBlobCopyTask { @@ -62,7 +61,7 @@ export async function listForkableChatFiles( chatId: string ): Promise { return db - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where( and( @@ -121,7 +120,7 @@ export async function planChatFileCopies(params: { const copyId = `wf_${generateShortId()}` const targetKey = generateWorkspaceFileKey(row.workspaceId, row.originalName) copyRows.push({ - ...omit(row, ['size']), + ...row, id: copyId, key: targetKey, chatId: newChatId, diff --git a/apps/sim/lib/copilot/tools/handlers/materialize-file.test.ts b/apps/sim/lib/copilot/tools/handlers/materialize-file.test.ts index 4bf8d1cff37..0d4d6d2f31b 100644 --- a/apps/sim/lib/copilot/tools/handlers/materialize-file.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/materialize-file.test.ts @@ -169,7 +169,7 @@ const mothershipRow = { originalName: 'upload.txt', displayName: 'report.txt', contentType: 'text/plain', - size: 100, + sizeBytes: 100, deletedAt: null, uploadedAt: new Date('2026-01-01'), updatedAt: new Date('2026-01-01'), @@ -629,7 +629,7 @@ describe('executeMaterializeFile - extract operation', () => { originalName: 'bundle.zip', displayName: 'bundle.zip', contentType: 'application/zip', - size: 2048, + sizeBytes: 2048, deletedAt: null, uploadedAt: new Date(), updatedAt: new Date(), @@ -783,7 +783,7 @@ describe('executeMaterializeFile - save operation on archives', () => { originalName: 'bundle.zip', displayName: 'bundle.zip', contentType: 'application/zip', - size: 2048, + sizeBytes: 2048, deletedAt: null, uploadedAt: new Date(), updatedAt: new Date(), diff --git a/apps/sim/lib/copilot/tools/handlers/materialize-file.ts b/apps/sim/lib/copilot/tools/handlers/materialize-file.ts index 3b13d8834ff..755c8b718d8 100644 --- a/apps/sim/lib/copilot/tools/handlers/materialize-file.ts +++ b/apps/sim/lib/copilot/tools/handlers/materialize-file.ts @@ -1,7 +1,12 @@ import { AuditAction, AuditResourceType, recordAudit } from '@sim/audit' import type { Principal } from '@sim/auth/principal' import { db } from '@sim/db' -import { folder as folderTable, workflow, workspaceFiles } from '@sim/db/schema' +import { + folder as folderTable, + type WorkspaceFileRow, + workflow, + workspaceFiles, +} from '@sim/db/schema' import { createLogger } from '@sim/logger' import { getErrorMessage, @@ -50,7 +55,7 @@ const logger = createLogger('SaveUpload') const MAX_MATERIALIZE_NAME_RETRIES = 8 const WORKSPACE_FILE_NAME_UNIQUE_INDEX = 'workspace_files_workspace_folder_name_active_unique' -function toFileRecord(row: typeof workspaceFiles.$inferSelect) { +function toFileRecord(row: WorkspaceFileRow) { const pathPrefix = getServePathPrefix() return { id: row.id, @@ -58,7 +63,7 @@ function toFileRecord(row: typeof workspaceFiles.$inferSelect) { name: row.displayName ?? row.originalName, key: row.key, path: `${pathPrefix}${encodeURIComponent(row.key)}?context=mothership`, - size: row.size, + size: getWorkspaceFileSize(row), type: row.contentType, uploadedBy: row.userId, deletedAt: row.deletedAt, diff --git a/apps/sim/lib/copilot/tools/handlers/upload-file-reader.test.ts b/apps/sim/lib/copilot/tools/handlers/upload-file-reader.test.ts index 633a4bacdd5..bb2deb7b37c 100644 --- a/apps/sim/lib/copilot/tools/handlers/upload-file-reader.test.ts +++ b/apps/sim/lib/copilot/tools/handlers/upload-file-reader.test.ts @@ -45,7 +45,7 @@ function makeRow(overrides: Partial> = {}) { originalName: 'image.png', displayName: 'image.png', contentType: 'image/png', - size: 1024, + sizeBytes: 1024, deletedAt: null, uploadedAt: NOW, updatedAt: NOW, @@ -194,7 +194,7 @@ describe('readChatUpload', () => { id: 'wf_z', displayName: 'huge.zip', contentType: 'application/zip', - size: 50 * 1024 * 1024, + sizeBytes: 50 * 1024 * 1024, }) mockOrderByThenLimit([row]) diff --git a/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts b/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts index 064b40adb16..b6546bc54d6 100644 --- a/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts +++ b/apps/sim/lib/copilot/tools/handlers/upload-file-reader.ts @@ -1,5 +1,5 @@ import { db } from '@sim/db' -import { workspaceFiles } from '@sim/db/schema' +import { type WorkspaceFileRow, workspaceFileColumns, workspaceFiles } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { toError } from '@sim/utils/errors' import { and, asc, desc, eq, isNull, or } from 'drizzle-orm' @@ -24,6 +24,7 @@ import { type WorkspaceFileRecord, } from '@/lib/uploads/contexts/workspace/workspace-file-manager' import type { WorkspaceFileSecretProvenanceEnvelope } from '@/lib/uploads/contexts/workspace/workspace-file-secret-provenance' +import { getWorkspaceFileSize } from '@/lib/uploads/shared/types' import { buildArchiveExtractGuidance, isArchiveFileName } from '@/lib/uploads/utils/file-utils' const logger = createLogger('UploadFileReader') @@ -76,11 +77,11 @@ function canonicalUploadKey(name: string): string { } /** VFS-visible name. Coalesces to originalName for legacy rows that predate displayName. */ -function vfsName(row: typeof workspaceFiles.$inferSelect): string { +function vfsName(row: WorkspaceFileRow): string { return row.displayName ?? row.originalName } -function toWorkspaceFileRecord(row: typeof workspaceFiles.$inferSelect): WorkspaceFileRecord { +function toWorkspaceFileRecord(row: WorkspaceFileRow): WorkspaceFileRecord { const pathPrefix = getServePathPrefix() return { id: row.id, @@ -88,7 +89,7 @@ function toWorkspaceFileRecord(row: typeof workspaceFiles.$inferSelect): Workspa name: vfsName(row), key: row.key, path: `${pathPrefix}${encodeURIComponent(row.key)}?context=mothership`, - size: row.size, + size: getWorkspaceFileSize(row), type: row.contentType, uploadedBy: row.userId, deletedAt: row.deletedAt, @@ -111,9 +112,9 @@ function toWorkspaceFileRecord(row: typeof workspaceFiles.$inferSelect): Workspa export async function findMothershipUploadRowByChatAndName( chatId: string, fileName: string -): Promise { +): Promise { const exactRows = await db - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where( and( @@ -134,7 +135,7 @@ export async function findMothershipUploadRowByChatAndName( } const allRows = await db - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where( and( @@ -155,7 +156,7 @@ export async function findMothershipUploadRowByChatAndName( export async function listChatUploads(chatId: string): Promise { try { const rows = await db - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where( and( diff --git a/apps/sim/lib/knowledge/documents/service.ts b/apps/sim/lib/knowledge/documents/service.ts index a8c22018b8a..05b0f5af21a 100644 --- a/apps/sim/lib/knowledge/documents/service.ts +++ b/apps/sim/lib/knowledge/documents/service.ts @@ -142,6 +142,7 @@ import { type FileMetadataRecord, getFileMetadataByKeys, } from '@/lib/uploads/server/metadata' +import { getWorkspaceFileSize } from '@/lib/uploads/shared/types' import { extractStorageKey } from '@/lib/uploads/utils/file-utils' import type { processDocument as processDocumentTask } from '@/background/knowledge-processing' import { calculateCost } from '@/providers/utils' @@ -1990,7 +1991,8 @@ function getServerKnownDocumentSize( bindingByKey: ReadonlyMap ): number { const storageKey = getKnowledgeBaseStorageKey(fileUrl) - return storageKey ? (bindingByKey.get(storageKey)?.size ?? fallbackSize) : fallbackSize + const binding = storageKey ? bindingByKey.get(storageKey) : undefined + return binding ? getWorkspaceFileSize(binding) : fallbackSize } /** diff --git a/apps/sim/lib/knowledge/documents/storage-billing.test.ts b/apps/sim/lib/knowledge/documents/storage-billing.test.ts index 0fd36063203..5c964eb2aac 100644 --- a/apps/sim/lib/knowledge/documents/storage-billing.test.ts +++ b/apps/sim/lib/knowledge/documents/storage-billing.test.ts @@ -292,7 +292,7 @@ describe('knowledge document storage attribution', () => { key: storageKey, workspaceId: 'workspace-1', userId: 'external-collaborator', - size: 8, + sizeBytes: 8, }, ]) mockIncrementStorageUsageForBillingContextInTx.mockResolvedValue(13) diff --git a/apps/sim/lib/public-shares/share-manager.ts b/apps/sim/lib/public-shares/share-manager.ts index 7508ffc510b..981d6a4b13e 100644 --- a/apps/sim/lib/public-shares/share-manager.ts +++ b/apps/sim/lib/public-shares/share-manager.ts @@ -1,5 +1,12 @@ import { db } from '@sim/db' -import { publicShare, user, workspace, workspaceFiles } from '@sim/db/schema' +import { + publicShare, + user, + type WorkspaceFileRow, + workspace, + workspaceFileColumns, + workspaceFiles, +} from '@sim/db/schema' import { createLogger } from '@sim/logger' import { generateId, generateShortId } from '@sim/utils/id' import { and, eq, inArray, isNull } from 'drizzle-orm' @@ -228,7 +235,7 @@ export async function upsertFileShare({ */ export interface ResolvedShare { share: PublicShareRow - file: typeof workspaceFiles.$inferSelect + file: WorkspaceFileRow /** Owning workspace name, for provenance on the public page. */ workspaceName: string | null /** Display name of the file's uploader. */ @@ -239,7 +246,7 @@ export async function resolveActiveShareByToken(token: string): Promise { +): Promise { const [inserted] = await tx .insert(workspaceFiles) .values({ @@ -262,7 +268,7 @@ async function insertWorkspaceFileMetadataInTx( contentUpdatedAt: new Date(), }) .onConflictDoNothing() - .returning() + .returning(workspaceFileColumns) return inserted } @@ -278,9 +284,9 @@ class WorkspaceFileRegistrationConflictError extends Error { async function findWorkspaceFileByRegistrationKey( executor: DbOrTx, key: string -): Promise { +): Promise { const files = await executor - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where(eq(workspaceFiles.key, key)) .orderBy(sql`${workspaceFiles.deletedAt} IS NULL DESC`) @@ -295,9 +301,9 @@ async function findWorkspaceFileForLifecycle( executor: DbOrTx, workspaceId: string, fileId: string -): Promise { +): Promise { const [file] = await executor - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where( and( @@ -316,7 +322,7 @@ async function findWorkspaceFileForLifecycle( * ownership and object attributes prevent unrelated callers from reusing it. */ function isSameWorkspaceFileRegistration( - file: typeof workspaceFiles.$inferSelect, + file: WorkspaceFileRow, params: { workspaceId: string userId: string @@ -464,7 +470,7 @@ export async function uploadWorkspaceFile( logger.info(`Upload returned key: ${uploadResult.key}`) let finalized: { - inserted: typeof workspaceFiles.$inferSelect + inserted: WorkspaceFileRow updatedUsage: number | undefined } try { @@ -839,7 +845,7 @@ async function markUploadSessionFileRegistered( if (!marked) throw new Error('Workspace upload registration marker could not be persisted') } -function assertActiveWorkspaceFileRegistration(file: typeof workspaceFiles.$inferSelect): void { +function assertActiveWorkspaceFileRegistration(file: WorkspaceFileRow): void { if (file.deletedAt) { throw new OrchestrationError('conflict', 'Upload result was deleted') } @@ -1124,7 +1130,7 @@ function mapWorkspaceFileRecord( } function mapUploadedWorkspaceFileRecord( - file: typeof workspaceFiles.$inferSelect, + file: WorkspaceFileRow, workspaceId: string, folderPath: string | null ): UploadedWorkspaceFileRecord { @@ -1144,7 +1150,7 @@ function mapUploadedWorkspaceFileRecord( } async function mapSingleWorkspaceFileRecord( - file: typeof workspaceFiles.$inferSelect, + file: WorkspaceFileRow, workspaceId: string ): Promise { if (!file.folderId) { @@ -1205,7 +1211,7 @@ export async function getWorkspaceFileByName( ): Promise { const folderId = options?.folderId ?? null const files = await db - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where( and( @@ -1257,10 +1263,7 @@ const workspaceFileListColumns = { } as const /** A row carrying exactly the columns {@link mapWorkspaceFileRecord} needs; a full row satisfies it. */ -type WorkspaceFileListRow = Pick< - typeof workspaceFiles.$inferSelect, - keyof typeof workspaceFileListColumns -> +type WorkspaceFileListRow = Pick /** Resolves `folderPath` for a page of rows, reading the folder tree only if any row needs it. */ async function hydrateWorkspaceFilePaths( @@ -1607,7 +1610,7 @@ export async function getWorkspaceFile( try { const { includeDeleted = false } = options ?? {} const files = await db - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where( includeDeleted @@ -1779,7 +1782,7 @@ export async function updateWorkspaceFileContent( }) let finalized: { - file: typeof workspaceFiles.$inferSelect + file: WorkspaceFileRow oldKey: string sizeDiff: number updatedUsage: number | undefined @@ -1787,7 +1790,7 @@ export async function updateWorkspaceFileContent( try { finalized = await db.transaction(async (tx) => { const [currentFile] = await tx - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where( and( @@ -1852,7 +1855,7 @@ export async function updateWorkspaceFileContent( isNull(workspaceFiles.deletedAt) ) ) - .returning() + .returning(workspaceFileColumns) if (!updatedFile) { throw new OrchestrationError('not_found', 'File not found or could not be updated') } @@ -2129,7 +2132,7 @@ export async function deleteWorkspaceFile(workspaceId: string, fileId: string): isNull(workspaceFiles.deletedAt) ) ) - .returning() + .returning(workspaceFileColumns) if (!archived) return logger.info(`Successfully archived workspace file: ${archived.originalName}`) @@ -2278,7 +2281,7 @@ export async function restoreWorkspaceFile(workspaceId: string, fileId: string): isNotNull(workspaceFiles.deletedAt) ) ) - .returning() + .returning(workspaceFileColumns) if (!restored) return logger.info(`Successfully restored workspace file: ${newName}`) diff --git a/apps/sim/lib/uploads/server/metadata.ts b/apps/sim/lib/uploads/server/metadata.ts index 0425a241a73..43fcbe5435b 100644 --- a/apps/sim/lib/uploads/server/metadata.ts +++ b/apps/sim/lib/uploads/server/metadata.ts @@ -1,5 +1,5 @@ import { db } from '@sim/db' -import { workspaceFiles } from '@sim/db/schema' +import { type WorkspaceFileRow, workspaceFileColumns, workspaceFiles } from '@sim/db/schema' import { createLogger } from '@sim/logger' import { generateId } from '@sim/utils/id' import { and, eq, inArray, isNotNull, isNull, sql } from 'drizzle-orm' @@ -13,7 +13,7 @@ import { const logger = createLogger('FileMetadata') -export type FileMetadataRecord = typeof workspaceFiles.$inferSelect +export type FileMetadataRecord = WorkspaceFileRow export interface FileMetadataInsertOptions { key: string @@ -76,7 +76,7 @@ async function findActiveFileMetadataByKey( key: string ): Promise { const [record] = await executor - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where(and(eq(workspaceFiles.key, key), isNull(workspaceFiles.deletedAt))) .limit(1) @@ -107,7 +107,7 @@ async function insertFileMetadataWithExecutor( } const [existingDeleted] = await executor - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where(and(eq(workspaceFiles.key, key), isNotNull(workspaceFiles.deletedAt))) .limit(1) @@ -129,7 +129,7 @@ async function insertFileMetadataWithExecutor( contentUpdatedAt: sql`GREATEST(CURRENT_TIMESTAMP, ${workspaceFiles.contentUpdatedAt} + INTERVAL '1 millisecond')`, }) .where(eq(workspaceFiles.id, existingDeleted.id)) - .returning() + .returning(workspaceFileColumns) if (restored) { return restored @@ -155,7 +155,7 @@ async function insertFileMetadataWithExecutor( deletedAt: null, uploadedAt: new Date(), }) - .returning() + .returning(workspaceFileColumns) if (!inserted) { throw new Error(`Failed to insert file metadata for key: ${key}`) @@ -200,7 +200,7 @@ async function insertImmutableFileMetadataWithExecutor( uploadedAt: new Date(), }) .onConflictDoNothing() - .returning() + .returning(workspaceFileColumns) if (inserted) return inserted @@ -278,13 +278,13 @@ export async function insertFileMetadataMany( })) ) .onConflictDoNothing() - .returning() + .returning(workspaceFileColumns) const insertedKeys = new Set(inserted.map((record) => record.key)) const conflictingRows = uniqueRows.filter((row) => !insertedKeys.has(row.key)) if (conflictingRows.length > 0) { const activeRows = await db - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where( and( @@ -326,7 +326,7 @@ export async function getFileMetadataByKey( } const [record] = await db - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where(conditions.length > 1 ? and(...conditions) : conditions[0]) // Prefer the active row when includeDeleted lets both an active and a @@ -378,7 +378,7 @@ export async function getFileMetadataByKeys( return [] } return executor - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where( and( @@ -400,7 +400,7 @@ export async function getFileMetadataById( const conditions = [eq(workspaceFiles.id, id)] if (!includeDeleted) conditions.push(isNull(workspaceFiles.deletedAt)) const [record] = await db - .select() + .select(workspaceFileColumns) .from(workspaceFiles) .where(conditions.length > 1 ? and(...conditions) : conditions[0]) .limit(1) diff --git a/packages/db/schema.ts b/packages/db/schema.ts index 696985bd776..1310c72ee5e 100644 --- a/packages/db/schema.ts +++ b/packages/db/schema.ts @@ -1,4 +1,5 @@ -import { type SQL, sql } from 'drizzle-orm' +import { omit } from '@sim/utils/object' +import { getTableColumns, type SQL, sql } from 'drizzle-orm' import { type AnyPgColumn, bigint, @@ -2105,7 +2106,7 @@ export const workspaceFiles = pgTable( */ displayName: text('display_name'), contentType: text('content_type').notNull(), - /** contract-pending(after #7112 is fully deployed): drop size, workspace_files_sync_size_columns, and the temporary dev cutover runner */ + /** contract-pending(after the cutover is fully deployed and size_bytes has no NULLs): drop size, workspace_files_sync_size_columns, and the temporary dev cutover runner — all application reads and writes use size_bytes */ size: integer('size').notNull().default(0), /** Exact byte size. The deploy migration backfills existing rows before this release serves traffic. */ sizeBytes: bigint('size_bytes', { mode: 'number' }), @@ -2173,6 +2174,10 @@ export const workspaceFiles = pgTable( }) ) +/** Canonical application projection; the legacy `size` bridge is migration-only. */ +export const workspaceFileColumns = omit(getTableColumns(workspaceFiles), ['size']) +export type WorkspaceFileRow = Omit + export const uploadSessionStatusEnum = pgEnum('upload_session_status', [ 'uploading', 'completing', diff --git a/packages/db/script-migrations/0003_backfill_workspace_storage_usage.ts b/packages/db/script-migrations/0003_backfill_workspace_storage_usage.ts index 5cd204df684..e26a7063768 100644 --- a/packages/db/script-migrations/0003_backfill_workspace_storage_usage.ts +++ b/packages/db/script-migrations/0003_backfill_workspace_storage_usage.ts @@ -1,4 +1,8 @@ import type { Sql } from 'postgres' +import { + backfillWorkspaceFileSizeBytes, + createPostgresWorkspaceFileSizeBytesBackfillStore, +} from './0008_backfill_workspace_file_size_bytes' import type { ScriptMigration } from './types' export const WORKSPACE_STORAGE_RECONCILIATION_BATCH_SIZE = 250 @@ -125,7 +129,7 @@ export function createPostgresStorageReconciliationStore(sql: Sql): StorageRecon const [invalid] = await tx>` SELECT count(*) AS invalid_count FROM ( - SELECT size::bigint AS bytes + SELECT size_bytes AS bytes FROM workspace_files WHERE workspace_id = ANY(${workspaceIds}::text[]) AND context = 'workspace' @@ -137,15 +141,15 @@ export function createPostgresStorageReconciliationStore(sql: Sql): StorageRecon AND d.connector_id IS NULL AND d.deleted_at IS NULL ) source - WHERE bytes < 0 + WHERE bytes IS NULL OR bytes < 0 ` if (Number(invalid?.invalid_count ?? 0) > 0) { - throw new Error('Cannot reconcile workspace storage: negative source metadata size') + throw new Error('Cannot reconcile workspace storage: invalid canonical size metadata') } await tx` WITH file_totals AS ( - SELECT workspace_id, sum(size)::bigint AS bytes + SELECT workspace_id, sum(size_bytes)::bigint AS bytes FROM workspace_files WHERE workspace_id = ANY(${workspaceIds}::text[]) AND context = 'workspace' @@ -247,6 +251,7 @@ export function createPostgresStorageReconciliationStore(sql: Sql): StorageRecon export const backfillWorkspaceStorageUsage: ScriptMigration = { name: '0003_backfill_workspace_storage_usage', async up(sql) { + await backfillWorkspaceFileSizeBytes(createPostgresWorkspaceFileSizeBytesBackfillStore(sql)) /** * Expand phase: seed only the additive workspace shadow ledger. Payer * aggregates remain under the old application's ownership until all old diff --git a/packages/db/script-migrations/0008_backfill_workspace_file_size_bytes.test.ts b/packages/db/script-migrations/0008_backfill_workspace_file_size_bytes.test.ts index f45de606342..262f66be383 100644 --- a/packages/db/script-migrations/0008_backfill_workspace_file_size_bytes.test.ts +++ b/packages/db/script-migrations/0008_backfill_workspace_file_size_bytes.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it, vi } from 'vitest' import { backfillWorkspaceFileSizeBytes, + WORKSPACE_FILE_SIZE_BYTES_BATCH_SIZE, type WorkspaceFileSizeBytesBackfillStore, } from './0008_backfill_workspace_file_size_bytes' @@ -35,4 +36,24 @@ describe('backfillWorkspaceFileSizeBytes', () => { await expect(backfillWorkspaceFileSizeBytes(store)).rejects.toThrow('non-advancing page') }) + + it('treats database-ordered text cursors as opaque', async () => { + const listCandidateIds = vi + .fn() + .mockResolvedValueOnce(['lowercase-z']) + .mockResolvedValueOnce(['UPPERCASE-A']) + .mockResolvedValueOnce([]) + const backfillCandidateIds = vi + .fn() + .mockResolvedValue(1) + + await expect( + backfillWorkspaceFileSizeBytes({ listCandidateIds, backfillCandidateIds }) + ).resolves.toBe(2) + expect(listCandidateIds.mock.calls).toEqual([ + ['', WORKSPACE_FILE_SIZE_BYTES_BATCH_SIZE], + ['lowercase-z', WORKSPACE_FILE_SIZE_BYTES_BATCH_SIZE], + ['UPPERCASE-A', WORKSPACE_FILE_SIZE_BYTES_BATCH_SIZE], + ]) + }) }) diff --git a/packages/db/script-migrations/0008_backfill_workspace_file_size_bytes.ts b/packages/db/script-migrations/0008_backfill_workspace_file_size_bytes.ts index 2ab645df197..88e69b2e4bc 100644 --- a/packages/db/script-migrations/0008_backfill_workspace_file_size_bytes.ts +++ b/packages/db/script-migrations/0008_backfill_workspace_file_size_bytes.ts @@ -7,6 +7,7 @@ const logger = createLogger('WorkspaceFileSizeBytesBackfill') export const WORKSPACE_FILE_SIZE_BYTES_BATCH_SIZE = 1000 export interface WorkspaceFileSizeBytesBackfillStore { + /** Treats `afterId` as an opaque cursor ordered by the backing database's collation. */ listCandidateIds(afterId: string, limit: number): Promise backfillCandidateIds(ids: readonly string[]): Promise } @@ -34,7 +35,7 @@ export async function backfillWorkspaceFileSizeBytes( throw new Error('Workspace file size_bytes backfill store returned an oversized page') } const lastId = ids.at(-1) - if (!lastId || lastId <= afterId) { + if (!lastId || lastId === afterId) { throw new Error('Workspace file size_bytes backfill store returned a non-advancing page') } backfilled += await store.backfillCandidateIds(ids) diff --git a/packages/db/workspace-files-schema.test.ts b/packages/db/workspace-files-schema.test.ts new file mode 100644 index 00000000000..66436d675db --- /dev/null +++ b/packages/db/workspace-files-schema.test.ts @@ -0,0 +1,9 @@ +import { describe, expect, it } from 'vitest' +import { workspaceFileColumns } from './schema' + +describe('workspaceFileColumns', () => { + it('excludes the legacy size bridge from application projections', () => { + expect(workspaceFileColumns).toHaveProperty('sizeBytes') + expect(workspaceFileColumns).not.toHaveProperty('size') + }) +}) diff --git a/packages/testing/src/mocks/schema.mock.ts b/packages/testing/src/mocks/schema.mock.ts index c3160db69b9..5ebb19bf2bc 100644 --- a/packages/testing/src/mocks/schema.mock.ts +++ b/packages/testing/src/mocks/schema.mock.ts @@ -15,6 +15,28 @@ * predicate's operands are observable at all. */ +const workspaceFilesMock = { + id: 'workspaceFiles.id', + key: 'workspaceFiles.key', + userId: 'workspaceFiles.userId', + workspaceId: 'workspaceFiles.workspaceId', + folderId: 'workspaceFiles.folderId', + context: 'workspaceFiles.context', + chatId: 'workspaceFiles.chatId', + messageId: 'workspaceFiles.messageId', + originalName: 'workspaceFiles.originalName', + displayName: 'workspaceFiles.displayName', + contentType: 'workspaceFiles.contentType', + sizeBytes: 'workspaceFiles.sizeBytes', + width: 'workspaceFiles.width', + height: 'workspaceFiles.height', + deletedAt: 'workspaceFiles.deletedAt', + uploadedAt: 'workspaceFiles.uploadedAt', + updatedAt: 'workspaceFiles.updatedAt', + contentUpdatedAt: 'workspaceFiles.contentUpdatedAt', + secretProvenanceVersion: 'workspaceFiles.secretProvenanceVersion', +} + export const schemaMock = { user: { id: 'user.id', @@ -640,22 +662,8 @@ export const schemaMock = { deletedAt: 'workspaceFile.deletedAt', uploadedAt: 'workspaceFile.uploadedAt', }, - workspaceFiles: { - id: 'workspaceFiles.id', - key: 'workspaceFiles.key', - userId: 'workspaceFiles.userId', - workspaceId: 'workspaceFiles.workspaceId', - context: 'workspaceFiles.context', - chatId: 'workspaceFiles.chatId', - originalName: 'workspaceFiles.originalName', - contentType: 'workspaceFiles.contentType', - sizeBytes: 'workspaceFiles.sizeBytes', - deletedAt: 'workspaceFiles.deletedAt', - uploadedAt: 'workspaceFiles.uploadedAt', - updatedAt: 'workspaceFiles.updatedAt', - contentUpdatedAt: 'workspaceFiles.contentUpdatedAt', - secretProvenanceVersion: 'workspaceFiles.secretProvenanceVersion', - }, + workspaceFiles: workspaceFilesMock, + workspaceFileColumns: workspaceFilesMock, workspaceFileSecretProvenance: { fileId: 'workspaceFileSecretProvenance.fileId', contentUpdatedAt: 'workspaceFileSecretProvenance.contentUpdatedAt',