Finish extensionless pages and document staging

Author: Sg312

apps/sim/app/api/files/utils.ts

@@ -239,9 +239,16 @@ export function encodeFilenameForHeader(storageKey: string): string {
 export function createFileResponse(file: FileResponse): NextResponse {
   const { contentType, disposition } = getSecureFileHeaders(file.filename, file.contentType)
 
+  // Sim pages store an extensionless name and serve/download as compiled
+  // HTML — re-append the extension so the saved file opens in a browser.
+  const servedFilename =
+    contentType === 'text/html' && !/\.[A-Za-z0-9]{1,8}$/.test(file.filename)
+      ? `${file.filename}.html`
+      : file.filename
+
   const headers: Record<string, string> = {
     'Content-Type': contentType,
-    'Content-Disposition': `${disposition}; ${encodeFilenameForHeader(file.filename)}`,
+    'Content-Disposition': `${disposition}; ${encodeFilenameForHeader(servedFilename)}`,
     // Default to PRIVATE: this response is served only after access verification, so it must never be
     // stored by a shared cache/CDN and re-served cross-user. Genuinely public assets (avatars, OG images,
     // workspace logos) pass an explicit `cacheControl` (see PUBLIC_ASSET_CACHE_CONTROL in the serve route).

apps/sim/app/workspace/[workspaceId]/files/files.tsx

@@ -208,6 +208,7 @@ const MIME_TYPE_LABELS: Record<string, string> = {
   'text/csv': 'CSV',
   'text/plain': 'Text',
   'text/html': 'HTML',
+  'text/x-sim-page': 'Page',
   'text/markdown': 'Markdown',
 }
 

apps/sim/components/icons/document-icons.tsx

@@ -301,6 +301,12 @@ export function getDocumentIcon(
     return ChartFileIcon
   }
 
+  // Sim pages present as plain documents, not as HTML artifacts — the .html
+  // is an implementation detail (legacy pages still carry the extension).
+  if (mimeType === 'text/x-sim-page') {
+    return DefaultFileIcon
+  }
+
   if (mimeType === 'text/html' || extension === 'html' || extension === 'htm') {
     return HtmlIcon
   }

apps/sim/lib/copilot/tools/server/files/create-file.ts

@@ -13,6 +13,7 @@ import {
   createWorkspaceFileByPath,
   updateWorkspaceFileContentByPath,
 } from '@/lib/workspace-files/application/write-workspace-file-by-path'
+import { SIM_PAGE_CONTENT_TYPE } from '@/lib/workspace-files/page-compile'
 
 const logger = createLogger('CreateFileServerTool')
 const CREATE_FILE_TOOL_ID = 'create_empty_file'
@@ -62,6 +63,11 @@ export const createFileServerTool: BaseServerTool<CreateFileArgs, CreateFileResu
     // creation) — or when the first apply_file_edit finds actual page
     // source, which re-stamps the record from reality either way.
     const contentType = outputFile?.mimeType ?? inferContentType(outputPath, explicitType)
+    // A Sim page's stored name drops the .html the agent signals format with:
+    // the record type carries the format, every surface then shows the bare
+    // name with the plain file icon, and downloads re-append the extension.
+    const storedPath =
+      contentType === SIM_PAGE_CONTENT_TYPE ? outputPath.replace(/\.html?$/i, '') : outputPath
     assertServerToolNotAborted(context)
     const mode = outputFile?.mode ?? 'create'
     // An empty shell provably contains no secrets; recording that keeps the
@@ -71,7 +77,7 @@ export const createFileServerTool: BaseServerTool<CreateFileArgs, CreateFileResu
     const createShell = () =>
       executeCopilotFileUseCase(context, createWorkspaceFileByPath, {
         workspaceId,
-        path: outputPath,
+        path: storedPath,
         mode: 'create',
         content: '',
         encoding: 'utf-8',
@@ -85,7 +91,7 @@ export const createFileServerTool: BaseServerTool<CreateFileArgs, CreateFileResu
         try {
           result = await executeCopilotFileUseCase(context, updateWorkspaceFileContentByPath, {
             workspaceId,
-            path: outputPath,
+            path: storedPath,
             mode,
             content: '',
             encoding: 'utf-8',

apps/sim/lib/copilot/tools/server/files/doc-asset-extract-pdf.ts

@@ -17,6 +17,12 @@ const EXTRACT_TIMEOUT_MS = 180_000
  * row adjacency and recomposites them into an RGBA PNG; masks are never
  * shipped as standalone assets.
  *
+ * Assets are shipped at presentation resolution: print-dpi originals are
+ * downscaled to a 2560px long edge (alpha preserved) and exotic formats
+ * (TIFF/JP2) are normalized to PNG/JPEG, so the extracted set stays inside
+ * the doc-compile staging budget and re-stages fast on every slide edit.
+ * Images already within range keep their original bytes.
+ *
  * Unlike OOXML there is no declared theme in a PDF, so the palette is
  * explicitly labeled inferred; fonts are names only (embedded font files are
  * subsetted and license-restricted).
@@ -167,6 +173,42 @@ for num, ref in alpha_of.items():
     except Exception:
         pass  # undecodable mask: the opaque base still beats losing the asset
 
+# Downscale print-resolution assets and normalize exotic formats. Embedded PDF
+# images are often 300-dpi originals several MB each, a slide never shows more
+# than ~2560px on the long edge, and every deck compile re-stages the whole
+# referenced set — so oversized extractions slow every edit and blow the byte
+# staging budget. Images already within range keep their original bytes.
+MAX_ASSET_EDGE = 2560
+shipped_dims = {}
+for num, path in list(files.items()):
+    try:
+        im = Image.open(path)
+        im.load()
+    except Exception:
+        continue  # undecodable (jbig2/ccitt params etc.): ship as-is
+    ext = path.rsplit(".", 1)[1].lower()
+    exotic = ext not in ("png", "jpg", "jpeg")
+    w, h = im.size
+    scale = MAX_ASSET_EDGE / float(max(w, h))
+    if scale >= 1 and not exotic:
+        continue
+    if scale < 1:
+        im = im.resize((max(1, int(w * scale)), max(1, int(h * scale))), Image.LANCZOS)
+    has_alpha = im.mode in ("RGBA", "LA", "PA") or (im.mode == "P" and "transparency" in im.info)
+    try:
+        if ext in ("jpg", "jpeg") or (exotic and not has_alpha):
+            out = path.rsplit(".", 1)[0] + ".jpg"
+            im.convert("RGB").save(out, "JPEG", quality=85, optimize=True)
+        else:
+            out = path.rsplit(".", 1)[0] + ".png"
+            im.convert("RGBA" if has_alpha else "RGB").save(out, "PNG", optimize=True)
+    except Exception:
+        continue  # unencodable: keep the original bytes
+    if out != path:
+        os.remove(path)
+    files[num] = out
+    shipped_dims[num] = im.size
+
 def to_hex(color):
     if color is None:
         return None
@@ -350,10 +392,11 @@ for num, path in sorted(files.items()):
     ]
     with open(path, "rb") as f:
         data = base64.b64encode(f.read()).decode()
+    dims = shipped_dims.get(num)
     images.append({
         "name": "image%d.%s" % (num, ext),
-        "widthPx": row["width"],
-        "heightPx": row["height"],
+        "widthPx": dims[0] if dims else row["width"],
+        "heightPx": dims[1] if dims else row["height"],
         "placements": pls,
         "base64": data,
     })

apps/sim/lib/copilot/tools/server/files/doc-compile.test.ts

@@ -108,32 +108,12 @@ describe('collectReferencedFileIds', () => {
     expect(collectReferencedFileIds(`slide.addText('hello', { x: 1, y: 1 })`)).toEqual(new Set())
   })
 
-  it('retains all references at the remote staging limit', () => {
-    const ids = collectReferencedFileIds(referencedFileSource(20))
-
-    expect(ids.size).toBe(20)
-    expect(ids.has('file-19')).toBe(true)
-  })
-
-  it('stops collecting at the one-over-limit sentinel', () => {
+  it('has no reference-count limit — a large deck rebuild references every extracted image', () => {
     const ids = collectReferencedFileIds(referencedFileSource(100))
 
-    expect(ids.size).toBe(21)
-    expect(ids.has('file-20')).toBe(true)
-    expect(ids.has('file-21')).toBe(false)
-  })
-
-  it('rejects remote compilation at 21 references before resolving file metadata', async () => {
-    await expect(
-      compileDoc({
-        source: referencedFileSource(21),
-        fileName: 'report.pdf',
-        workspaceId: 'workspace-1',
-        filePrincipal: FILE_PRINCIPAL,
-      })
-    ).rejects.toThrow('More than 20 referenced input files; maximum is 20')
-    expect(readWorkspaceFileMetadataMock).not.toHaveBeenCalled()
-    expect(executeInSandboxMock).not.toHaveBeenCalled()
+    expect(ids.size).toBe(100)
+    expect(ids.has('file-0')).toBe(true)
+    expect(ids.has('file-99')).toBe(true)
   })
 
   it('compiles referenced bytes and returns their canonical contributor identity', async () => {

apps/sim/lib/copilot/tools/server/files/doc-compile.ts

@@ -106,11 +106,11 @@ const FILE_HELPER_RE =
   /\b(?:getFileBase64|addImage|drawImage)\(\s*(?:[A-Za-z_$][\w$]*\s*,\s*)?['"]([A-Za-z0-9_-]+)['"]/g
 
 // The doc source is user/LLM-controlled, so bound how much it can pull into the
-// sandbox: each `/home/user/inputs/<id>` reference is only ~35 bytes, so the
-// source-size cap alone does not bound staging. These caps prevent an
-// authenticated member from forcing thousands of (or very large) workspace files
-// to be downloaded and base64-held in-process per compile request.
-const MAX_STAGED_INPUTS = 20
+// sandbox by BYTES (per file and total) — an authenticated member must not be
+// able to force very large workspace downloads to be base64-held in-process per
+// compile. There is deliberately no count cap: a deck rebuild references every
+// extracted image across the whole source (references accumulate over slides),
+// and the byte caps already bound the volume.
 const MAX_STAGED_FILE_BYTES = 25 * 1024 * 1024
 const MAX_STAGED_TOTAL_BYTES = 50 * 1024 * 1024
 
@@ -147,16 +147,14 @@ function referencedImageIdentities(
  * image-helper call sites and the legacy `/home/user/inputs/<id>` path. Matching
  * is scoped to the helper calls (not bare id-like strings in slide text), and the
  * caller skips any id that does not resolve to a real file, so over-matching is
- * harmless. Retention stops at one over the remote staging limit: callers need
- * only distinguish no references, an admissible set, and an oversized set.
+ * harmless.
  */
 export function collectReferencedFileIds(source: string): Set<string> {
   const ids = new Set<string>()
   for (const re of [INPUT_PATH_RE, FILE_HELPER_RE]) {
     for (const match of source.matchAll(re)) {
       if (match[1]) {
         ids.add(match[1])
-        if (ids.size > MAX_STAGED_INPUTS) return ids
       }
     }
   }
@@ -169,11 +167,6 @@ async function resolveReferencedImages(
   principal: Principal,
   ids = collectReferencedFileIds(source)
 ): Promise<ReferencedImageResolution> {
-  if (ids.size > MAX_STAGED_INPUTS) {
-    throw new Error(
-      `More than ${MAX_STAGED_INPUTS} referenced input files; maximum is ${MAX_STAGED_INPUTS}. Reference fewer files.`
-    )
-  }
   if (ids.size === 0) {
     return { images: [], referenceCount: 0 }
   }
@@ -207,12 +200,6 @@ async function stageReferencedImages(
   workspaceId: string,
   principal: Principal
 ): Promise<SandboxFile[]> {
-  if (resolution.referenceCount > MAX_STAGED_INPUTS) {
-    throw new Error(
-      `More than ${MAX_STAGED_INPUTS} referenced input files; maximum is ${MAX_STAGED_INPUTS}. Reference fewer files.`
-    )
-  }
-
   const files: SandboxFile[] = []
   let totalBytes = 0
   for (const { fileId, record } of resolution.images) {
@@ -225,8 +212,8 @@ async function stageReferencedImages(
       continue
     }
     if (totalBytes + (record.size ?? 0) > MAX_STAGED_TOTAL_BYTES) {
-      throw new Error(
-        `Referenced input files exceed the ${MAX_STAGED_TOTAL_BYTES} byte staging budget.`
+      throw new DocCompileUserError(
+        `Referenced input files exceed the ${Math.round(MAX_STAGED_TOTAL_BYTES / (1024 * 1024))} MB total staging budget (the whole document's references count). Use smaller/compressed copies of the largest images.`
       )
     }
     const content = await readWorkspaceFileContent.execute({
@@ -263,8 +250,10 @@ async function stageReferencedImages(
     // outside the catch above so it fails the compile rather than being skipped.
     totalBytes += buffer.length
     if (totalBytes > MAX_STAGED_TOTAL_BYTES) {
-      throw new Error(
-        `Referenced input files exceed the ${MAX_STAGED_TOTAL_BYTES} byte staging budget.`
+      // A user-fixable condition, not a transient failure: retrying with the
+      // same references can never succeed — the images must shrink.
+      throw new DocCompileUserError(
+        `Referenced input files exceed the ${Math.round(MAX_STAGED_TOTAL_BYTES / (1024 * 1024))} MB total staging budget (the whole document's references count). Use smaller/compressed copies of the largest images.`
       )
     }
     files.push({
@@ -608,7 +597,7 @@ export async function loadCompiledDocByExt(
   const fmt = await getE2BDocFormat(`x.${ext}`)
   if (!fmt) return null
   const referencedFileIds = collectReferencedFileIds(source)
-  if (referencedFileIds.size > MAX_STAGED_INPUTS || !options.filePrincipal) {
+  if (!options.filePrincipal) {
     if (referencedFileIds.size === 0) {
       const buffer = await loadCompiledDoc(workspaceId, source, fmt.ext)
       return buffer ? { buffer, contentType: fmt.contentType } : null

apps/sim/lib/copilot/tools/server/files/edit-content.ts

@@ -94,7 +94,9 @@ export const editContentServerTool: BaseServerTool<EditContentArgs, EditContentR
       // the source format, so it is rejected with the steer back to source.
       // Patches are exempt: small in-place fixes on a legacy stored-compiled
       // page legitimately contain compiled fragments.
-      const isHtmlTarget = fileRecord.name.toLowerCase().endsWith('.html')
+      // Sim pages store an extensionless name; the record type marks them.
+      const isHtmlTarget =
+        fileRecord.name.toLowerCase().endsWith('.html') || fileRecord.type === SIM_PAGE_CONTENT_TYPE
       if (
         isHtmlTarget &&
         (operation === 'append' || operation === 'update') &&