sim:chart hydrated embeds: inline or .chart file refs, live table reads per serve

Author: Sg312

apps/sim/app/api/files/serve/[...path]/route.ts

@@ -93,7 +93,12 @@ async function resolveServableBytes(params: {
     const text = buffer.toString('utf8')
     if (isSimPageSource(text)) {
       return {
-        buffer: Buffer.from(await renderSimPageDocumentWithAssets(text, { workspaceId }), 'utf8'),
+        buffer: Buffer.from(
+          // The principal lets referenced table-backed charts read CURRENT
+          // rows under the viewer's own authorization on every serve.
+          await renderSimPageDocumentWithAssets(text, { workspaceId, principal: filePrincipal }),
+          'utf8'
+        ),
         contentType: 'text/html',
       }
     }

apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/chart-preview.tsx

@@ -5,6 +5,7 @@ import { getErrorMessage } from '@sim/utils/errors'
 import type { EChartsOption } from 'echarts'
 import { useTheme } from 'next-themes'
 import { buildChartRenderOption } from '@/lib/charts/option'
+import { type ChartSpec, parseChartSpec, shapeTableRows } from '@/lib/charts/spec'
 import { getColumnId } from '@/lib/table/column-keys'
 import { useTable, useTableRowsSample } from '@/hooks/queries/tables'
 import { PreviewLoadingFrame } from './preview-shared'
@@ -13,166 +14,6 @@ import { PreviewLoadingFrame } from './preview-shared'
 const CHART_ROWS_MAX = 5000
 const CHART_ROWS_DEFAULT = 1000
 
-type ChartAggregateOp = 'sum' | 'avg' | 'min' | 'max' | 'count'
-
-const CHART_AGGREGATE_OPS = new Set<string>(['sum', 'avg', 'min', 'max', 'count'])
-
-interface ChartTableSource {
-  type: 'table'
-  tableId: string
-  filter?: unknown
-  sort?: unknown
-  limit?: number
-  /** Group rows by these columns; requires `aggregate`. */
-  groupBy?: string[]
-  /** Metric column → op, computed per group. */
-  aggregate?: Record<string, ChartAggregateOp>
-  /** Fan the aggregated metric(s) out into one column per distinct value of this column. */
-  pivot?: string
-}
-
-interface ChartStaticSource {
-  type: 'static'
-  rows?: Array<Record<string, unknown>>
-}
-
-/**
- * A `.chart` file (`text/x-sim-chart`): a declarative ECharts document. The
- * `option` is a plain ECharts option object; `source` optionally supplies the
- * data — inline rows, or a live read of a Sim table injected as
- * `option.dataset.source` so the chart stays current with the table.
- */
-interface ChartSpec {
-  schema_version: number
-  title?: string
-  source?: ChartStaticSource | ChartTableSource
-  option: Record<string, unknown>
-}
-
-export function parseChartSpec(content: string): { spec?: ChartSpec; error?: string } {
-  let raw: unknown
-  try {
-    raw = JSON.parse(content)
-  } catch (e) {
-    return { error: getErrorMessage(e, 'not valid JSON') }
-  }
-  if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) {
-    return { error: 'chart document must be a JSON object' }
-  }
-  const doc = raw as Record<string, unknown>
-  if (doc.schema_version !== 1) {
-    return { error: 'chart document must declare "schema_version": 1' }
-  }
-  if (doc.option === null || typeof doc.option !== 'object' || Array.isArray(doc.option)) {
-    return { error: 'chart document must declare an "option" object (an ECharts option)' }
-  }
-  const source = doc.source as ChartSpec['source']
-  if (source !== undefined) {
-    if (source === null || typeof source !== 'object') {
-      return { error: '"source" must be an object' }
-    }
-    if (source.type === 'table') {
-      if (typeof source.tableId !== 'string' || source.tableId === '') {
-        return { error: 'a table source must declare "tableId"' }
-      }
-      if (source.groupBy !== undefined) {
-        if (!Array.isArray(source.groupBy) || source.groupBy.some((c) => typeof c !== 'string')) {
-          return { error: '"groupBy" must be an array of column names' }
-        }
-        if (source.aggregate === null || typeof source.aggregate !== 'object') {
-          return { error: '"groupBy" requires an "aggregate" object ({column: op})' }
-        }
-        const ops = Object.values(source.aggregate)
-        if (ops.length === 0 || ops.some((op) => !CHART_AGGREGATE_OPS.has(String(op)))) {
-          return { error: '"aggregate" ops must be sum, avg, min, max, or count' }
-        }
-      }
-      if (source.pivot !== undefined) {
-        if (typeof source.pivot !== 'string') return { error: '"pivot" must be a column name' }
-        if (!source.groupBy) return { error: '"pivot" requires "groupBy" and "aggregate"' }
-      }
-    } else if (source.type === 'static') {
-      if (source.rows !== undefined && !Array.isArray(source.rows)) {
-        return { error: 'a static source\'s "rows" must be an array' }
-      }
-    } else {
-      return { error: '"source.type" must be "static" or "table"' }
-    }
-  }
-  return { spec: doc as unknown as ChartSpec }
-}
-
-function aggregateValues(
-  rows: Array<Record<string, unknown>>,
-  column: string,
-  op: ChartAggregateOp
-): number {
-  if (op === 'count') return rows.length
-  const values = rows.map((r) => Number(r[column])).filter((n) => Number.isFinite(n))
-  if (values.length === 0) return 0
-  switch (op) {
-    case 'sum':
-      return values.reduce((a, b) => a + b, 0)
-    case 'avg':
-      return values.reduce((a, b) => a + b, 0) / values.length
-    case 'min':
-      return Math.min(...values)
-    case 'max':
-      return Math.max(...values)
-  }
-}
-
-/**
- * Client-side shaping for table sources: group → aggregate → optionally pivot
- * one column's distinct values into per-value columns. Groups keep first-seen
- * order, so the source's `sort` decides the category order. This is the whole
- * "query engine" — deliberately tiny; anything fancier belongs in a static
- * source with precomputed rows.
- */
-export function shapeTableRows(
-  rows: Array<Record<string, unknown>>,
-  source: ChartTableSource
-): Array<Record<string, unknown>> {
-  const { groupBy, aggregate, pivot } = source
-  if (!groupBy || groupBy.length === 0 || !aggregate) return rows
-
-  const groups = new Map<string, Array<Record<string, unknown>>>()
-  for (const row of rows) {
-    const key = groupBy.map((c) => String(row[c] ?? '')).join('\u0000')
-    const bucket = groups.get(key)
-    if (bucket) bucket.push(row)
-    else groups.set(key, [row])
-  }
-
-  const metrics = Object.entries(aggregate)
-  const out: Array<Record<string, unknown>> = []
-  for (const bucket of groups.values()) {
-    const shaped: Record<string, unknown> = {}
-    for (const c of groupBy) shaped[c] = bucket[0][c]
-    if (pivot) {
-      const byValue = new Map<string, Array<Record<string, unknown>>>()
-      for (const row of bucket) {
-        const value = String(row[pivot] ?? '')
-        const slice = byValue.get(value)
-        if (slice) slice.push(row)
-        else byValue.set(value, [row])
-      }
-      for (const [value, slice] of byValue) {
-        for (const [column, op] of metrics) {
-          const name = metrics.length === 1 ? value : `${value} ${column}`
-          shaped[name] = aggregateValues(slice, column, op)
-        }
-      }
-    } else {
-      for (const [column, op] of metrics) {
-        shaped[column] = aggregateValues(bucket, column, op)
-      }
-    }
-    out.push(shaped)
-  }
-  return out
-}
-
 function buildOption(spec: ChartSpec, rows: Array<Record<string, unknown>> | null): EChartsOption {
   return buildChartRenderOption({ title: spec.title, option: spec.option, rows }) as EChartsOption
 }

apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/preview-panel.tsx

@@ -14,6 +14,7 @@ import {
 } from '@/lib/workspace-files/artifact-stylesheet'
 import { compileSimPage, isSimPageSource } from '@/lib/workspace-files/page-compile'
 import { useHorizontalWheelScroll } from '@/app/workspace/[workspaceId]/files/components/file-viewer/use-horizontal-wheel-scroll'
+import { useWorkspaceFileBinary } from '@/hooks/queries/workspace-files'
 import { ChartPreview } from './chart-preview'
 import { type CsvImportFileDescriptor, useCsvTruncationImport } from './csv-import'
 import { DataTable } from './data-table'
@@ -80,7 +81,15 @@ export const PreviewPanel = memo(function PreviewPanel({
   const previewType = resolvePreviewType(mimeType, filename)
 
   if (previewType === 'html')
-    return <HtmlPreview content={content} isStreaming={isStreaming} workspaceId={workspaceId} />
+    return (
+      <HtmlPreview
+        content={content}
+        isStreaming={isStreaming}
+        workspaceId={workspaceId}
+        fileId={fileId}
+        fileKey={fileKey}
+      />
+    )
   if (previewType === 'csv')
     return (
       <CsvPreview
@@ -291,16 +300,35 @@ const HtmlPreview = memo(function HtmlPreview({
   content,
   isStreaming,
   workspaceId,
+  fileId,
+  fileKey,
 }: {
   content: string
   isStreaming?: boolean
   workspaceId?: string
+  fileId?: string
+  fileKey?: string
 }) {
   const { resolvedTheme } = useTheme()
   const router = useRouter()
   const batchedContent = useStreamBatchedValue(content, isStreaming === true, 2000)
+  // A SAVED sim page prefers the server-compiled document — the pptx/docx
+  // model: the serve route resolves chart references (reading a table's
+  // CURRENT rows under the viewer's authorization) and inlines the chart
+  // runtime, and reopening/refocusing refetches, so the page recompiles on
+  // reload. While it loads — and always while streaming/editing — the client
+  // compile below stands in, with chart figures as placeholders.
+  const isSavedPage =
+    Boolean(fileId && fileKey && workspaceId) && isStreaming !== true && isSimPageSource(content)
+  const served = useWorkspaceFileBinary(workspaceId ?? '', fileId ?? '', fileKey ?? '', {
+    enabled: isSavedPage,
+  })
+  const servedHtml = useMemo(
+    () => (isSavedPage && served.data ? new TextDecoder().decode(served.data) : null),
+    [isSavedPage, served.data]
+  )
   const builtContent = buildHtmlPreviewDocument(
-    batchedContent,
+    servedHtml ?? batchedContent,
     resolvedTheme === 'dark' ? 'dark' : 'light',
     workspaceId
   )

apps/sim/lib/charts/fence.ts

@@ -0,0 +1,121 @@
+import { buildChartRenderOption } from '@/lib/charts/option'
+
+/**
+ * The `sim:chart` page fence, rendered as PURE markup: the final ECharts
+ * option rides inside the figure as JSON, and a single inline hydration
+ * script (appended once per page by the server pass or the client preview)
+ * boots every figure. That keeps the compiled page self-contained — fully
+ * interactive in the app's sandboxed iframe (inline scripts are allowed) and
+ * in a downloaded copy, which carries the library and the data with it.
+ *
+ * Two payload shapes are valid:
+ *  - inline: `{ option, rows?, height? }` — data baked into the page.
+ *  - reference: `{ file, height? }` — a `.chart` workspace file; emitted as a
+ *    marker figure the SERVER pass resolves (reading the file and, for table
+ *    sources, the table's CURRENT rows under the viewer's authorization), so
+ *    every in-app view is fresh while a download freezes that serve's data.
+ */
+
+export const CHART_FENCE_DEFAULT_HEIGHT = 420
+const CHART_FENCE_MIN_HEIGHT = 200
+const CHART_FENCE_MAX_HEIGHT = 800
+
+export interface ChartFenceInlinePayload {
+  option: Record<string, unknown>
+  rows?: Array<Record<string, unknown>>
+  height?: number
+}
+
+function clampHeight(height: unknown): number {
+  const parsed = typeof height === 'number' ? Math.round(height) : CHART_FENCE_DEFAULT_HEIGHT
+  return Math.min(Math.max(parsed, CHART_FENCE_MIN_HEIGHT), CHART_FENCE_MAX_HEIGHT)
+}
+
+function escapeHtml(text: string): string {
+  return text
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+}
+
+/** A `</script` or `<!--` inside the JSON would terminate/confuse the carrier tag. */
+function escapeJsonForScriptTag(json: string): string {
+  return json.replace(/</g, '\\u003c')
+}
+
+/** Accepts `sim:file/<id>` or a bare file id; returns the bare id. */
+function chartFileRef(value: unknown): string | null {
+  if (typeof value !== 'string' || value === '') return null
+  const match = value.match(/^(?:sim:file\/)?([A-Za-z0-9-]+)$/)
+  return match ? match[1] : null
+}
+
+/** The resolved-figure markup: spec JSON carrier + canvas the hydrator fills. */
+export function renderResolvedChartFigure(
+  payload: ChartFenceInlinePayload,
+  caption: string
+): string {
+  const height = clampHeight(payload.height)
+  const option = buildChartRenderOption({ option: payload.option, rows: payload.rows ?? null })
+  const spec = escapeJsonForScriptTag(JSON.stringify({ option }))
+  const figcaption = caption ? `<figcaption>${escapeHtml(caption)}</figcaption>` : ''
+  return `<figure class="sim-chart"><script type="application/json" class="sim-chart-spec">${spec}</script><div class="sim-chart-canvas" style="height:${height}px"><div class="sim-chart-placeholder">Chart</div></div>${figcaption}</figure>`
+}
+
+/**
+ * Renders a `sim:chart` fence payload to figure markup, or null when the
+ * payload matches neither valid shape (the compiler reports a diagnostic).
+ */
+export function renderChartFenceMarkup(payload: unknown, caption: string): string | null {
+  if (payload === null || typeof payload !== 'object' || Array.isArray(payload)) return null
+  const doc = payload as Record<string, unknown>
+
+  const fileRef = doc.file === undefined ? null : chartFileRef(doc.file)
+  if (doc.file !== undefined) {
+    if (fileRef === null) return null
+    const height = clampHeight(doc.height)
+    const figcaption = caption ? `<figcaption>${escapeHtml(caption)}</figcaption>` : ''
+    return `<figure class="sim-chart" data-sim-chart-file="${escapeHtml(fileRef)}" data-sim-chart-height="${height}" data-sim-chart-caption="${escapeHtml(caption)}"><div class="sim-chart-placeholder">Chart — resolves from the workspace when the page is served</div>${figcaption}</figure>`
+  }
+
+  if (doc.option === null || typeof doc.option !== 'object' || Array.isArray(doc.option)) {
+    return null
+  }
+  if (doc.rows !== undefined && !Array.isArray(doc.rows)) return null
+  return renderResolvedChartFigure(doc as unknown as ChartFenceInlinePayload, caption)
+}
+
+/** Matches the unresolved reference markers emitted for `{"file": ...}` payloads. */
+export const CHART_REF_FIGURE_RE =
+  /<figure class="sim-chart" data-sim-chart-file="([^"]+)" data-sim-chart-height="(\d+)" data-sim-chart-caption="([^"]*)">[\s\S]*?<\/figure>/g
+
+export function chartUnavailableFigure(reason: string): string {
+  return `<figure class="sim-chart"><div class="sim-chart-placeholder">${reason}</div></figure>`
+}
+
+export function htmlHasChartFigures(html: string): boolean {
+  return html.includes('class="sim-chart"')
+}
+
+/**
+ * The per-page bootstrap appended after the ECharts library source: hydrates
+ * every resolved figure, themed by the page's `data-theme` stamp (falling
+ * back to the OS scheme), and follows container resizes.
+ */
+export const CHART_HYDRATION_SNIPPET = `(() => {
+  const stamped = document.documentElement.getAttribute('data-theme')
+  const dark = stamped === 'dark' || (stamped !== 'light' && window.matchMedia('(prefers-color-scheme: dark)').matches)
+  for (const figure of document.querySelectorAll('figure.sim-chart')) {
+    const specTag = figure.querySelector('script.sim-chart-spec')
+    const canvas = figure.querySelector('.sim-chart-canvas')
+    if (!specTag || !canvas) continue
+    let spec
+    try { spec = JSON.parse(specTag.textContent || '') } catch { continue }
+    if (!spec || typeof spec.option !== 'object') continue
+    canvas.textContent = ''
+    const chart = echarts.init(canvas, dark ? 'dark' : undefined)
+    try { chart.setOption(spec.option) } catch { chart.dispose(); continue }
+    new ResizeObserver(() => chart.resize()).observe(canvas)
+  }
+})()`

…onents/file-viewer/chart-preview.test.ts apps/sim/lib/charts/spec.test.tsapps/sim/app/workspace/[workspaceId]/files/components/file-viewer/chart-preview.test.ts renamed to apps/sim/lib/charts/spec.test.ts apps/sim/app/workspace/[workspaceId]/files/components/file-viewer/chart-preview.test.ts renamed to apps/sim/lib/charts/spec.test.ts

@@ -2,10 +2,7 @@
  * @vitest-environment node
  */
 import { describe, expect, it } from 'vitest'
-import {
-  parseChartSpec,
-  shapeTableRows,
-} from '@/app/workspace/[workspaceId]/files/components/file-viewer/chart-preview'
+import { parseChartSpec, shapeTableRows } from '@/lib/charts/spec'
 
 const rows = [
   { month: '2024-01', region: 'NA', revenue: 100, conversion: 4 },