diff --git a/apps/web/src/components/network/NetworkViewer.tsx b/apps/web/src/components/network/NetworkViewer.tsx index 6c400b4..56a74cc 100644 --- a/apps/web/src/components/network/NetworkViewer.tsx +++ b/apps/web/src/components/network/NetworkViewer.tsx @@ -1,6 +1,6 @@ 'use client' -import { useCallback, useMemo, useRef } from 'react' +import { memo, useCallback, useMemo, useRef } from 'react' import { VisualizationContainer } from '@/components/visualization/VisualizationContainer' import { @@ -24,7 +24,7 @@ import { } from '@/lib/network/labels' import { NODE_RADIUS } from '@/lib/network/layout' import { availableEdgeTypes, availableNodeTypes, edgeById, nodeById } from '@/lib/network/model' -import type { GraphEdge, GraphNode } from '@/lib/network/types' +import type { Graph, GraphEdge, GraphLayout, GraphNode, NetworkViewport } from '@/lib/network/types' import type { NetworkViewerResult } from '@/lib/network/useNetworkViewer' function NetworkSummary({ result }: { result: NetworkViewerResult }) { @@ -138,11 +138,22 @@ function NetworkControls({ result }: { result: NetworkViewerResult }) { ) } -function EdgeElement({ edge, result }: { edge: GraphEdge; result: NetworkViewerResult }) { - const graph = result.graph - if (graph === undefined) return null - const points = edgeScreenPoints(edge, result.layout, result.viewport) - const selected = edge.id === result.selectedEdgeId +const EdgeElement = memo(function EdgeElement({ + edge, + graph, + layout, + viewport, + selected, + onSelect, +}: { + edge: GraphEdge + graph: Graph + layout: GraphLayout + viewport: NetworkViewport + selected: boolean + onSelect: (edgeId: string | null) => void +}) { + const points = edgeScreenPoints(edge, layout, viewport) const label = edgeAccessibleLabel(edge, graph) const color = edgeTypeColor(edge.type) return ( @@ -177,22 +188,33 @@ function EdgeElement({ edge, result }: { edge: GraphEdge; result: NetworkViewerR onKeyDown={(event) => { if (event.key === 'Enter' || event.key === ' ') { event.preventDefault() - result.selectEdge(selected ? null : edge.id) + onSelect(selected ? null : edge.id) } }} - onClick={() => result.selectEdge(selected ? null : edge.id)} + onClick={() => onSelect(selected ? null : edge.id)} /> ) -} +}) -function NodeElement({ node, result }: { node: GraphNode; result: NetworkViewerResult }) { - const position = result.layout.positions.get(node.id) +const NodeElement = memo(function NodeElement({ + node, + layout, + viewport, + selected, + onSelect, +}: { + node: GraphNode + layout: GraphLayout + viewport: NetworkViewport + selected: boolean + onSelect: (nodeId: string | null) => void +}) { + const position = layout.positions.get(node.id) if (position === undefined) return null - const selected = node.id === result.selectedNodeId const label = nodeAccessibleLabel(node) - const box = nodeScreenBox(position, result.viewport) - const scale = result.viewport.scale + const box = nodeScreenBox(position, viewport) + const scale = viewport.scale const centerX = box.x + box.width / 2 const centerY = box.y + NODE_RADIUS * scale return ( @@ -228,14 +250,14 @@ function NodeElement({ node, result }: { node: GraphNode; result: NetworkViewerR onKeyDown={(event) => { if (event.key === 'Enter' || event.key === ' ') { event.preventDefault() - result.selectNode(selected ? null : node.id) + onSelect(selected ? null : node.id) } }} - onClick={() => result.selectNode(selected ? null : node.id)} + onClick={() => onSelect(selected ? null : node.id)} /> ) -} +}) function NetworkGraph({ result }: { result: NetworkViewerResult }) { const graph = result.graph @@ -305,10 +327,25 @@ function NetworkGraph({ result }: { result: NetworkViewerResult }) { {hasNodes ? ( {edges.map((edge) => ( - + ))} {nodes.map((node) => ( - + ))} ) : ( diff --git a/apps/web/src/components/scientific/CoverageChart.tsx b/apps/web/src/components/scientific/CoverageChart.tsx index 4d8d2ae..8321546 100644 --- a/apps/web/src/components/scientific/CoverageChart.tsx +++ b/apps/web/src/components/scientific/CoverageChart.tsx @@ -9,6 +9,7 @@ import type { AxisTick } from '@/lib/genome/geometry' import { createScale } from '@/lib/genome/geometry' import type { CoverageBin } from '@/lib/scientific/advancedTypes' import { coverageBinTooltip } from '@/lib/scientific/coverage' +import { coverageColumns } from '@/lib/scientific/downsample' import { DEFAULT_CHART_HEIGHT, DEFAULT_CHART_MARGINS, @@ -26,6 +27,7 @@ const AXIS_COLOR = '#cbd5e1' const GRID_COLOR = '#e2e8f0' const COVERAGE_COLOR = '#2563eb' const CAPTION_COLOR = '#94a3b8' +const MAX_COVERAGE_COLUMNS = 2000 export interface CoverageChartProps { /** View model produced by `useCoverageChart`. */ @@ -285,14 +287,30 @@ function CoverageBins({ () => createScale(viewport.start, viewport.end, plot.width), [viewport, plot], ) - if (dataset === undefined || chromosome === '') return null - const bins = dataset.bins.filter((bin) => bin.chromosome === chromosome) + const bins = useMemo( + () => + dataset === undefined ? [] : dataset.bins.filter((bin) => bin.chromosome === chromosome), + [dataset, chromosome], + ) + // When a chromosome has far more bins than pixels, aggregate to per-pixel + // peak-preserving columns so both the path and the interactive targets stay + // bounded (see docs/visualization/performance.md). Under the cap, bins are + // rendered exactly as provided. + const columns = useMemo( + () => coverageColumns(bins, (base) => scale.toX(base), plot.width, MAX_COVERAGE_COLUMNS), + [bins, scale, plot.width], + ) + const points = useMemo( + () => + columns.map((bin) => { + const x0 = scale.toX(bin.start) + const x1 = scale.toX(bin.end + 1) + return { bin, x0, x1, y: yScale.toPixel(bin.coverage) } + }), + [columns, scale, yScale], + ) - const points = bins.map((bin) => { - const x0 = scale.toX(bin.start) - const x1 = scale.toX(bin.end + 1) - return { bin, x0, x1, y: yScale.toPixel(bin.coverage) } - }) + if (dataset === undefined || chromosome === '') return null const areaPath = buildAreaPath(plot, points) const linePath = buildLinePath(points) diff --git a/apps/web/src/components/scientific/DistributionChart.tsx b/apps/web/src/components/scientific/DistributionChart.tsx index 8bd313b..14c4858 100644 --- a/apps/web/src/components/scientific/DistributionChart.tsx +++ b/apps/web/src/components/scientific/DistributionChart.tsx @@ -5,6 +5,7 @@ import { useMemo, useState } from 'react' import { VisualizationContainer } from '@/components/visualization/VisualizationContainer' import type { DistributionDataset } from '@/lib/scientific/advancedTypes' import { distributionTooltip, valuesForGroup } from '@/lib/scientific/distribution' +import { decimateItems } from '@/lib/scientific/downsample' import { DEFAULT_CHART_HEIGHT, DEFAULT_CHART_MARGINS, @@ -28,6 +29,7 @@ const MEDIAN_COLOR = '#0f172a' const OUTLIER_COLOR = '#dc2626' const JITTER_RADIUS = 2.5 const HIT_RADIUS = 12 +const MAX_SCATTER_POINTS_PER_GROUP = 1000 export interface DistributionChartProps { /** View model produced by `useDistributionChart`. */ @@ -212,7 +214,16 @@ function GroupBox({ if (slot === undefined) return null const [focused, setFocused] = useState(false) - const values = useMemo(() => valuesForGroup(dataset, entry.group), [dataset, entry.group]) + const scatterValues = useMemo(() => { + const all = valuesForGroup(dataset, entry.group) + if (all.length <= MAX_SCATTER_POINTS_PER_GROUP) return all + const isOutlier = (value: number) => + whiskers !== undefined && (value < whiskers.lower || value > whiskers.upper) + const outliers = all.filter(isOutlier) + const rest = all.filter((value) => !isOutlier(value)) + const sampled = decimateItems(rest, Math.max(0, MAX_SCATTER_POINTS_PER_GROUP - outliers.length)) + return [...sampled, ...outliers] + }, [dataset, entry.group, whiskers]) if (summary === undefined) return null @@ -227,13 +238,13 @@ function GroupBox({ const jitteredValues = useMemo(() => { const spread = Math.max(0, boxWidth - 14) - const count = Math.max(1, values.length) - return values.map((value, index) => { + const count = Math.max(1, scatterValues.length) + return scatterValues.map((value, index) => { const offset = count === 1 ? 0 : (index / (count - 1) - 0.5) * spread const isOutlier = whiskers !== undefined && (value < whiskers.lower || value > whiskers.upper) return { x: centerX + offset, y: yScale.toPixel(value), outlier: isOutlier, value } }) - }, [values, whiskers, centerX, boxWidth, yScale]) + }, [scatterValues, whiskers, centerX, boxWidth, yScale]) return ( yScale: ReturnType valueField: 'value' | 'normalizedValue' }) { return ( - {dataset.series.map((series, seriesIndex) => { + {series.map((item, seriesIndex) => { const color = seriesColor(seriesIndex) - const positions = series.points + const positions = item.points .map((point) => { const value = point[valueField] if (value === undefined || !Number.isFinite(value)) return null @@ -122,13 +124,13 @@ function SeriesLines({ const pointsAttribute = positions.map((position) => `${position.x},${position.y}`).join(' ') return ( ) })} @@ -237,6 +239,18 @@ export function ExpressionChart({ const dataset = result.dataset const field = result.valueField + // Bound the rendered points for very large datasets: the original series + // stay the source of truth for tooltips/selection; only the drawn marks are + // decimated (see docs/visualization/performance.md). + const renderedSeries = useMemo( + () => + dataset?.series.map((series) => ({ + ...series, + points: [...decimateItems(series.points, MAX_SERIES_POINTS)], + })) ?? [], + [dataset], + ) + const xScale = useMemo( () => createCategoryScale(result.samples, [plot.x0, plot.x0 + plot.width]), [result.samples, plot], @@ -320,9 +334,14 @@ export function ExpressionChart({ yLabel={field === 'value' ? 'Expression value' : 'Normalized value'} formatValue={formatTickValue} /> - + - {dataset.series.map((series, seriesIndex) => { + {renderedSeries.map((series, seriesIndex) => { const color = seriesColor(seriesIndex) return series.points.map((point) => { const value = point[field] diff --git a/apps/web/src/components/scientific/Heatmap.tsx b/apps/web/src/components/scientific/Heatmap.tsx index fb474ec..7aec836 100644 --- a/apps/web/src/components/scientific/Heatmap.tsx +++ b/apps/web/src/components/scientific/Heatmap.tsx @@ -4,6 +4,7 @@ import { useId, useMemo, useState } from 'react' import { VisualizationContainer } from '@/components/visualization/VisualizationContainer' import type { HeatmapDataset } from '@/lib/scientific/advancedTypes' +import { aggregateHeatmap } from '@/lib/scientific/downsample' import { DEFAULT_CHART_HEIGHT, DEFAULT_CHART_MARGINS, plotArea } from '@/lib/scientific/geometry' import type { PlotArea } from '@/lib/scientific/geometry' import { @@ -20,6 +21,8 @@ import { ChartTooltip } from './ChartTooltip' const AXIS_COLOR = '#cbd5e1' const MISSING_FILL = '#eef2f7' +const MAX_HEATMAP_ROWS = 150 +const MAX_HEATMAP_COLS = 150 export interface HeatmapProps { /** View model produced by `useHeatmap`. */ @@ -62,6 +65,19 @@ export function Heatmap({ : `${dataset.rows.length} rows · ${dataset.columns.length} columns` : undefined + // Oversized matrices are block-averaged for rendering so the SVG stays + // bounded; the original dataset remains the source of truth for counts and + // selection (see docs/visualization/performance.md). + const renderData = useMemo( + () => + dataset === undefined + ? undefined + : aggregateHeatmap(dataset, MAX_HEATMAP_ROWS, MAX_HEATMAP_COLS), + [dataset], + ) + const gridData = renderData ?? dataset + const aggregated = gridData !== dataset + return ( - {result.status === 'success' && dataset ? ( + {result.status === 'success' && gridData ? (
- {dataset.rows.length} rows · {dataset.columns.length} columns + {gridData.rows.length} rows · {gridData.columns.length} columns + {aggregated ? ' (block-summarized for display)' : ''}
{hoveredKey !== null && hoveredPosition !== null ? ( ) : null}
- +
) : null} @@ -281,19 +298,24 @@ function HeatmapHoverTooltip({ return } -function HeatmapDetail({ result }: { result: HeatmapResult }) { - const dataset = result.dataset +function HeatmapDetail({ + result, + gridData, +}: { + result: HeatmapResult + gridData: HeatmapDataset +}) { const selectedKey = result.selectedKey const headingId = useId() - if (dataset === undefined || selectedKey === null) return null + if (selectedKey === null) return null const coords = parseHeatmapCellKey(selectedKey) if (coords === undefined) return null - const rowIndex = dataset.rows.indexOf(coords.row) - const columnIndex = dataset.columns.indexOf(coords.column) + const rowIndex = gridData.rows.indexOf(coords.row) + const columnIndex = gridData.columns.indexOf(coords.column) const value = - rowIndex >= 0 && columnIndex >= 0 ? dataset.values[rowIndex]?.[columnIndex] : undefined - const tooltip = heatmapCellTooltip(dataset, coords.row, coords.column, value) + rowIndex >= 0 && columnIndex >= 0 ? gridData.values[rowIndex]?.[columnIndex] : undefined + const tooltip = heatmapCellTooltip(gridData, coords.row, coords.column, value) return (
(dataset === undefined ? [] : decimateItems(dataset.points, MAX_RENDERED_POINTS)), + [dataset], + ) + const significantCount = useMemo( + () => (dataset === undefined ? 0 : highlightCount(dataset, result.thresholds)), + [dataset, result.thresholds], + ) + const xScale = useMemo( () => domains === undefined @@ -118,8 +132,8 @@ export function VolcanoPlot({ {result.status === 'success' && dataset ? (
- {dataset.points.length} features tested · {highlightCount(dataset, result.thresholds)}{' '} - significant at the current thresholds + {dataset.points.length} features tested · {significantCount} significant at the current + thresholds
- {dataset.points.map((point) => { + {renderedPoints.map((point) => { const highlighted = isVolcanoHighlighted(point, result.thresholds) const key = point.identifier const selected = result.selectedKey === key diff --git a/apps/web/src/lib/scientific/distribution.test.ts b/apps/web/src/lib/scientific/distribution.test.ts index 759ba6a..d0d4c6b 100644 --- a/apps/web/src/lib/scientific/distribution.test.ts +++ b/apps/web/src/lib/scientific/distribution.test.ts @@ -9,6 +9,7 @@ import { hasRenderableValues, normalizeDistributionDataset, validateDistributionDataset, + valuesByGroup, valuesForGroup, } from './distribution' @@ -121,3 +122,26 @@ describe('hasRenderableValues / distributionTooltip', () => { expect(tooltip.rows[2]).toEqual({ label: 'Median', value: '2' }) }) }) + +describe('valuesByGroup', () => { + it('groups all values in a single pass, preserving order', () => { + const data = dataset([value('Tumor', 3), value('Normal', 2), value('Tumor', 1)]) + const grouped = valuesByGroup(data) + expect([...grouped.keys()]).toEqual(['Tumor', 'Normal']) + expect(grouped.get('Tumor')).toEqual([3, 1]) + expect(grouped.get('Normal')).toEqual([2]) + }) + + it('matches valuesForGroup for every group', () => { + const data = dataset([value('Tumor', 3), value('Normal', 2), value('Tumor', 1)]) + const grouped = valuesByGroup(data) + for (const group of distributionGroups(data)) { + expect(grouped.get(group)).toEqual(valuesForGroup(data, group)) + } + }) + + it('skips empty group identifiers', () => { + const data = dataset([value('', 1), value('Tumor', 2)]) + expect(valuesByGroup(data).has('')).toBe(false) + }) +}) diff --git a/apps/web/src/lib/scientific/distribution.ts b/apps/web/src/lib/scientific/distribution.ts index b6d789c..4034f81 100644 --- a/apps/web/src/lib/scientific/distribution.ts +++ b/apps/web/src/lib/scientific/distribution.ts @@ -109,11 +109,35 @@ export function distributionGroups(dataset: DistributionDataset): string[] { return [...groups].sort(compareText) } -/** The values belonging to a group, in their stored order. */ +/** + * The values belonging to a group, in their stored order. This scans the + * whole dataset per call; prefer `valuesByGroup` (one pass) when summarizing + * many groups at once. + */ export function valuesForGroup(dataset: DistributionDataset, group: string): number[] { return dataset.values.filter((value) => value.group === group).map((value) => value.value) } +/** + * Groups all dataset values by group name in a single pass, preserving the + * stored order within each group. Used by the chart hook to derive every + * group's statistics without re-scanning the dataset per group (O(values) + * instead of O(groups × values)). + */ +export function valuesByGroup(dataset: DistributionDataset): Map { + const grouped = new Map() + for (const value of dataset.values) { + if (value.group.trim().length === 0) continue + const list = grouped.get(value.group) + if (list === undefined) { + grouped.set(value.group, [value.value]) + } else { + list.push(value.value) + } + } + return grouped +} + /** Summary statistics for a single group, or `undefined` when empty. */ export function groupStatistics( dataset: DistributionDataset, diff --git a/apps/web/src/lib/scientific/downsample.test.ts b/apps/web/src/lib/scientific/downsample.test.ts new file mode 100644 index 0000000..4c82a02 --- /dev/null +++ b/apps/web/src/lib/scientific/downsample.test.ts @@ -0,0 +1,166 @@ +import { describe, expect, it } from 'vitest' + +import type { CoverageBin, HeatmapDataset } from './advancedTypes' +import { aggregateHeatmap, coverageColumns, decimateItems } from './downsample' + +describe('decimateItems', () => { + it('returns the same array when the input already fits', () => { + const input = [1, 2, 3, 4] + expect(decimateItems(input, 10)).toBe(input) + }) + + it('returns an empty sample for a non-positive limit', () => { + expect(decimateItems([1, 2, 3], 0)).toEqual([]) + }) + + it('never exceeds the requested count', () => { + const items = Array.from({ length: 10_000 }, (_, index) => index) + expect(decimateItems(items, 100).length).toBeLessThanOrEqual(100) + }) + + it('always preserves the first and last elements', () => { + const items = Array.from({ length: 500 }, (_, index) => index) + const sampled = decimateItems(items, 50) + expect(sampled[0]).toBe(items[0]) + expect(sampled[sampled.length - 1]).toBe(items[items.length - 1]) + }) + + it('is deterministic for a given input', () => { + const items = Array.from({ length: 1234 }, (_, index) => index) + expect(decimateItems(items, 100)).toEqual(decimateItems(items, 100)) + }) + + it('returns a strict in-order subset when decimating', () => { + const items = Array.from({ length: 300 }, (_, index) => index) + const sampled = [...decimateItems(items, 10)] + expect(sampled.length).toBeLessThan(items.length) + expect(sampled).toEqual([...sampled].sort((a, b) => a - b)) + }) +}) + +describe('coverageColumns', () => { + const toX = (base: number) => (base - 1) * 1 + + it('returns the same bins when the count already fits', () => { + const bins: CoverageBin[] = [ + { chromosome: 'chr1', start: 1, end: 10, coverage: 5 }, + { chromosome: 'chr1', start: 11, end: 20, coverage: 8 }, + ] + expect(coverageColumns(bins, toX, 100, 50)).toBe(bins) + }) + + it('returns the same bins for an empty input', () => { + const bins: CoverageBin[] = [] + expect(coverageColumns(bins, toX, 100, 50)).toBe(bins) + }) + + it('aggregates into at most the requested number of columns', () => { + const bins: CoverageBin[] = Array.from({ length: 5000 }, (_, index) => ({ + chromosome: 'chr1', + start: index * 10 + 1, + end: index * 10 + 10, + coverage: (index % 7) + 1, + })) + const columns = coverageColumns(bins, toX, 200, 100) + expect(columns.length).toBeLessThanOrEqual(100) + }) + + it('preserves the peak coverage within each column', () => { + const plotWidth = 200 + const maxColumns = 50 + const bucket = (base: number) => + Math.min(maxColumns - 1, Math.max(0, Math.floor((toX(base) / plotWidth) * maxColumns))) + const bins: CoverageBin[] = Array.from({ length: 5000 }, (_, index) => ({ + chromosome: 'chr1', + start: index * 2 + 1, + end: index * 2 + 2, + coverage: index, + })) + const columns = coverageColumns(bins, toX, plotWidth, maxColumns) + for (const column of columns) { + const merged = bins.filter((bin) => bucket(bin.start) === bucket(column.start)) + const expectedMax = + merged.length > 0 ? Math.max(...merged.map((bin) => bin.coverage)) : undefined + if (expectedMax !== undefined) { + expect(column.coverage).toBe(expectedMax) + } + } + }) + + it('is deterministic for a given input', () => { + const bins: CoverageBin[] = Array.from({ length: 4000 }, (_, index) => ({ + chromosome: 'chr1', + start: index + 1, + end: index + 1, + coverage: index % 9, + })) + expect(coverageColumns(bins, toX, 300, 80)).toEqual(coverageColumns(bins, toX, 300, 80)) + }) +}) + +describe('aggregateHeatmap', () => { + it('returns the same dataset when the matrix already fits', () => { + const dataset: HeatmapDataset = { + id: 'h', + title: 'Small', + rows: ['r1', 'r2'], + columns: ['c1'], + values: [[1], [2]], + } + expect(aggregateHeatmap(dataset, 10, 10)).toBe(dataset) + }) + + it('bounds the aggregated rows and columns', () => { + const rows = Array.from({ length: 400 }, (_, index) => `r${index}`) + const columns = Array.from({ length: 300 }, (_, index) => `c${index}`) + const values = rows.map(() => columns.map(() => 1)) + const dataset: HeatmapDataset = { id: 'h', title: 'Big', rows, columns, values } + const aggregated = aggregateHeatmap(dataset, 50, 40) + expect(aggregated.rows.length).toBeLessThanOrEqual(50) + expect(aggregated.columns.length).toBeLessThanOrEqual(40) + expect(aggregated.values.length).toBe(aggregated.rows.length) + for (const row of aggregated.values) { + expect(row.length).toBe(aggregated.columns.length) + } + }) + + it('block-averages finite values', () => { + const dataset: HeatmapDataset = { + id: 'h', + title: 'Blocks', + rows: ['r1', 'r2', 'r3', 'r4'], + columns: ['c1', 'c2', 'c3', 'c4'], + values: [ + [1, 2, 5, 6], + [3, 4, 7, 8], + [9, 10, 13, 14], + [11, 12, 15, 16], + ], + } + const aggregated = aggregateHeatmap(dataset, 2, 2) + expect(aggregated.rows).toEqual(['r1', 'r3']) + expect(aggregated.columns).toEqual(['c1', 'c3']) + // Top-left block: mean(1,2,3,4) = 2.5; top-right: mean(5,6,7,8) = 6.5 + expect(aggregated.values[0]?.[0]).toBe(2.5) + expect(aggregated.values[0]?.[1]).toBe(6.5) + // Bottom-left block: mean(9,10,11,12) = 10.5; bottom-right: mean(13..16) = 14.5 + expect(aggregated.values[1]?.[0]).toBe(10.5) + expect(aggregated.values[1]?.[1]).toBe(14.5) + }) + + it('keeps missing blocks missing and averages over finite values only', () => { + const dataset: HeatmapDataset = { + id: 'h', + title: 'Missing', + rows: ['r1', 'r2'], + columns: ['c1', 'c2'], + values: [ + [undefined, undefined], + [1, 3], + ], + } + const aggregated = aggregateHeatmap(dataset, 1, 1) + // Whole matrix collapses to one block; finite values are 1 and 3. + expect(aggregated.values[0]?.[0]).toBe(2) + }) +}) diff --git a/apps/web/src/lib/scientific/downsample.ts b/apps/web/src/lib/scientific/downsample.ts new file mode 100644 index 0000000..87752e2 --- /dev/null +++ b/apps/web/src/lib/scientific/downsample.ts @@ -0,0 +1,180 @@ +/** + * Deterministic downsampling and aggregation for large scientific datasets + * (Phase 6 — Visualization Performance). + * + * All functions are pure, deterministic, and never mutate their input. They + * exist to bound the DOM / path complexity of charts when a dataset is far + * larger than any screen can resolve, while keeping the result scientifically + * meaningful: + * + * - `decimateItems` keeps an evenly-spaced stride sample (first and last + * elements always preserved) so the distribution shape survives. + * - `coverageColumns` aggregates per-bin read depth into at most `maxColumns` + * pixel columns using the **max** coverage within each column. Max-based + * (peak-preserving) binning is the standard for read-depth tracks (e.g. + * IGV-style coverage) because averaging would hide genuine signal peaks. + * - `aggregateHeatmap` block-averages an oversized expression matrix into at + * most `maxRows` × `maxCols` blocks so each rendered cell still represents + * a real measurement (the mean of its block). + * + * Every helper is a no-op (returns equivalent output) when the input already + * fits inside the requested limit, so small/typical datasets render at full + * resolution and existing behavior is unchanged. + * + * See `docs/visualization/performance.md` for the downsampling strategy and + * its limitations. + */ + +import type { CoverageBin, HeatmapDataset } from './advancedTypes' + +/** + * Returns a deterministic, evenly-spaced sample of at most `maxCount` items. + * + * When the input already fits, the items are returned unchanged (same array + * reference). Otherwise one slot is reserved for the final element and the + * remaining budget is sampled evenly across the array, so the result is never + * longer than `maxCount`, always includes the first and last elements (for + * `maxCount >= 2`), and is fully deterministic for a given input. + */ +export function decimateItems(items: readonly T[], maxCount: number): readonly T[] { + if (maxCount <= 0) return [] + if (items.length <= maxCount) return items + if (maxCount === 1) return [items[0]] + const budget = maxCount - 1 + const stride = Math.ceil((items.length - 1) / budget) + const sampled: T[] = [] + for (let index = 0; index < items.length - 1; index += stride) { + sampled.push(items[index]) + } + sampled.push(items[items.length - 1]) + return sampled +} + +/** + * A coverage measurement after pixel-column aggregation: the representative + * interval of the column and its peak (max) coverage. + */ +export interface CoverageColumn { + chromosome: string + /** 1-based inclusive start of the first bin merged into the column. */ + start: number + /** 1-based inclusive end of the last bin merged into the column. */ + end: number + /** Max coverage across the merged bins (peak-preserving). */ + coverage: number +} + +/** + * Aggregates coverage bins into at most `maxColumns` pixel columns using the + * peak (max) coverage per column. Bins are bucketed by their x-pixel position + * (`toX(base)` maps a base position to a pixel; the plot is `plotWidth` pixels + * wide). When the bin count already fits, the input bins are returned unchanged + * (same array reference). Deterministic for a given input. + */ +export function coverageColumns( + bins: readonly CoverageBin[], + toX: (base: number) => number, + plotWidth: number, + maxColumns: number, +): readonly CoverageBin[] { + if (bins.length === 0) return bins + if (bins.length <= maxColumns || maxColumns <= 0 || plotWidth <= 0) return bins + + const columns: CoverageColumn[] = [] + const columnIndex = new Map() + for (const bin of bins) { + const pixel = toX(bin.start) + const raw = Math.floor((pixel / plotWidth) * maxColumns) + const bucket = Math.min(maxColumns - 1, Math.max(0, raw)) + let column = columnIndex.get(bucket) + if (column === undefined) { + column = { + chromosome: bin.chromosome, + start: bin.start, + end: bin.end, + coverage: bin.coverage, + } + columnIndex.set(bucket, column) + columns.push(column) + } else { + if (bin.start < column.start) column.start = bin.start + if (bin.end > column.end) column.end = bin.end + if (bin.coverage > column.coverage) column.coverage = bin.coverage + } + } + columns.sort((left, right) => left.start - right.start) + return columns +} + +/** Shape of the block-summarized heatmap produced by `aggregateHeatmap`. */ +export interface AggregatedHeatmap { + rows: string[] + columns: string[] + values: Array> +} + +/** + * Block-averages a heatmap into at most `maxRows` × `maxCols` blocks. + * + * Each output cell is the mean of the finite values in its block; blocks with + * no finite value stay `undefined` (missing). Row/column identifiers are the + * first identifier of each block, so labels resolve through the original + * dataset. When the matrix already fits, the dataset is returned unchanged + * (same reference). + */ +export function aggregateHeatmap( + dataset: HeatmapDataset, + maxRows: number, + maxCols: number, +): HeatmapDataset { + const rows = dataset.rows + const columns = dataset.columns + if (rows.length <= maxRows && columns.length <= maxCols) return dataset + + const safeRows = Math.max(1, maxRows) + const safeCols = Math.max(1, maxCols) + const rowBlock = Math.max(1, Math.ceil(rows.length / safeRows)) + const colBlock = Math.max(1, Math.ceil(columns.length / safeCols)) + + const aggregatedRows: string[] = [] + const aggregatedCols: string[] = [] + const values: Array> = [] + + for (let r = 0; r < rows.length; r += rowBlock) { + aggregatedRows.push(rows[r]) + } + for (let c = 0; c < columns.length; c += colBlock) { + aggregatedCols.push(columns[c]) + } + + for (let r = 0; r < rows.length; r += rowBlock) { + const rowOut: Array = [] + for (let c = 0; c < columns.length; c += colBlock) { + let sum = 0 + let count = 0 + for (let br = r; br < Math.min(r + rowBlock, rows.length); br += 1) { + const sourceRow = dataset.values[br] + if (sourceRow === undefined) continue + for (let bc = c; bc < Math.min(c + colBlock, columns.length); bc += 1) { + const value = sourceRow[bc] + if (value === undefined || !Number.isFinite(value)) continue + sum += value + count += 1 + } + } + rowOut.push(count > 0 ? sum / count : undefined) + } + values.push(rowOut) + } + + return { + id: dataset.id, + title: dataset.title, + rows: aggregatedRows, + columns: aggregatedCols, + values, + ...(dataset.rowLabels !== undefined ? { rowLabels: { ...dataset.rowLabels } } : {}), + ...(dataset.columnLabels !== undefined ? { columnLabels: { ...dataset.columnLabels } } : {}), + ...(dataset.metadata !== undefined ? { metadata: { ...dataset.metadata } } : {}), + } +} diff --git a/apps/web/src/lib/scientific/useDistributionChart.ts b/apps/web/src/lib/scientific/useDistributionChart.ts index 440e7f5..57661fe 100644 --- a/apps/web/src/lib/scientific/useDistributionChart.ts +++ b/apps/web/src/lib/scientific/useDistributionChart.ts @@ -14,13 +14,9 @@ import type { VisualizationError, VisualizationStatus } from '@/lib/visualizatio import { fetchDistributionDataset } from './advancedApi' import type { DistributionDataset } from './advancedTypes' -import { - distributionGroups, - groupStatistics, - groupWhiskers, - hasRenderableValues, -} from './distribution' +import { distributionGroups, hasRenderableValues, valuesByGroup } from './distribution' import type { SummaryStatistics, Whiskers } from './statistics' +import { boxPlotWhiskers, summarize } from './statistics' import { useChartData } from './useChartData' export interface GroupStatistics { @@ -74,16 +70,26 @@ export function useDistributionChart( const groups = useMemo(() => (data === undefined ? [] : distributionGroups(data)), [data]) + // Single-pass grouping so per-group statistics are not re-scanned from the + // whole dataset for every group (O(values) instead of O(groups × values)). + const groupedValues = useMemo( + () => (data === undefined ? undefined : valuesByGroup(data)), + [data], + ) + const statistics = useMemo( () => - data === undefined + data === undefined || groupedValues === undefined ? [] - : groups.map((group) => ({ - group, - summary: groupStatistics(data, group), - whiskers: groupWhiskers(data, group), - })), - [data, groups], + : groups.map((group) => { + const values = groupedValues.get(group) ?? [] + return { + group, + summary: summarize(values), + whiskers: boxPlotWhiskers(values), + } + }), + [data, groups, groupedValues], ) const valueDomain = useMemo(() => { diff --git a/docs/visualization/README.md b/docs/visualization/README.md index 808d345..897cc2f 100644 --- a/docs/visualization/README.md +++ b/docs/visualization/README.md @@ -4,11 +4,15 @@ This directory documents the GenomeAI visualization platform (Phase 6). ## Status -**Phase 6.8 — Advanced Scientific Charts** is implemented, on top of the -Phase 6.7 scientific charts and the Phase 6.1 foundation. Phase 6.8 adds four -reusable scientific primitives — an expression heatmap, a volcano plot, a -genomic coverage chart, and a statistical distribution chart — built on the -Phase 6.7 chart infrastructure (native scales, tooltips, chart primitives). +**Phase 6.10 — Visualization Performance & Large Dataset Handling** is +implemented, on top of the Phase 6.8 advanced scientific charts and the Phase +6.1 foundation. Phase 6.10 makes the whole platform scale to substantially +larger datasets by bounding the SVG/DOM work (deterministic downsampling, +pixel-column aggregation, heatmap block-averaging), avoiding per-render +recomputation (memoized derivations, single-pass grouping, `React.memo` marks), +and keeping the Genome Browser viewport-scoped — while preserving correctness, +accessibility, and scientific meaning. See +[Performance](./performance.md). | Milestone | Description | Status | |-----------|-------------|--------| @@ -21,7 +25,7 @@ Phase 6.7 chart infrastructure (native scales, tooltips, chart primitives). | 6.7 | Scientific Charts | ✅ Implemented | | 6.8 | Advanced Scientific Charts | ✅ Implemented | | 6.9 | Integrated Research Workspace | 📋 Planned | -| 6.10 | Visualization Performance & Optimization | 📋 Planned | +| 6.10 | Visualization Performance & Optimization | ✅ Implemented | | 6.11 | Visualization Testing & Documentation | 📋 Planned | ## What Phase 6.1 Provides @@ -155,6 +159,27 @@ Phase 6.7 chart infrastructure (native scales, tooltips, chart primitives). normalizers as production. - Demo integrated at `/visualization` (`AdvancedScientificDemo`). +## What Phase 6.10 Provides + +- Visualization performance & large-dataset handling (see + [Performance](./performance.md)) across every Phase 6 module — no new + dependencies, no C++/WebAssembly/WebGPU, no second rendering architecture. +- Pure deterministic downsampling/aggregation + (`lib/scientific/downsample.ts`): stride-based `decimateItems` for points and + scatter (volcano, expression, distribution), **peak-preserving** pixel-column + aggregation for coverage bins, and block-average heatmap aggregation. Every + helper is a no-op below its cap, so typical datasets render at full + resolution. +- Per-render work reduction: memoized derived data (chromosome bins, coverage + columns, rendered point/series sets, highlight counts, scatter samples), + single-pass distribution grouping (`valuesByGroup`), and `React.memo` marks + in the Network Viewer so selection changes do not recompute geometry or + re-render every node/edge. +- The Genome Browser remains viewport-scoped: track loaders fetch only the + settled (debounced) visible interval. +- Deterministic, non-flaky performance tests + (`downsample.test.ts`, distribution grouping tests) and updated docs. + ## Documents | Document | Description | @@ -167,6 +192,7 @@ Phase 6.7 chart infrastructure (native scales, tooltips, chart primitives). | [Network Viewer](network-viewer.md) | Phase 6.6 Biological Network Viewer: scope, design decision, data flow, API, a11y, tests | | [Scientific Charts](scientific-charts.md) | Phase 6.7 Scientific Charts: scope, design decision, data flow, API, a11y, tests | | [Advanced Scientific Charts](advanced-scientific-charts.md) | Phase 6.8 Advanced Scientific Charts: heatmap / volcano / coverage / distribution, data models, API, fixtures, a11y, tests | +| [Performance](performance.md) | Phase 6.10 Visualization performance & large-dataset handling: data flow, strategies, downsampling limitations, a11y, testing | | [Roadmap](roadmap.md) | Detailed phase tracking and future work | ## Technology Notes diff --git a/docs/visualization/performance.md b/docs/visualization/performance.md new file mode 100644 index 0000000..ec458bd --- /dev/null +++ b/docs/visualization/performance.md @@ -0,0 +1,173 @@ +# Visualization Performance & Large Dataset Handling + +This document describes how the GenomeAI visualization platform (Phase 6) +stays responsive on large datasets. It covers the data flow, the measured hot +spots, the strategies used to bound work, and their limitations. + +## Architecture and data flow + +The platform is a React/SVG stack — no C++, WebAssembly, WebGPU, or a second +rendering architecture. Every visualization follows the same layered flow: + +1. **Load once per dataset id.** `useVisualizationData` owns the + loading / empty / success / error lifecycle with `AbortSignal` cancellation + and stale-response protection. The scientific chart hooks wrap it in + `useChartData`; network / protein / genome hooks use it directly. +2. **Derive pure view models.** Pure modules under `lib/scientific`, + `lib/network`, `lib/genome`, and `lib/protein` turn raw data into render- + ready structures (normalization, domains, statistics, layout, scales). + These are memoized in the view-model hooks so they run once per dataset, not + per render. +3. **Render to SVG.** Thin components project the view model through pure + geometry helpers and emit SVG. Pan / zoom / filter / selection are + client-side state; they re-render the SVG but never refetch data (except the + Genome Browser, whose per-track loaders are region-scoped). + +## Where the cost actually is + +Reading the implementation identified these hot spots before any change was +made: + +| Area | Hot spot | Why it is expensive at large scale | +|------|----------|------------------------------------| +| Network Viewer | Every `EdgeElement`/`NodeElement` recomputed screen geometry and re-rendered on *any* `result` change | A selection or filter change re-created the `result` object, so every node/edge re-ran projection math and re-rendered | +| Volcano / Expression | One SVG `` + 2–3 circles per point; `isVolcanoHighlighted`, scale projection, and highlight counting re-run per point per render | Thousands of points → thousands of DOM nodes and O(n) per render | +| Heatmap | One `` (plus a string cell key) per matrix cell | A 1000×1000 matrix is a million DOM nodes | +| Coverage | One hit-target `` per bin plus a full path rebuild and a chromosome `.filter` per render | A dense chromosome is tens of thousands of bins | +| Distribution | Per-group statistics re-scanned the whole dataset (O(groups × values)); one `` per value | Many groups × many values, or huge groups, explode circles | +| Genome Browser | Track loaders are already region-scoped | Already correct; see below | + +## Strategies + +### 1. Bound the DOM with deterministic downsampling + +`lib/scientific/downsample.ts` provides pure, deterministic helpers. Each is a +**no-op (pass-through) below its cap**, so small and typical datasets render at +full resolution and behavior is unchanged; only oversized datasets are reduced. + +- `decimateItems(items, maxCount)` — evenly-spaced stride sample, always + keeping the first and last elements, never exceeding `maxCount`. Used by the + volcano plot (`MAX_RENDERED_POINTS = 2000`), the expression chart + (`MAX_SERIES_POINTS = 1000`), and the distribution scatter + (`MAX_SCATTER_POINTS_PER_GROUP = 1000`). +- `coverageColumns(bins, toX, plotWidth, maxColumns)` — **peak-preserving** + (max coverage) pixel-column aggregation for coverage bins + (`MAX_COVERAGE_COLUMNS = 2000`). Max-based binning is the standard for + read-depth tracks (IGV-style) because averaging would hide genuine peaks. +- `aggregateHeatmap(dataset, maxRows, maxCols)` — block-average of an oversized + expression matrix (`MAX_HEATMAP_ROWS/COLS = 150`). Each rendered cell is the + mean of its block, so it still represents a real measurement; all-missing + blocks stay missing. + +Crucially, the **full dataset remains the source of truth** for tooltips, +selection, summaries, and detail panels. Only the marks *drawn* are decimated; +the underlying interaction reads the full data, so hover/selection never +silently reports a decimated point's neighbors. + +### 2. Preserve scientific meaning + +Downsampling never discards signal silently: + +- Coverage aggregation keeps the **max** (peak) per pixel column. +- Distribution scatter keeps **all outliers** and stride-samples the rest. +- Heatmap aggregation reports block means (with missing preserved), and the + chart adds a "(block-summarized for display)" note in the summary line when + aggregation is active. +- Volcano decimation spans the whole sorted feature list evenly, so the + effect-size distribution shape survives. + +### 3. Avoid redoing work on every render + +- **Memoized derived data.** Chromosome-filtered bins, aggregated coverage + columns, rendered point/series sets, and per-group scatter samples are + `useMemo`-derived so pan/zoom/hover re-renders reuse them. +- **Single-pass grouping.** `valuesByGroup` in `lib/scientific/distribution.ts` + groups all distribution values in one pass; `useDistributionChart` derives + every group's statistics from that map instead of scanning the dataset once + per group (O(values) instead of O(groups × values)). +- **`React.memo` for large mark sets.** `NetworkViewer`'s `EdgeElement` and + `NodeElement` are memoized and receive primitive/stable props (the node/edge, + the memoized layout, the viewport, a `selected` boolean, and a stable + callback). A selection change no longer recomputes projection geometry or + re-renders every node/edge — only the viewport (pan/zoom) or the filter + (node/edge set) forces a full re-render, which is unavoidable. +- **Memoized highlight counts.** The volcano summary's "significant at current + thresholds" count is computed once per thresholds change instead of on every + render. + +### 4. Viewport-aware data loading (Genome Browser) + +The Genome Browser already fetches **only the visible region**: each track +loader receives the settled (debounced) `GenomicInterval` and returns just the +features intersecting it (`GenomeTrackLoader` in +`lib/genome/useGenomeBrowser.ts`). Rapid pan/zoom is debounced (default 300 ms) +so navigation never fires a request per animation frame. The +`featuresInViewport` / `variantsInViewport` filters in the track components are +a defensive second clipping of already-region-scoped data. This is the primary +large-data strategy for the browser; the backend search API remains the filter +authority when it becomes reachable. + +### 5. Caching policy + +Caching is limited to memoization of derivations and rendered marks (above). +There is deliberately **no new cache abstraction and no cross-mount cache**: the +existing `useVisualizationData` lifecycle already re-fetches only when the +dataset id changes, and adding an app-level cache would introduce stale-data +risk without a measured benefit. If a later milestone needs one, it must define +explicit invalidation and reuse the existing hooks rather than adding a second +loading path. + +## Downsampling limitations + +These are intentional and documented so future work can revisit them: + +- **Coverage peak-preservation** shows local maxima; a *valley* within a pixel + column that is narrower than one pixel can be hidden (the column shows the + peak). This matches standard read-depth rendering, but a per-pixel + min/max band is a possible future improvement. +- **Volcano/expression decimation** renders a stride sample; two features that + land on the same mark are not distinguished (they share a pixel anyway). + Tooltips and selection still resolve the exact underlying feature. +- **Heatmap block-averaging** changes the *granularity* of what is shown when + the matrix exceeds the cap; the summary line indicates this. Selection of an + aggregated block is valid within the rendered (aggregated) matrix. +- **Distribution scatter capping** keeps outliers but reduces the visual + density of non-outlier values in oversized groups; the box/whisker summary + (which is full-resolution) remains the statistical authority. + +## Accessibility + +Large datasets may reduce visual detail but must not regress accessibility: + +- Decimated marks keep the same `role="button"`, `aria-label`, `aria-pressed`, + keyboard (Enter/Space), and focus-ring behavior as full-resolution marks. +- Tooltips and detail panels read the **full** dataset, so hover/selection is + as accurate for large datasets as for small ones. +- Summaries and `aria-label`s report the underlying counts (full dataset), with + an explicit "(block-summarized for display)" note when heatmap aggregation is + active. + +## Testing + +Performance work is tested by algorithmic behavior, never by wall-clock timing: + +- `lib/scientific/downsample.test.ts` — decimation bounds (never exceeds the + cap, first/last preserved), determinism, coverage column peak preservation, + and heatmap block-mean correctness (including missing-value handling). +- `lib/scientific/distribution.test.ts` — `valuesByGroup` grouping matches + `valuesForGroup` for every group. +- All existing component tests pass unchanged, confirming the refactors are + behavior-preserving below the caps. + +## Future options + +If datasets outgrow the current caps, the following are candidates (each with a +measured requirement): + +- **Per-pixel min/max coverage bands** for coverage valleys. +- **Canvas/WebGL fallback** for the largest mark sets — note this is a second + rendering architecture and is intentionally deferred (see the + [README](README.md#technology-notes) technology constraints). +- **Worker-based statistics** for very large distributions. +- **Server-side region queries** once the Phase 5 API exposes range-based + endpoints, moving filtering out of the client entirely. \ No newline at end of file diff --git a/docs/visualization/roadmap.md b/docs/visualization/roadmap.md index 37a303b..bb2b59f 100644 --- a/docs/visualization/roadmap.md +++ b/docs/visualization/roadmap.md @@ -4,7 +4,51 @@ Tracks the Phase 6 visualization platform milestones. See [Phase 6 of the project ROADMAP]() for the authoritative milestone list. -## Current Milestone: 6.8 — Advanced Scientific Charts ✅ +## Current Milestone: 6.10 — Visualization Performance & Large Dataset Handling ✅ + +Implemented on top of 6.8. + +Delivered: + +- Performance analysis across the Phase 6 platform identifying the real hot + spots before any change (per-mark geometry on every network render, one SVG + node per cell/point/bin, O(groups × values) distribution statistics, per- + render path/filter rebuilds) — see [Performance](performance.md) +- Pure deterministic downsampling (`lib/scientific/downsample.ts`): + `decimateItems` (stride sample, first/last preserved, hard bound), + `coverageColumns` (peak-preserving pixel-column aggregation for coverage), + `aggregateHeatmap` (block-average for oversized matrices) — all no-ops below + their caps so typical datasets render unchanged +- Component bounds: volcano (`MAX_RENDERED_POINTS = 2000`), expression chart + (`MAX_SERIES_POINTS = 1000`), distribution scatter (`MAX_SCATTER_POINTS_PER_GROUP + = 1000`, outliers always kept), coverage (`MAX_COVERAGE_COLUMNS = 2000`), + heatmap (`MAX_HEATMAP_ROWS/COLS = 150` with a "(block-summarized)" note) +- Per-render work reduction: memoized chromosome bins / coverage columns / + rendered point+series sets / highlight counts / scatter samples; single-pass + `valuesByGroup` grouping in `useDistributionChart` (O(values) not + O(groups × values)); `React.memo` Network Viewer marks with primitive props so + selection/filter changes skip most re-renders +- Genome Browser stays viewport-scoped (per-track loaders fetch only the + settled debounced interval) +- Full data remains the source of truth for tooltips/selection/summaries — + downsampling only affects the marks drawn, never the interaction +- Deterministic, non-flaky performance tests (`downsample.test.ts`, distribution + grouping tests) and docs (see [Performance](performance.md)) + +Constraints honored: + +- No C++, WebAssembly, WebGPU, Three.js, Cytoscape.js, D3.js, or a second + rendering architecture; no new runtime dependencies +- No new cache abstraction; memoization is scoped to justified derivations and + the existing `useVisualizationData` lifecycle remains the data authority +- Downsampling is documented, deterministic, and never silently discards signal + (peaks, outliers, block means preserved); accessibility (labels, keyboard, + aria-pressed, focus rings) is unchanged for decimated marks +- Phase 5 search untouched + +## Previous milestones + +### 6.8 — Advanced Scientific Charts ✅ Implemented on top of 6.7. @@ -304,6 +348,5 @@ Constraints honored: | # | Milestone | Notes | |---|-----------|-------| | 6.9 | Integrated Research Workspace | Assembles 6.5–6.8 into a UI | -| 6.10 | Visualization Performance & Optimization | Virtualization / density rendering for large data | | 6.11 | Visualization Testing & Documentation | Stabilization + docs pass | | 6.12 | Molecular Structure Viewer (3D) | 3D protein structures; Three.js only if 3D is truly required | \ No newline at end of file