From 99304afd770acf77c1146442a01c8667d7a00add Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Tue, 25 Aug 2026 00:03:48 -0700 Subject: [PATCH 01/19] add `SlotMap` --- src/data_manager/SlotMap.ts | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 src/data_manager/SlotMap.ts diff --git a/src/data_manager/SlotMap.ts b/src/data_manager/SlotMap.ts new file mode 100644 index 00000000..fa26166e --- /dev/null +++ b/src/data_manager/SlotMap.ts @@ -0,0 +1,53 @@ +const NEXT_FREE = Symbol("SlotMap.nextFree"); + +type Slot = T | { [NEXT_FREE]: number }; + +const isFreeSlot = (slot: Slot): slot is { [NEXT_FREE]: number } => { + return slot !== undefined && slot !== null && Object.hasOwn(slot, NEXT_FREE); +}; + +export default class SlotMap { + private slots: Slot[] = []; + private head = 0; + + public insert(value: T): number { + if (this.head < this.slots.length) { + const freeSlot = this.slots[this.head]; + if (!isFreeSlot(freeSlot)) { + // should never happen + throw new Error(`SlotMap: free list head (${this.head}) points to occupied slot`); + } + + const index = this.head; + this.head = freeSlot[NEXT_FREE]; + this.slots[index] = value; + return index; + } + + const index = this.slots.length; + this.slots.push(value); + this.head = this.slots.length; + return index; + } + + public get(index: number): T | undefined { + if (index >= this.slots.length) { + return undefined; + } + const slot = this.slots[index]; + if (isFreeSlot(slot)) { + return undefined; + } + return slot; + } + + public remove(index: number): T | undefined { + const value = this.get(index); + if (value === undefined) { + return undefined; + } + this.slots[index] = { [NEXT_FREE]: this.head }; + this.head = index; + return value; + } +} From 43513a93354c0619a93f1acc10cba046d61831bc Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Tue, 8 Sep 2026 08:06:43 -0700 Subject: [PATCH 02/19] give `SlotMap` a `size` --- src/data_manager/SlotMap.ts | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/data_manager/SlotMap.ts b/src/data_manager/SlotMap.ts index fa26166e..cf2703ed 100644 --- a/src/data_manager/SlotMap.ts +++ b/src/data_manager/SlotMap.ts @@ -9,8 +9,15 @@ const isFreeSlot = (slot: Slot): slot is { [NEXT_FREE]: number } => { export default class SlotMap { private slots: Slot[] = []; private head = 0; + private count = 0; + + public get size(): number { + return this.count; + } public insert(value: T): number { + this.count += 1; + if (this.head < this.slots.length) { const freeSlot = this.slots[this.head]; if (!isFreeSlot(freeSlot)) { @@ -48,6 +55,7 @@ export default class SlotMap { } this.slots[index] = { [NEXT_FREE]: this.head }; this.head = index; + this.count -= 1; return value; } } From 0e33216d126b3133dd5918d205c4efa5f7ca4c74 Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Tue, 8 Sep 2026 08:54:20 -0700 Subject: [PATCH 03/19] handle chunks with multiple channels/times --- src/data_manager/data_manager.ts | 142 ++++++++++++-------- src/data_manager/sources/ChunkSource.ts | 29 ++++ src/data_manager/sources/ome_zarr_source.ts | 44 ++++-- src/data_manager/types.ts | 8 +- 4 files changed, 156 insertions(+), 67 deletions(-) create mode 100644 src/data_manager/sources/ChunkSource.ts diff --git a/src/data_manager/data_manager.ts b/src/data_manager/data_manager.ts index 8f08de53..4bd60794 100644 --- a/src/data_manager/data_manager.ts +++ b/src/data_manager/data_manager.ts @@ -1,4 +1,4 @@ -import type { ChunkId, ChunkPriority, ChunkEntry, DataManagerLimits, LocalChunkId, Chunk } from "./types.js"; +import type { ChunkId, ChunkPriority, ChunkEntry, DataManagerLimits, Chunk } from "./types.js"; import { chunkIdToString, ChunkState, @@ -14,21 +14,13 @@ import { } from "./types.js"; import type { NumberType, TypedArray } from "../types.js"; import PriorityQueue from "./priority_queue.js"; -import { VolumeDims } from "../VolumeDims.js"; import type { DeviceInterface } from "./device_interface.js"; import { dataTypeToSize } from "./utils.js"; +import { ChunkSource, type ExtVolumeDims } from "./sources/ChunkSource.js"; +import SlotMap from "./SlotMap.js"; const SUBSCRIBER_ID = Symbol("DataManager.subscriberId"); -// TODO not modifying original `VolumeDims` for compatibility, but at some point this will either need to be -// incorporated into `VolumeDims` or replaced with an entirely new type -export type ExtVolumeDims = VolumeDims & { chunkShape: [number, number, number, number, number] }; - -export interface IChunkSource { - getDims(): ExtVolumeDims[]; - getChunk(id: LocalChunkId, signal?: AbortSignal): Promise>; -} - export interface IDataSubscriber { [SUBSCRIBER_ID]?: number; onChunkLoaded?: (id: ChunkId, chunk: Chunk) => void; @@ -36,12 +28,16 @@ export interface IDataSubscriber { // TODO events for when chunks are evicted? } -// TODO move to types file? along with above interfaces...? type SourceEntry = { - source: IChunkSource; + source: ChunkSource; subscribers: IDataSubscriber[]; }; +type RequestEntry = { + chunkKeys: string[]; + controller: AbortController; +}; + const swapRemove = (arr: T[], index: number) => { if (index < 0) { return; @@ -77,10 +73,10 @@ export default class DataManager { * * This could be stored in `chunks`, but requests don't necessarily correspond one-to-one with chunks. */ - private requests = new Map(); - /** The amount of chunk data currently cached in memory, in bytes */ + private requests = new SlotMap(); + /** The amount of chunk data currently cached in memory, in bytes. */ private memorySize = 0; - /** The amount of GPU texture memory managed by this class, in bytes */ + /** The amount of GPU texture memory managed by this class, in bytes. */ private deviceSize = 0; /** A counter for assigning chunks a priority at the `RECENT` level. */ private recentCounter = 0; @@ -99,6 +95,8 @@ export default class DataManager { }); } + // MARK: Helpers + /** Gets the id of data subscriber `subscriber`, assigning it one if it doesn't have one. */ private getIdForSubscriber(subscriber: IDataSubscriber): number { if (subscriber[SUBSCRIBER_ID] === undefined) { @@ -113,7 +111,7 @@ export default class DataManager { * * Assumes that the chunk is not in any queues that don't match its state. */ - private updateChunkInQueue(key: string, entry: ChunkEntry) { + private updateChunkInQueue(key: string, entry: ChunkEntry, noAbort = false) { switch (entry.data.state) { case ChunkState.QUEUED: this.queues.load.insert(key, entry.memoryPriority); @@ -126,7 +124,7 @@ export default class DataManager { this.queues.deviceEvict.insert(key, entry.devicePriority); break; case ChunkState.LOADING: - if (entry.memoryPriority.level === ChunkPriorityLevel.RECENT) { + if (entry.memoryPriority.level === ChunkPriorityLevel.RECENT && !noAbort) { // TODO cancel request here } break; @@ -162,6 +160,24 @@ export default class DataManager { } } + /** + * Adds an entry for a chunk that has no requests. + * + * If we're adding an entry for a chunk that no one asked for, something at least a little unexpected has happened. + */ + private insertChunkUnprioritized(key: string, data: ChunkEntry["data"]): ChunkEntry { + const entry = { + data, + subscriberPriorities: [], + memoryPriority: { level: ChunkPriorityLevel.RECENT, score: this.recentCounter }, + devicePriority: getMinChunkPriority(), + }; + this.chunks.set(key, entry); + this.recentCounter += 1; + this.updateChunkInQueue(key, entry, true); + return entry; + } + private getChunkDims(chunkId: ChunkId): { x: number; y: number; z: number; dataType: NumberType } | undefined { const sourceEntry = this.sources[chunkId.source]; if (sourceEntry === undefined) { @@ -186,6 +202,7 @@ export default class DataManager { } private estimateChunkSize(chunkId: ChunkId): number { + // TODO account for edge chunks const dims = this.getChunkDims(chunkId); if (dims === undefined) { return 0; @@ -194,6 +211,8 @@ export default class DataManager { return x * y * z * dataTypeToSize[dataType]; } + // MARK: Update cycle + /** * Resolves which chunks should be uploaded to the GPU, and which should be evicted from it. * @@ -370,47 +389,63 @@ export default class DataManager { return; } let [requestPriority, requestKey] = nextRequest; - let requestId = stringToChunkId(requestKey); + let chunkId = stringToChunkId(requestKey); const nextEvictPriority = this.queues.evict.peek()?.[0] ?? getMinChunkPriority(); while ( // keep submitting requests while concurrency is available and... this.requests.size < requestLimitForPriority(this.limits, requestPriority) && // ...either space will be available or this chunk has higher priority than an already cached chunk - (this.memorySize + this.estimateChunkSize(requestId) < this.limits.size || + (this.memorySize + this.estimateChunkSize(chunkId) < this.limits.size || comparePriority(requestPriority, nextEvictPriority) < 0) ) { - this.queues.load.pop(); - + // Ignore requests at the `RECENT` level. That's supposed to be for chunks that are already loaded! if (requestPriority.level === ChunkPriorityLevel.RECENT) { - // Ignore requests at the `RECENT` level. That's supposed to be for chunks that are already loaded! + this.queues.load.pop(); this.chunks.delete(requestKey); continue; } - const chunkEntry = this.chunks.get(requestKey); - if (chunkEntry === undefined) { - continue; - } - chunkEntry.data = { state: ChunkState.LOADING }; - - const sourceEntry = this.sources[requestId.source]; + // Get the source that this chunk comes from + const sourceId = chunkId.source; + const sourceEntry = this.sources[sourceId]; if (sourceEntry === undefined) { - console.error(`chunk ${requestKey} queued for load with invalid source id ${requestId.source}`); + console.error(`chunk ${requestKey} queued for load with invalid source id ${sourceId}`); + this.queues.load.pop(); this.chunks.delete(requestKey); continue; } - const abortController = new AbortController(); - this.requests.set(requestKey, abortController); + const { source } = sourceEntry; + + // Determine which chunks will be fetched with this request, and set up bookkeeping for the request + // Accounts for sources that store multiple chunks at the same storage id + const storageId = source.chunkIdToStorageId(chunkId); + const chunkIdsAtKey = source.storageIdToChunkIds(storageId); + const chunkKeys = chunkIdsAtKey.map((id) => chunkIdToString({ ...id, source: sourceId })); + const controller = new AbortController(); + const requestId = this.requests.insert({ chunkKeys, controller }); + + for (const key of chunkKeys) { + this.queues.load.remove(key); + const entry = this.chunks.get(key); + const data = { state: ChunkState.LOADING as const, requestId }; + if (entry === undefined) { + this.insertChunkUnprioritized(key, data); + } else if (entry.data.state === ChunkState.QUEUED) { + entry.data = data; + } + } - // Make a stable by-reference copy of `requestId` for the closures below - const requestedId = requestId; + // Request the storage key sourceEntry.source - .getChunk(requestedId, abortController.signal) - .then((data) => this.onChunkLoad(requestedId, data)) + .getKey(storageId, controller.signal) + .then((chunks) => { + this.requests.remove(requestId); + chunks.map((chunk) => this.onChunkLoad(sourceId, chunk)); + }) .catch(() => { - this.chunks.delete(requestKey); - this.requests.delete(requestKey); + this.requests.remove(requestId); + chunkKeys.map((key) => this.chunks.delete(key)); }); const nextRequest = this.queues.load.peek(); @@ -418,7 +453,7 @@ export default class DataManager { return; } [requestPriority, requestKey] = nextRequest; - requestId = stringToChunkId(requestKey); + chunkId = stringToChunkId(requestKey); } } @@ -437,32 +472,25 @@ export default class DataManager { this.submitRequests(); } - private onChunkLoad(id: ChunkId, chunk: Chunk) { - const key = chunkIdToString(id); + private onChunkLoad(source: number, chunk: Chunk) { + const globalId = { ...chunk.id, source }; + const key = chunkIdToString(globalId); const { data: memory, dtype } = chunk; - this.requests.delete(key); if (memory.byteLength > this.limits.size) { console.error(`received chunk ${key} which is larger than the cache limit`); return; } - const sourceEntry = this.sources[id.source]; + const sourceEntry = this.sources[source]; if (sourceEntry === undefined) { - console.error(`received chunk ${key} with invalid source id ${id.source}`); + console.error(`received chunk ${key} with invalid source id ${source}`); return; } const data = { state: ChunkState.MEMORY as const, memory, dtype }; let chunkEntry = this.chunks.get(key); if (chunkEntry === undefined) { - console.error(`received chunk ${key} without data manager entry`); - chunkEntry = { - data, - subscriberPriorities: [], - memoryPriority: { level: ChunkPriorityLevel.RECENT, score: this.recentCounter }, - devicePriority: { level: ChunkPriorityLevel.RECENT, score: this.recentCounter }, - }; - this.chunks.set(key, chunkEntry); - this.recentCounter += 1; + console.warn(`received chunk ${key} without data manager entry`); + chunkEntry = this.insertChunkUnprioritized(key, data); } else { chunkEntry.data = data; } @@ -471,11 +499,13 @@ export default class DataManager { this.queues.evict.insert(key, chunkEntry.memoryPriority); this.memorySize += memory.byteLength; - sourceEntry.subscribers.forEach((s) => s.onChunkLoaded?.(id, chunk)); + sourceEntry.subscribers.forEach((s) => s.onChunkLoaded?.(globalId, chunk)); this.update(); } + // MARK: Public interface + /** * Declares that subscriber `subscriber` requires chunk `chunkId` at priority level `priority`. * @@ -562,7 +592,7 @@ export default class DataManager { * * Returns the ID that will be used to identify this source in chunk requests and event subscriptions. */ - addSource(source: IChunkSource): number { + addSource(source: ChunkSource): number { const id = this.sources.length; this.sources.push({ source, subscribers: [] }); return id; diff --git a/src/data_manager/sources/ChunkSource.ts b/src/data_manager/sources/ChunkSource.ts new file mode 100644 index 00000000..7b6c02d7 --- /dev/null +++ b/src/data_manager/sources/ChunkSource.ts @@ -0,0 +1,29 @@ +import type { NumberType } from "../../types.js"; +import type { VolumeDims } from "../../VolumeDims.js"; +import { type Chunk, type LocalChunkId } from "../types.js"; + +// TODO not modifying original `VolumeDims` for compatibility, but at some point this will either need to be +// incorporated into `VolumeDims` or replaced with an entirely new type +export type ExtVolumeDims = VolumeDims & { chunkShape: [number, number, number, number, number] }; + +export abstract class ChunkSource { + /** + * Maps a chunk `id` to the *storage key* that contains it, and all chunk ids contained in that key. + * + * In most cases, a chunk id and its storage key are the same. However, `DataManager` requires that chunks extend + * over exactly one channel and exactly one time point, and some data formats may allow multiple channels and/or time + * points to be stored at the same storage key. + */ + chunkIdToStorageId(id: LocalChunkId): LocalChunkId { + return id; + } + + /** Maps a storage key to a list of the chunk ids it contains. */ + storageIdToChunkIds(id: LocalChunkId): LocalChunkId[] { + return [id]; + } + + abstract getDims(): ExtVolumeDims[]; + + abstract getKey(key: LocalChunkId, signal?: AbortSignal): Promise[]>; +} diff --git a/src/data_manager/sources/ome_zarr_source.ts b/src/data_manager/sources/ome_zarr_source.ts index ceb2ccd0..88431166 100644 --- a/src/data_manager/sources/ome_zarr_source.ts +++ b/src/data_manager/sources/ome_zarr_source.ts @@ -1,6 +1,6 @@ import * as zarr from "zarrita"; -import type { ExtVolumeDims, IChunkSource } from "../data_manager.js"; +import { type ExtVolumeDims, ChunkSource } from "./ChunkSource.js"; import { assertMetadataHasMultiscales, toOMEZarrMetaV4, @@ -15,13 +15,15 @@ import type { NumberType, TypedArray } from "../../types.js"; const PLACEHOLDER_NAME = "zarr source"; const PLACEHOLDER_SCENE_INDEX = 0; -export class OMEZarrSource implements IChunkSource { +export class OMEZarrSource extends ChunkSource { private constructor( private scaleLevels: zarr.Array[], private multiscaleMetadata: OMEMultiscale, private omeroMetadata: OmeroTransitionalMetadata | undefined, private axesTCZYX: [number, number, number, number, number] - ) {} + ) { + super(); + } static async new(url: string): Promise { const store = new zarr.FetchStore(url); @@ -57,6 +59,32 @@ export class OMEZarrSource implements IChunkSource { return [spaceUnitSymbol, timeUnitSymbol]; } + chunkIdToStorageId({ multiscale, tczyx: [t, c, z, y, x] }: LocalChunkId): LocalChunkId { + const level = this.scaleLevels[multiscale]; + const [shapeT, shapeC] = orderByTCZYX(level.chunks, this.axesTCZYX, 1); + const keyT = Math.floor(t / shapeT); + const keyC = Math.floor(c / shapeC); + return { multiscale, tczyx: [keyT, keyC, z, y, x] }; + } + + storageIdToChunkIds({ multiscale, tczyx: [t, c, z, y, x] }: LocalChunkId): LocalChunkId[] { + const level = this.scaleLevels[multiscale]; + const [shapeT, shapeC] = orderByTCZYX(level.chunks, this.axesTCZYX, 1); + + const minT = t * shapeT; + const maxT = minT + shapeT; + const minC = c * shapeC; + const maxC = minC + shapeC; + + const chunksInKey: LocalChunkId[] = []; + for (let chunkT = minT; chunkT < maxT; chunkT++) { + for (let chunkC = minC; chunkC < maxC; chunkC++) { + chunksInKey.push({ multiscale, tczyx: [chunkT, chunkC, z, y, x] }); + } + } + return chunksInKey; + } + getDims(): ExtVolumeDims[] { const [spaceUnit, timeUnit] = this.getUnitSymbols(); return this.scaleLevels.map((level, i) => { @@ -72,10 +100,10 @@ export class OMEZarrSource implements IChunkSource { }); } - async getChunk(chunkId: LocalChunkId): Promise> { - const multiscale = this.scaleLevels[chunkId.multiscale]; - const coords = orderByDimension(chunkId.tczyx, this.axesTCZYX); - const { data } = await multiscale.getChunk(coords); - return { data: data as TypedArray, dtype: multiscale.dtype }; + async getKey(id: LocalChunkId, signal?: AbortSignal): Promise[]> { + const multiscale = this.scaleLevels[id.multiscale]; + const coords = orderByDimension(id.tczyx, this.axesTCZYX); + const { data } = await multiscale.getChunk(coords, { signal }); + return [{ data: data as TypedArray, dtype: multiscale.dtype, id }]; } } diff --git a/src/data_manager/types.ts b/src/data_manager/types.ts index a20dfb7e..0db4339c 100644 --- a/src/data_manager/types.ts +++ b/src/data_manager/types.ts @@ -112,8 +112,9 @@ export const enum ChunkState { WORKER = "worker", } -export type ChunkData = - | { state: ChunkState.QUEUED | ChunkState.WORKER | ChunkState.LOADING } +type ChunkEntryData = + | { state: ChunkState.QUEUED | ChunkState.WORKER } + | { state: ChunkState.LOADING; requestId: number } | { state: ChunkState.MEMORY; memory: TypedArray; dtype: NumberType } | { state: ChunkState.DEVICE; memory: TypedArray; dtype: NumberType; texture: Tex }; @@ -126,7 +127,7 @@ export type SubscriberPriority = { /** `DataManager`-internal type containing full state of a single chunk. */ export type ChunkEntry = { /** The current lifecycle state of a chunk, and the data associated with it. */ - data: ChunkData; + data: ChunkEntryData; /** A record of every request for this chunk. */ subscriberPriorities: SubscriberPriority[]; /** Priority at which the chunk is required in memory. Derived from `subscriberPriorities` on request add/remove. */ @@ -159,6 +160,7 @@ export const stringToChunkId = (id: string): ChunkId => { }; export type Chunk = { + id: LocalChunkId; data: TypedArray; dtype: T; }; From f4eec6a0a53fa2cbc9b03adf32de3a0e3a9a5c95 Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Tue, 8 Sep 2026 09:02:47 -0700 Subject: [PATCH 04/19] make subscriber ids global --- src/data_manager/data_manager.ts | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/data_manager/data_manager.ts b/src/data_manager/data_manager.ts index 4bd60794..7147d69e 100644 --- a/src/data_manager/data_manager.ts +++ b/src/data_manager/data_manager.ts @@ -28,6 +28,18 @@ export interface IDataSubscriber { // TODO events for when chunks are evicted? } +// Subscriber IDs increment globally, like timeout IDs +let subscriberCount = 0; + +// eslint-disable-next-line @typescript-eslint/no-explicit-any +const getIdForSubscriber = (subscriber: IDataSubscriber): number => { + if (subscriber[SUBSCRIBER_ID] === undefined) { + subscriber[SUBSCRIBER_ID] = subscriberCount; + subscriberCount += 1; + } + return subscriber[SUBSCRIBER_ID]; +}; + type SourceEntry = { source: ChunkSource; subscribers: IDataSubscriber[]; @@ -80,8 +92,6 @@ export default class DataManager { private deviceSize = 0; /** A counter for assigning chunks a priority at the `RECENT` level. */ private recentCounter = 0; - /** A counter for assigning subscriber IDs. */ - private subscriberCount = 0; public limits: DataManagerLimits; @@ -97,15 +107,6 @@ export default class DataManager { // MARK: Helpers - /** Gets the id of data subscriber `subscriber`, assigning it one if it doesn't have one. */ - private getIdForSubscriber(subscriber: IDataSubscriber): number { - if (subscriber[SUBSCRIBER_ID] === undefined) { - subscriber[SUBSCRIBER_ID] = this.subscriberCount; - this.subscriberCount += 1; - } - return subscriber[SUBSCRIBER_ID]; - } - /** * Inserts a chunk into the appropriate queue, or updates its queue position. * @@ -513,7 +514,7 @@ export default class DataManager { * submitted until the next call to `update`. */ addChunkRequest(subscriber: IDataSubscriber, chunkId: ChunkId, priority: ChunkPriority, device: boolean) { - const subscriberId = this.getIdForSubscriber(subscriber); + const subscriberId = getIdForSubscriber(subscriber); const chunkIdString = chunkIdToString(chunkId); const chunkEntry = this.chunks.get(chunkIdString); @@ -546,7 +547,7 @@ export default class DataManager { * The `DataManager` will not respond to this change until the next call to `update`. */ removeChunkRequest(subscriber: IDataSubscriber, chunkId: ChunkId) { - const subscriberId = this.getIdForSubscriber(subscriber); + const subscriberId = getIdForSubscriber(subscriber); const chunkKey = chunkIdToString(chunkId); const chunkEntry = this.chunks.get(chunkKey); if (chunkEntry === undefined) { From b6351cbf5ab7e290125ecc63b3f791287ab0ee1b Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Wed, 9 Sep 2026 10:53:04 -0700 Subject: [PATCH 05/19] add abort request helper method --- src/data_manager/data_manager.ts | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/data_manager/data_manager.ts b/src/data_manager/data_manager.ts index 7147d69e..11db6301 100644 --- a/src/data_manager/data_manager.ts +++ b/src/data_manager/data_manager.ts @@ -107,6 +107,23 @@ export default class DataManager { // MARK: Helpers + private tryAbortRequest(id: number): boolean { + const request = this.requests.get(id); + if (request === undefined) { + return false; + } + + const canAbort = request.chunkKeys.every((key) => { + const chunk = this.chunks.get(key); + return chunk === undefined || chunk.priority.level === ChunkPriorityLevel.RECENT; + }); + + if (canAbort) { + request.controller.abort(); + } + return canAbort; + } + /** * Inserts a chunk into the appropriate queue, or updates its queue position. * From 00c63453868d7d6f1cb147c784af0b79954c9ad7 Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Fri, 11 Sep 2026 10:04:25 -0700 Subject: [PATCH 06/19] use that new function --- src/data_manager/data_manager.ts | 6 +++--- src/data_manager/sources/ChunkSource.ts | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/data_manager/data_manager.ts b/src/data_manager/data_manager.ts index 11db6301..707ecc17 100644 --- a/src/data_manager/data_manager.ts +++ b/src/data_manager/data_manager.ts @@ -143,7 +143,7 @@ export default class DataManager { break; case ChunkState.LOADING: if (entry.memoryPriority.level === ChunkPriorityLevel.RECENT && !noAbort) { - // TODO cancel request here + this.tryAbortRequest(entry.data.requestId); } break; case ChunkState.WORKER: @@ -181,7 +181,7 @@ export default class DataManager { /** * Adds an entry for a chunk that has no requests. * - * If we're adding an entry for a chunk that no one asked for, something at least a little unexpected has happened. + * This happens when fetching a requested chunk also requires getting a chunk that no one asked for. */ private insertChunkUnprioritized(key: string, data: ChunkEntry["data"]): ChunkEntry { const entry = { @@ -383,7 +383,7 @@ export default class DataManager { break; case ChunkState.LOADING: console.error(`chunk ${evictKey} queued for eviction while in the "loading" state`); - // TODO cancel request + this.tryAbortRequest(evictEntry.data.requestId); break; default: // TODO `WORKER` state may be weird to manage here diff --git a/src/data_manager/sources/ChunkSource.ts b/src/data_manager/sources/ChunkSource.ts index 7b6c02d7..2d304e75 100644 --- a/src/data_manager/sources/ChunkSource.ts +++ b/src/data_manager/sources/ChunkSource.ts @@ -8,7 +8,7 @@ export type ExtVolumeDims = VolumeDims & { chunkShape: [number, number, number, export abstract class ChunkSource { /** - * Maps a chunk `id` to the *storage key* that contains it, and all chunk ids contained in that key. + * Maps a chunk `id` to the *storage key* that contains it. * * In most cases, a chunk id and its storage key are the same. However, `DataManager` requires that chunks extend * over exactly one channel and exactly one time point, and some data formats may allow multiple channels and/or time From c453b7335659fac0769d8c42d07649fdc1a8c5e2 Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Thu, 24 Sep 2026 13:13:37 -0700 Subject: [PATCH 07/19] add meta type --- src/data_manager/sources/ChunkSource.ts | 16 ++++++++++++++++ src/data_manager/types.ts | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/data_manager/sources/ChunkSource.ts b/src/data_manager/sources/ChunkSource.ts index 2d304e75..2c0a51f7 100644 --- a/src/data_manager/sources/ChunkSource.ts +++ b/src/data_manager/sources/ChunkSource.ts @@ -6,6 +6,20 @@ import { type Chunk, type LocalChunkId } from "../types.js"; // incorporated into `VolumeDims` or replaced with an entirely new type export type ExtVolumeDims = VolumeDims & { chunkShape: [number, number, number, number, number] }; +// This is more or less the subset of `ImageInfo` that is *not* derivable from `[Ext]VolumeDims` and is therefore *not* +// essential to handling volume data and rendering it at the correct scale. +export type VolumeMetadata = { + name?: string; + channelNames: string[]; + channelColors?: ([number, number, number] | undefined)[]; + transform: { + translation: [number, number, number]; + rotation: [number, number, number]; + scale: [number, number, number]; + }; + userData?: Record; +}; + export abstract class ChunkSource { /** * Maps a chunk `id` to the *storage key* that contains it. @@ -23,6 +37,8 @@ export abstract class ChunkSource { return [id]; } + abstract getMeta(): VolumeMetadata; + abstract getDims(): ExtVolumeDims[]; abstract getKey(key: LocalChunkId, signal?: AbortSignal): Promise[]>; diff --git a/src/data_manager/types.ts b/src/data_manager/types.ts index 0db4339c..c5375341 100644 --- a/src/data_manager/types.ts +++ b/src/data_manager/types.ts @@ -159,7 +159,7 @@ export const stringToChunkId = (id: string): ChunkId => { }; }; -export type Chunk = { +export type Chunk = { id: LocalChunkId; data: TypedArray; dtype: T; From a82c9646325a7c3c71cd062e65086fb55130b25f Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Thu, 24 Sep 2026 13:29:26 -0700 Subject: [PATCH 08/19] implement `getMeta` for `OMEZarrSource` --- src/data_manager/sources/ome_zarr_source.ts | 38 +++++++++++++++++---- src/loaders/zarr_utils/utils.ts | 4 ++- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/data_manager/sources/ome_zarr_source.ts b/src/data_manager/sources/ome_zarr_source.ts index 88431166..a2a05a39 100644 --- a/src/data_manager/sources/ome_zarr_source.ts +++ b/src/data_manager/sources/ome_zarr_source.ts @@ -1,13 +1,19 @@ import * as zarr from "zarrita"; -import { type ExtVolumeDims, ChunkSource } from "./ChunkSource.js"; +import { type ExtVolumeDims, ChunkSource, type VolumeMetadata } from "./ChunkSource.js"; import { assertMetadataHasMultiscales, toOMEZarrMetaV4, validateOMEZarrMetadata, } from "../../loaders/zarr_utils/validation.js"; import type { OMEMultiscale, OmeroTransitionalMetadata } from "../../loaders/zarr_utils/types.js"; -import { getScale, orderByDimension, orderByTCZYX, remapAxesToTCZYX } from "../../loaders/zarr_utils/utils.js"; +import { + getScale, + getSourceChannelMeta, + orderByDimension, + orderByTCZYX, + remapAxesToTCZYX, +} from "../../loaders/zarr_utils/utils.js"; import { unitNameToSymbol } from "../../loaders/VolumeLoaderUtils.js"; import type { Chunk, LocalChunkId } from "../types.js"; import type { NumberType, TypedArray } from "../../types.js"; @@ -16,11 +22,16 @@ const PLACEHOLDER_NAME = "zarr source"; const PLACEHOLDER_SCENE_INDEX = 0; export class OMEZarrSource extends ChunkSource { + // Magic number that makes `getSourceChannelMeta` (below) work. + // TODO if/when this class becomes the primary adapter for zarrs, move that util over here and adapt it such that + // this property can be removed and the remaining properties can be made `private`. + public readonly channelOffset = 0; + private constructor( - private scaleLevels: zarr.Array[], - private multiscaleMetadata: OMEMultiscale, - private omeroMetadata: OmeroTransitionalMetadata | undefined, - private axesTCZYX: [number, number, number, number, number] + public readonly scaleLevels: zarr.Array[], + public readonly multiscaleMetadata: OMEMultiscale, + public readonly omeroMetadata: OmeroTransitionalMetadata | undefined, + public readonly axesTCZYX: [number, number, number, number, number] ) { super(); } @@ -85,6 +96,21 @@ export class OMEZarrSource extends ChunkSource { return chunksInKey; } + getMeta(): VolumeMetadata { + const { names: channelNames, colors: channelColors } = getSourceChannelMeta(this); + + return { + name: this.omeroMetadata?.name ?? PLACEHOLDER_NAME, + channelNames, + channelColors, + transform: { + translation: [0, 0, 0], + rotation: [0, 0, 0], + scale: [1, 1, 1], + }, + }; + } + getDims(): ExtVolumeDims[] { const [spaceUnit, timeUnit] = this.getUnitSymbols(); return this.scaleLevels.map((level, i) => { diff --git a/src/loaders/zarr_utils/utils.ts b/src/loaders/zarr_utils/utils.ts index 2f8736f7..366c4c81 100644 --- a/src/loaders/zarr_utils/utils.ts +++ b/src/loaders/zarr_utils/utils.ts @@ -28,7 +28,9 @@ export function parseHexColor(color: string | undefined): [number, number, numbe } /** Extracts channel names from a `ZarrSource`. Handles missing `omeroMetadata`. Does *not* resolve name collisions. */ -export function getSourceChannelMeta(src: ZarrSource): { +export function getSourceChannelMeta( + src: Pick +): { names: string[]; colors: ([number, number, number] | undefined)[]; } { From a4849861fcf8f75833539fd5f7e1f0863d767df9 Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Thu, 24 Sep 2026 16:00:57 -0700 Subject: [PATCH 09/19] add method for getting metadata --- src/data_manager/data_manager.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/data_manager/data_manager.ts b/src/data_manager/data_manager.ts index 707ecc17..4e539383 100644 --- a/src/data_manager/data_manager.ts +++ b/src/data_manager/data_manager.ts @@ -16,7 +16,7 @@ import type { NumberType, TypedArray } from "../types.js"; import PriorityQueue from "./priority_queue.js"; import type { DeviceInterface } from "./device_interface.js"; import { dataTypeToSize } from "./utils.js"; -import { ChunkSource, type ExtVolumeDims } from "./sources/ChunkSource.js"; +import { ChunkSource, type VolumeMetadata, type ExtVolumeDims } from "./sources/ChunkSource.js"; import SlotMap from "./SlotMap.js"; const SUBSCRIBER_ID = Symbol("DataManager.subscriberId"); @@ -621,6 +621,11 @@ export default class DataManager { return this.sources[sourceId]?.source.getDims(); } + /** Returns the metadata of the given source. */ + getSourceMeta(sourceId: number): VolumeMetadata | undefined { + return this.sources[sourceId]?.source.getMeta(); + } + /** Subscribes `subscriber` to data events from the source with id `sourceId` */ subscribeToSource(subscriber: IDataSubscriber, sourceId: number): boolean { const sourceEntry = this.sources[sourceId]; From 5599042b98d9c84e3395687190ffa16b6071f8ee Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Mon, 5 Oct 2026 15:38:13 -0700 Subject: [PATCH 10/19] fix type errors --- src/data_manager/data_manager.ts | 2 +- src/test/data_manager.test.ts | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/data_manager/data_manager.ts b/src/data_manager/data_manager.ts index 4e539383..e3c0110c 100644 --- a/src/data_manager/data_manager.ts +++ b/src/data_manager/data_manager.ts @@ -115,7 +115,7 @@ export default class DataManager { const canAbort = request.chunkKeys.every((key) => { const chunk = this.chunks.get(key); - return chunk === undefined || chunk.priority.level === ChunkPriorityLevel.RECENT; + return chunk === undefined || chunk.memoryPriority.level === ChunkPriorityLevel.RECENT; }); if (canAbort) { diff --git a/src/test/data_manager.test.ts b/src/test/data_manager.test.ts index 47b4fb52..fb87cbfc 100644 --- a/src/test/data_manager.test.ts +++ b/src/test/data_manager.test.ts @@ -1,12 +1,13 @@ -import type { IChunkSource, IDataSubscriber } from "../data_manager/data_manager.js"; +import type { IDataSubscriber } from "../data_manager/data_manager.js"; import DataManager from "../data_manager/data_manager.js"; import type { DeviceInterface } from "../data_manager/device_interface.js"; +import type { ChunkSource } from "../data_manager/sources/ChunkSource.js"; import { type ChunkPriority, ChunkPriorityLevel } from "../data_manager/types.js"; import type { TypedArray, NumberType } from "../types.js"; const mockSource = () => { return { - getDims: vi.fn(() => [ + getDims: vi.fn(() => [ { shape: [7, 7, 7, 7, 7], spacing: [1, 1, 1, 1, 1], @@ -16,13 +17,13 @@ const mockSource = () => { timeUnit: "second", }, ]), - getChunk: vi.fn(async ({ tczyx }) => { + getChunk: vi.fn(async ({ tczyx }) => { const length = tczyx.slice(2).reduce((len, coord) => (coord >= 3 ? len : len * 2), 1); const data = new Uint8Array(length).fill(0); data.set(tczyx); return { data, dtype: "uint8" }; }), - } satisfies IChunkSource; + } satisfies ChunkSource; }; type MockTex = { From af8f0ac0b9cfbe74264147546519800835f52b51 Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Mon, 5 Oct 2026 15:45:10 -0700 Subject: [PATCH 11/19] fix types in tests --- src/test/data_manager.test.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/test/data_manager.test.ts b/src/test/data_manager.test.ts index fb87cbfc..6e84b042 100644 --- a/src/test/data_manager.test.ts +++ b/src/test/data_manager.test.ts @@ -17,12 +17,22 @@ const mockSource = () => { timeUnit: "second", }, ]), - getChunk: vi.fn(async ({ tczyx }) => { - const length = tczyx.slice(2).reduce((len, coord) => (coord >= 3 ? len : len * 2), 1); + getKey: vi.fn(async (id) => { + const length = id.tczyx.slice(2).reduce((len, coord) => (coord >= 3 ? len : len * 2), 1); const data = new Uint8Array(length).fill(0); - data.set(tczyx); - return { data, dtype: "uint8" }; + data.set(id.tczyx); + return [{ data, dtype: "uint8", id }]; }), + getMeta: vi.fn(() => ({ + channelNames: ["one", "two", "three"], + transform: { + translation: [0, 0, 0], + rotation: [0, 0, 0], + scale: [1, 1, 1], + }, + })), + chunkIdToStorageId: vi.fn((id) => id), + storageIdToChunkIds: vi.fn((id) => [id]), } satisfies ChunkSource; }; @@ -70,7 +80,7 @@ describe("request priority", () => { queue(2, 2); manager.update(); - const getCalls = sourceMock.getChunk.mock.calls; + const getCalls = sourceMock.getKey.mock.calls; expect(getCalls[0][0].tczyx[4]).toEqual(2); expect(getCalls[1][0].tczyx[4]).toEqual(0); expect(getCalls[2][0].tczyx[4]).toEqual(1); @@ -88,7 +98,7 @@ describe("request priority", () => { queue(1, { level: ChunkPriorityLevel.VISIBLE, score: 0 }); manager.update(); - const getCalls = sourceMock.getChunk.mock.calls; + const getCalls = sourceMock.getKey.mock.calls; expect(getCalls[0][0].tczyx[4]).toEqual(1); expect(getCalls[1][0].tczyx[4]).toEqual(0); }); From 12d8ed39c9a5d165cd95c352c879dd36d046a1c0 Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Tue, 6 Oct 2026 15:57:05 -0700 Subject: [PATCH 12/19] rename to use underscores --- src/data_manager/data_manager.ts | 2 +- src/data_manager/{SlotMap.ts => slot_map.ts} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename src/data_manager/{SlotMap.ts => slot_map.ts} (100%) diff --git a/src/data_manager/data_manager.ts b/src/data_manager/data_manager.ts index e3c0110c..044dfef0 100644 --- a/src/data_manager/data_manager.ts +++ b/src/data_manager/data_manager.ts @@ -17,7 +17,7 @@ import PriorityQueue from "./priority_queue.js"; import type { DeviceInterface } from "./device_interface.js"; import { dataTypeToSize } from "./utils.js"; import { ChunkSource, type VolumeMetadata, type ExtVolumeDims } from "./sources/ChunkSource.js"; -import SlotMap from "./SlotMap.js"; +import SlotMap from "./slot_map.js"; const SUBSCRIBER_ID = Symbol("DataManager.subscriberId"); diff --git a/src/data_manager/SlotMap.ts b/src/data_manager/slot_map.ts similarity index 100% rename from src/data_manager/SlotMap.ts rename to src/data_manager/slot_map.ts From 268e59100256159140a094821dd10f4c6f5d376a Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Tue, 6 Oct 2026 16:05:02 -0700 Subject: [PATCH 13/19] another module rename --- src/data_manager/sources/{ChunkSource.ts => chunk_source.ts} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/data_manager/sources/{ChunkSource.ts => chunk_source.ts} (100%) diff --git a/src/data_manager/sources/ChunkSource.ts b/src/data_manager/sources/chunk_source.ts similarity index 100% rename from src/data_manager/sources/ChunkSource.ts rename to src/data_manager/sources/chunk_source.ts From ab31cac3ddfc52ca7986b876a3730b6e24181c99 Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Tue, 6 Oct 2026 16:16:50 -0700 Subject: [PATCH 14/19] fix imports --- src/data_manager/data_manager.ts | 2 +- src/data_manager/sources/ome_zarr_source.ts | 2 +- src/test/data_manager.test.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/data_manager/data_manager.ts b/src/data_manager/data_manager.ts index 044dfef0..188d9979 100644 --- a/src/data_manager/data_manager.ts +++ b/src/data_manager/data_manager.ts @@ -16,7 +16,7 @@ import type { NumberType, TypedArray } from "../types.js"; import PriorityQueue from "./priority_queue.js"; import type { DeviceInterface } from "./device_interface.js"; import { dataTypeToSize } from "./utils.js"; -import { ChunkSource, type VolumeMetadata, type ExtVolumeDims } from "./sources/ChunkSource.js"; +import { ChunkSource, type VolumeMetadata, type ExtVolumeDims } from "./sources/chunk_source.js"; import SlotMap from "./slot_map.js"; const SUBSCRIBER_ID = Symbol("DataManager.subscriberId"); diff --git a/src/data_manager/sources/ome_zarr_source.ts b/src/data_manager/sources/ome_zarr_source.ts index a2a05a39..66c43c82 100644 --- a/src/data_manager/sources/ome_zarr_source.ts +++ b/src/data_manager/sources/ome_zarr_source.ts @@ -1,6 +1,6 @@ import * as zarr from "zarrita"; -import { type ExtVolumeDims, ChunkSource, type VolumeMetadata } from "./ChunkSource.js"; +import { type ExtVolumeDims, ChunkSource, type VolumeMetadata } from "./chunk_source.js"; import { assertMetadataHasMultiscales, toOMEZarrMetaV4, diff --git a/src/test/data_manager.test.ts b/src/test/data_manager.test.ts index 6e84b042..9858fca9 100644 --- a/src/test/data_manager.test.ts +++ b/src/test/data_manager.test.ts @@ -1,7 +1,7 @@ import type { IDataSubscriber } from "../data_manager/data_manager.js"; import DataManager from "../data_manager/data_manager.js"; import type { DeviceInterface } from "../data_manager/device_interface.js"; -import type { ChunkSource } from "../data_manager/sources/ChunkSource.js"; +import type { ChunkSource } from "../data_manager/sources/chunk_source.js"; import { type ChunkPriority, ChunkPriorityLevel } from "../data_manager/types.js"; import type { TypedArray, NumberType } from "../types.js"; From e570374e33c241682147dfd7aa71b84a4e83cbd9 Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Thu, 15 Oct 2026 16:04:50 -0700 Subject: [PATCH 15/19] convert `ChunkSource` back to an interface --- src/data_manager/sources/chunk_source.ts | 16 ++++++---------- src/data_manager/sources/ome_zarr_source.ts | 6 ++---- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/data_manager/sources/chunk_source.ts b/src/data_manager/sources/chunk_source.ts index 2c0a51f7..17bfb245 100644 --- a/src/data_manager/sources/chunk_source.ts +++ b/src/data_manager/sources/chunk_source.ts @@ -20,7 +20,7 @@ export type VolumeMetadata = { userData?: Record; }; -export abstract class ChunkSource { +export interface ChunkSource { /** * Maps a chunk `id` to the *storage key* that contains it. * @@ -28,18 +28,14 @@ export abstract class ChunkSource { * over exactly one channel and exactly one time point, and some data formats may allow multiple channels and/or time * points to be stored at the same storage key. */ - chunkIdToStorageId(id: LocalChunkId): LocalChunkId { - return id; - } + chunkIdToStorageId(id: LocalChunkId): LocalChunkId; /** Maps a storage key to a list of the chunk ids it contains. */ - storageIdToChunkIds(id: LocalChunkId): LocalChunkId[] { - return [id]; - } + storageIdToChunkIds(id: LocalChunkId): LocalChunkId[]; - abstract getMeta(): VolumeMetadata; + getMeta(): VolumeMetadata; - abstract getDims(): ExtVolumeDims[]; + getDims(): ExtVolumeDims[]; - abstract getKey(key: LocalChunkId, signal?: AbortSignal): Promise[]>; + getKey(key: LocalChunkId, signal?: AbortSignal): Promise[]>; } diff --git a/src/data_manager/sources/ome_zarr_source.ts b/src/data_manager/sources/ome_zarr_source.ts index 66c43c82..798a11fd 100644 --- a/src/data_manager/sources/ome_zarr_source.ts +++ b/src/data_manager/sources/ome_zarr_source.ts @@ -21,7 +21,7 @@ import type { NumberType, TypedArray } from "../../types.js"; const PLACEHOLDER_NAME = "zarr source"; const PLACEHOLDER_SCENE_INDEX = 0; -export class OMEZarrSource extends ChunkSource { +export class OMEZarrSource implements ChunkSource { // Magic number that makes `getSourceChannelMeta` (below) work. // TODO if/when this class becomes the primary adapter for zarrs, move that util over here and adapt it such that // this property can be removed and the remaining properties can be made `private`. @@ -32,9 +32,7 @@ export class OMEZarrSource extends ChunkSource { public readonly multiscaleMetadata: OMEMultiscale, public readonly omeroMetadata: OmeroTransitionalMetadata | undefined, public readonly axesTCZYX: [number, number, number, number, number] - ) { - super(); - } + ) {} static async new(url: string): Promise { const store = new zarr.FetchStore(url); From a401e7369bff2429f7743f0eff06216b7759d5d8 Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Thu, 15 Oct 2026 16:17:34 -0700 Subject: [PATCH 16/19] put `getChunk` `AbortSignal` in an options object --- src/data_manager/data_manager.ts | 2 +- src/data_manager/sources/chunk_source.ts | 6 +++++- src/data_manager/sources/ome_zarr_source.ts | 6 +++--- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/data_manager/data_manager.ts b/src/data_manager/data_manager.ts index 188d9979..b853c1f5 100644 --- a/src/data_manager/data_manager.ts +++ b/src/data_manager/data_manager.ts @@ -456,7 +456,7 @@ export default class DataManager { // Request the storage key sourceEntry.source - .getKey(storageId, controller.signal) + .getKey(storageId, { signal: controller.signal }) .then((chunks) => { this.requests.remove(requestId); chunks.map((chunk) => this.onChunkLoad(sourceId, chunk)); diff --git a/src/data_manager/sources/chunk_source.ts b/src/data_manager/sources/chunk_source.ts index 17bfb245..27ddfc3b 100644 --- a/src/data_manager/sources/chunk_source.ts +++ b/src/data_manager/sources/chunk_source.ts @@ -20,6 +20,10 @@ export type VolumeMetadata = { userData?: Record; }; +export type GetKeyOptions = { + signal?: AbortSignal; +}; + export interface ChunkSource { /** * Maps a chunk `id` to the *storage key* that contains it. @@ -37,5 +41,5 @@ export interface ChunkSource { getDims(): ExtVolumeDims[]; - getKey(key: LocalChunkId, signal?: AbortSignal): Promise[]>; + getKey(key: LocalChunkId, options?: GetKeyOptions): Promise[]>; } diff --git a/src/data_manager/sources/ome_zarr_source.ts b/src/data_manager/sources/ome_zarr_source.ts index 798a11fd..ff7aac43 100644 --- a/src/data_manager/sources/ome_zarr_source.ts +++ b/src/data_manager/sources/ome_zarr_source.ts @@ -1,6 +1,6 @@ import * as zarr from "zarrita"; -import { type ExtVolumeDims, ChunkSource, type VolumeMetadata } from "./chunk_source.js"; +import type { ExtVolumeDims, ChunkSource, VolumeMetadata, GetKeyOptions } from "./chunk_source.js"; import { assertMetadataHasMultiscales, toOMEZarrMetaV4, @@ -124,10 +124,10 @@ export class OMEZarrSource implements ChunkSource { }); } - async getKey(id: LocalChunkId, signal?: AbortSignal): Promise[]> { + async getKey(id: LocalChunkId, options: GetKeyOptions): Promise[]> { const multiscale = this.scaleLevels[id.multiscale]; const coords = orderByDimension(id.tczyx, this.axesTCZYX); - const { data } = await multiscale.getChunk(coords, { signal }); + const { data } = await multiscale.getChunk(coords, { signal: options?.signal }); return [{ data: data as TypedArray, dtype: multiscale.dtype, id }]; } } From b51c737505496f69e0835bae147f28252556b39b Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Thu, 15 Oct 2026 16:26:54 -0700 Subject: [PATCH 17/19] compute chunks per request in zarr source --- src/data_manager/sources/ome_zarr_source.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/data_manager/sources/ome_zarr_source.ts b/src/data_manager/sources/ome_zarr_source.ts index ff7aac43..ef0f4ece 100644 --- a/src/data_manager/sources/ome_zarr_source.ts +++ b/src/data_manager/sources/ome_zarr_source.ts @@ -127,6 +127,8 @@ export class OMEZarrSource implements ChunkSource { async getKey(id: LocalChunkId, options: GetKeyOptions): Promise[]> { const multiscale = this.scaleLevels[id.multiscale]; const coords = orderByDimension(id.tczyx, this.axesTCZYX); + const [sizeT, sizeC] = orderByTCZYX(multiscale.chunks, this.axesTCZYX, 1); + const chunkCount = sizeT * sizeC; const { data } = await multiscale.getChunk(coords, { signal: options?.signal }); return [{ data: data as TypedArray, dtype: multiscale.dtype, id }]; } From fae26d9e72a8587d5c23c54c872a7bde69668afa Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Thu, 15 Oct 2026 17:44:32 -0700 Subject: [PATCH 18/19] properly separate t/c chunks in zarr source --- src/data_manager/sources/ome_zarr_source.ts | 35 +++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/src/data_manager/sources/ome_zarr_source.ts b/src/data_manager/sources/ome_zarr_source.ts index ef0f4ece..63a31a67 100644 --- a/src/data_manager/sources/ome_zarr_source.ts +++ b/src/data_manager/sources/ome_zarr_source.ts @@ -124,12 +124,35 @@ export class OMEZarrSource implements ChunkSource { }); } - async getKey(id: LocalChunkId, options: GetKeyOptions): Promise[]> { - const multiscale = this.scaleLevels[id.multiscale]; - const coords = orderByDimension(id.tczyx, this.axesTCZYX); - const [sizeT, sizeC] = orderByTCZYX(multiscale.chunks, this.axesTCZYX, 1); - const chunkCount = sizeT * sizeC; + async getKey(key: LocalChunkId, options: GetKeyOptions): Promise { + const multiscale = this.scaleLevels[key.multiscale]; + const coords = orderByDimension(key.tczyx, this.axesTCZYX); + + const [shapeT, shapeC] = orderByTCZYX(multiscale.chunks, this.axesTCZYX, 1); + const [orderT, orderC] = this.axesTCZYX; + const [keyT, keyC, idZ, idY, idX] = key.tczyx; + + const chunkCount = shapeT * shapeC; + const cMinor = shapeT > 1 && shapeC > 1 && orderT > -1 && orderC > -1 && orderC < orderT; + const majorLength = cMinor ? shapeT : shapeC; + const tMin = keyT * shapeT; + const cMin = keyC * shapeC; + const { data } = await multiscale.getChunk(coords, { signal: options?.signal }); - return [{ data: data as TypedArray, dtype: multiscale.dtype, id }]; + + const chunkLength = data.length / chunkCount; + const result: Chunk[] = []; + + for (let i = 0; i < chunkCount; i++) { + const chunkStart = i * chunkLength; + const subarray = (data as TypedArray).slice(chunkStart, chunkStart + chunkLength); + const offsetMinor = Math.floor(i / majorLength); + const offsetMajor = i % majorLength; + const [idT, idC] = cMinor ? [tMin + offsetMajor, cMin + offsetMinor] : [tMin + offsetMinor, cMin + offsetMajor]; + const id: LocalChunkId = { multiscale: key.multiscale, tczyx: [idT, idC, idZ, idY, idX] }; + result.push({ data: subarray, dtype: multiscale.dtype, id }); + } + + return result; } } From 9d8b66e439ef7f7b5a7dda16668d2d52a0047a54 Mon Sep 17 00:00:00 2001 From: Cameron Fraser Date: Thu, 15 Oct 2026 18:11:01 -0700 Subject: [PATCH 19/19] account for max-edge chunks when separating t/c --- src/data_manager/sources/ome_zarr_source.ts | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/data_manager/sources/ome_zarr_source.ts b/src/data_manager/sources/ome_zarr_source.ts index 63a31a67..02018d85 100644 --- a/src/data_manager/sources/ome_zarr_source.ts +++ b/src/data_manager/sources/ome_zarr_source.ts @@ -126,18 +126,25 @@ export class OMEZarrSource implements ChunkSource { async getKey(key: LocalChunkId, options: GetKeyOptions): Promise { const multiscale = this.scaleLevels[key.multiscale]; - const coords = orderByDimension(key.tczyx, this.axesTCZYX); + const [keyT, keyC, idZ, idY, idX] = key.tczyx; + // If this chunk contains more than one timepoint and/or channel, we need to know how to separate them const [shapeT, shapeC] = orderByTCZYX(multiscale.chunks, this.axesTCZYX, 1); + const [sizeT, sizeC] = orderByTCZYX(multiscale.shape, this.axesTCZYX, 1); + // Does C come before T in the dimension order? const [orderT, orderC] = this.axesTCZYX; - const [keyT, keyC, idZ, idY, idX] = key.tczyx; - - const chunkCount = shapeT * shapeC; const cMinor = shapeT > 1 && shapeC > 1 && orderT > -1 && orderC > -1 && orderC < orderT; - const majorLength = cMinor ? shapeT : shapeC; + const tMin = keyT * shapeT; const cMin = keyC * shapeC; + // Account for max-edge chunks + // (e.g. if each chunk has 2 channels and there are 5 channels total, the last chunk has one channel) + const chunksT = Math.min(shapeT, sizeT - tMin); + const chunksC = Math.min(shapeC, sizeC - cMin); + const chunkCount = chunksT * chunksC; + const majorLength = cMinor ? chunksT : chunksC; + const coords = orderByDimension(key.tczyx, this.axesTCZYX); const { data } = await multiscale.getChunk(coords, { signal: options?.signal }); const chunkLength = data.length / chunkCount;