From 34ffa0df841b03ed32e1718fb6560632b4feef9c Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Mon, 30 Mar 2026 17:24:24 +0200 Subject: [PATCH 01/10] Feat: download a file range of bytes --- src/apps/main/network/downloadv2.ts | 1 + .../download-file/build-crypto-lib.ts | 14 ++++ .../download-file/decrypt-at-offset.ts | 40 ++++++++++ .../download-file/download-file.ts | 74 +++++++++++++++++++ src/infra/environment/download-file/types.ts | 13 ++++ 5 files changed, 142 insertions(+) create mode 100644 src/infra/environment/download-file/build-crypto-lib.ts create mode 100644 src/infra/environment/download-file/decrypt-at-offset.ts create mode 100644 src/infra/environment/download-file/download-file.ts create mode 100644 src/infra/environment/download-file/types.ts diff --git a/src/apps/main/network/downloadv2.ts b/src/apps/main/network/downloadv2.ts index b928ccdd28..93f58c6b5c 100644 --- a/src/apps/main/network/downloadv2.ts +++ b/src/apps/main/network/downloadv2.ts @@ -95,6 +95,7 @@ const downloadOwnFile: DownloadOwnFileFunction = (params) => { const downloadFileV2: DownloadFileFunction = (params) => { if (params.token && params.encryptionKey) { + // This is de facto dead code as its never called with params.token return downloadSharedFile(params); } else if (params.creds && params.mnemonic) { return downloadOwnFile(params); diff --git a/src/infra/environment/download-file/build-crypto-lib.ts b/src/infra/environment/download-file/build-crypto-lib.ts new file mode 100644 index 0000000000..2c7c262134 --- /dev/null +++ b/src/infra/environment/download-file/build-crypto-lib.ts @@ -0,0 +1,14 @@ +import { Network } from '@internxt/sdk'; +import { validateMnemonic } from 'bip39'; +import { Environment } from '@internxt/inxt-js'; +import { randomBytes } from 'node:crypto'; + +export function buildCryptoLib(): Network.Crypto { + return { + algorithm: Network.ALGORITHMS.AES256CTR, + validateMnemonic: (mnemonic: string) => validateMnemonic(mnemonic), + generateFileKey: (mnemonic, bucketId, index) => + Environment.utils.generateFileKey(mnemonic, bucketId, index as Buffer), + randomBytes, + }; +} diff --git a/src/infra/environment/download-file/decrypt-at-offset.ts b/src/infra/environment/download-file/decrypt-at-offset.ts new file mode 100644 index 0000000000..2d38222108 --- /dev/null +++ b/src/infra/environment/download-file/decrypt-at-offset.ts @@ -0,0 +1,40 @@ +import { createDecipheriv } from 'node:crypto'; + +/** + * Decrypts a byte range of an AES-256-CTR encrypted file starting at a given position. + * + * AES-CTR is a stream cipher that works by encrypting sequential counter blocks and XORing + * the result with the plaintext. This makes it seekable: to decrypt bytes starting at position N, + * you only need to know which counter block N falls in, rather than decrypting all preceding bytes. + * + * The counter block for position N is: originalIV + floor(N / 16) + * If N is mid-block (N % 16 !== 0), we advance the decipher by the partial block remainder + * before decrypting the actual bytes. + * + * @param encryptedBytes - The raw encrypted bytes for the requested range (fetched via HTTP Range header) + * @param key - The AES-256 file key + * @param iv - Initialization Vector: a random 16-byte value generated when the file was encrypted, + * stored in the file's network metadata index. Ensures that two files with the same key + * produce different ciphertext. Retrieved by the SDK as the first 16 bytes of the file index. + * @param position - The byte offset in the full file where this range starts + */ +export function decryptAtOffset(encryptedBytes: Buffer, key: Buffer, iv: Buffer, position: number): Buffer { + const AES_BLOCK_SIZE = 16; + const partialBlock = position % AES_BLOCK_SIZE; + const startBlockNumber = (position - partialBlock) / AES_BLOCK_SIZE; + + // Compute the IV for the starting block by adding the block number to the original IV + const ivForRange = (BigInt('0x' + iv.toString('hex')) + BigInt(startBlockNumber)) + .toString(16) + .padStart(32, '0'); + const offsetIv = Buffer.from(ivForRange, 'hex'); + + const decipher = createDecipheriv('aes-256-ctr', new Uint8Array(key), new Uint8Array(offsetIv)); + + // If position is mid-block, skip the leading partial block bytes + if (partialBlock > 0) { + decipher.update(new Uint8Array(partialBlock)); + } + + return decipher.update(new Uint8Array(encryptedBytes)); +} diff --git a/src/infra/environment/download-file/download-file.ts b/src/infra/environment/download-file/download-file.ts new file mode 100644 index 0000000000..6b702da1c1 --- /dev/null +++ b/src/infra/environment/download-file/download-file.ts @@ -0,0 +1,74 @@ +import { DecryptFileFunction, DownloadFileFunction } from '@internxt/sdk/dist/network'; +import { downloadFile as sdkDownloadFile } from '@internxt/sdk/dist/network/download'; +import axios from 'axios'; +import { buildCryptoLib } from './build-crypto-lib'; +import { DownloadFileProps } from './types'; +import { DriveDesktopError } from '../../../context/shared/domain/errors/DriveDesktopError'; +import { decryptAtOffset } from './decrypt-at-offset'; + +async function fetchEncryptedRange(url: string, position: number, length: number): Promise { + const response = await axios.get(url, { + responseType: 'stream', + headers: { + range: `bytes=${position}-${position + length - 1}`, + }, + }); + + return new Promise((resolve, reject) => { + const chunks: Uint8Array[] = []; + response.data.on('data', (chunk: Uint8Array) => chunks.push(chunk)); + response.data.on('end', () => resolve(Buffer.concat(chunks))); + response.data.on('error', reject); + }); +} + + + +export async function downloadFileRange({ + signal, + fileId, + bucketId, + mnemonic, + network, + range, +}: DownloadFileProps): Promise { + let encryptedBytes: Buffer | undefined; + let decryptedBuffer: Buffer | undefined; + + const downloadFileCb: DownloadFileFunction = async (downloadables) => { + if (range && downloadables.length > 1) { + throw new Error('Multi-Part Download with Range-Requests is not implemented'); + } + for (const downloadable of downloadables) { + if (signal.signal.aborted) { + throw new DriveDesktopError('ABORTED'); + } + // eslint-disable-next-line no-await-in-loop + encryptedBytes = await fetchEncryptedRange(downloadable.url, range.position, range.length); + } + }; + + const decryptFileCb: DecryptFileFunction = async (_, key, iv) => { + if (!encryptedBytes) throw new Error('No encrypted bytes to decrypt'); + decryptedBuffer = decryptAtOffset( + encryptedBytes, + Buffer.from(key.toString('hex'), 'hex'), + Buffer.from(iv.toString('hex'), 'hex'), + range.position, + ); + }; + + await sdkDownloadFile( + fileId, + bucketId, + mnemonic, + network, + buildCryptoLib(), + Buffer.from, + downloadFileCb, + decryptFileCb, + ); + + if (!decryptedBuffer) throw new Error('Decryption did not produce a buffer'); + return decryptedBuffer; +} diff --git a/src/infra/environment/download-file/types.ts b/src/infra/environment/download-file/types.ts new file mode 100644 index 0000000000..5c6c681983 --- /dev/null +++ b/src/infra/environment/download-file/types.ts @@ -0,0 +1,13 @@ +import { Network } from '@internxt/sdk'; + +export type DownloadFileProps = { + signal: AbortController; + fileId: string; + bucketId: string; + mnemonic: string; + network: Network.Network; + range: { + position: number; + length: number; + }; +}; From 93eadd852e85b6805232b9811e69968eefd6c63d Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Wed, 1 Apr 2026 18:15:42 +0200 Subject: [PATCH 02/10] WIP: download files on demand --- src/apps/drive/fuse/callbacks/ReadCallback.ts | 40 ++-- src/apps/drive/index.ts | 9 +- .../features/fuse/on-read/constants.ts | 1 + .../on-read/download-cache/allocate-file.ts | 21 +++ .../fuse/on-read/download-cache/constants.ts | 7 + .../download-cache/download-and-save-block.ts | 42 +++++ .../expand-to-block-boundaries.ts | 17 ++ .../download-cache/file-exists-on-disk.ts | 7 + .../on-read/download-cache/hydration-state.ts | 169 +++++++++++++++++ .../download-cache/hydration-stopwatch.ts | 17 ++ .../fuse/on-read/handle-read-callback.ts | 176 ++++++++++-------- .../fuse/on-read/read-chunk-from-disk.ts | 9 + .../download-file/build-network-client.ts | 23 +++ 13 files changed, 436 insertions(+), 102 deletions(-) create mode 100644 src/backend/features/fuse/on-read/constants.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/allocate-file.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/constants.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/expand-to-block-boundaries.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/file-exists-on-disk.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/hydration-state.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/hydration-stopwatch.ts create mode 100644 src/infra/environment/download-file/build-network-client.ts diff --git a/src/apps/drive/fuse/callbacks/ReadCallback.ts b/src/apps/drive/fuse/callbacks/ReadCallback.ts index ff131b32ca..cc475bc91c 100644 --- a/src/apps/drive/fuse/callbacks/ReadCallback.ts +++ b/src/apps/drive/fuse/callbacks/ReadCallback.ts @@ -4,15 +4,15 @@ import { TemporalFileByPathFinder } from '../../../../context/storage/TemporalFi import { TemporalFileChunkReader } from '../../../../context/storage/TemporalFiles/application/read/TemporalFileChunkReader'; import { FirstsFileSearcher } from '../../../../context/virtual-drive/files/application/search/FirstsFileSearcher'; import { StorageFilesRepository } from '../../../../context/storage/StorageFiles/domain/StorageFilesRepository'; -import { StorageFileId } from '../../../../context/storage/StorageFiles/domain/StorageFileId'; import { StorageFile } from '../../../../context/storage/StorageFiles/domain/StorageFile'; -import { StorageFileDownloader } from '../../../../context/storage/StorageFiles/application/download/StorageFileDownloader/StorageFileDownloader'; import { DownloadProgressTracker } from '../../../../context/shared/domain/DownloadProgressTracker'; -import { type File } from '../../../../context/virtual-drive/files/domain/File'; import { handleReadCallback, type HandleReadCallbackDeps, } from '../../../../backend/features/fuse/on-read/handle-read-callback'; +import { buildNetworkClient } from '../../../../infra/environment/download-file/build-network-client'; +import { getCredentials } from '../../../main/auth/get-credentials'; +import { DependencyInjectionUserProvider } from '../../../shared/dependency-injection/DependencyInjectionUserProvider'; import Fuse from '@gcas/fuse'; @@ -21,15 +21,17 @@ export class ReadCallback { async execute( path: string, - _fd: any, + _fd: unknown, buf: Buffer, len: number, pos: number, - cb: (code: number, params?: any) => void, + cb: (code: number, params?: unknown) => void, ) { try { + const { mnemonic } = getCredentials(); + const user = DependencyInjectionUserProvider.get(); + const network = buildNetworkClient({ bridgeUser: user.bridgeUser, userId: user.userId }); const repo = this.container.get(StorageFilesRepository); - const downloader = this.container.get(StorageFileDownloader); const tracker = this.container.get(DownloadProgressTracker); const deps: HandleReadCallbackDeps = { @@ -39,30 +41,20 @@ export class ReadCallback { const result = await this.container.get(TemporalFileChunkReader).run(p, length, position); return result.isPresent() ? result.get() : undefined; }, - existsOnDisk: (contentsId: string) => repo.exists(new StorageFileId(contentsId)), - - startDownload: async (virtualFile: File) => { - const storage = StorageFile.from({ - id: virtualFile.contentsId, - virtualId: virtualFile.uuid, - size: virtualFile.size, + onDownloadProgress: (name, extension, bytesDownloaded, fileSize, elapsedTime) => { + tracker.downloadUpdate(name, extension, { + percentage: Math.min(bytesDownloaded / fileSize, 1), + elapsedTime, }); - tracker.downloadStarted(virtualFile.name, virtualFile.type); - const { stream, handler } = await downloader.run(storage, virtualFile); - return { stream, elapsedTime: () => handler.elapsedTime() }; - }, - onDownloadProgress: (name, extension, progress) => { - tracker.downloadUpdate(name, extension, progress); }, saveToRepository: async (contentsId, size, uuid, name, extension) => { - const storage = StorageFile.from({ - id: contentsId, - virtualId: uuid, - size, - }); + const storage = StorageFile.from({ id: contentsId, virtualId: uuid, size }); await repo.register(storage); tracker.downloadFinished(name, extension); }, + bucketId: user.bucket, + mnemonic, + network, }; const result = await handleReadCallback(deps, path, len, pos); diff --git a/src/apps/drive/index.ts b/src/apps/drive/index.ts index c48f372b68..55922fb7ab 100644 --- a/src/apps/drive/index.ts +++ b/src/apps/drive/index.ts @@ -28,7 +28,14 @@ export async function startVirtualDrive() { fuseApp.on('mount-error', () => broadcastToWindows('virtual-drive-status-change', 'ERROR')); await hydrationApi.start({ debug: false, timeElapsed: false }); - + /** + * v2.5.4 + * Alexis Mora + * If a user abruptly quits the app, all the hydrated files will be orphaned. + * Hence why we clear the cache before starting up the virtual drive. + * To ensure that every time we get a fresh start. + */ + fuseApp.clearCache(); await fuseApp.start(); } diff --git a/src/backend/features/fuse/on-read/constants.ts b/src/backend/features/fuse/on-read/constants.ts new file mode 100644 index 0000000000..d0b25712aa --- /dev/null +++ b/src/backend/features/fuse/on-read/constants.ts @@ -0,0 +1 @@ +export const EMPTY = Buffer.alloc(0); diff --git a/src/backend/features/fuse/on-read/download-cache/allocate-file.ts b/src/backend/features/fuse/on-read/download-cache/allocate-file.ts new file mode 100644 index 0000000000..e8a1c1ae00 --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/allocate-file.ts @@ -0,0 +1,21 @@ +import fs from 'node:fs/promises'; + +/** + * Pre-allocates a file on disk to the full expected size before any ranges are downloaded. + * + * This is necessary for random-access writes: since FUSE reads can arrive in any order, + * we need the file to exist at its full size so we can write each range at its correct + * byte offset. Without pre-allocation, writing at offset 500MB would fail because the + * file doesn't exist yet. + * + * The file is filled with zeros initially, the {@link rangeRegistry} tracks which regions + * contain real downloaded bytes vs unfilled zeros. + */ +export async function allocateFile(filePath: string, size: number): Promise { + const handle = await fs.open(filePath, 'w'); + try { + await handle.truncate(size); + } finally { + await handle.close(); + } +} diff --git a/src/backend/features/fuse/on-read/download-cache/constants.ts b/src/backend/features/fuse/on-read/download-cache/constants.ts new file mode 100644 index 0000000000..1fc635e970 --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/constants.ts @@ -0,0 +1,7 @@ +/** + * 4MB blocks — matches the chunk size used by the legacy downloader, proven to work well + * for this codebase. Each block is downloaded in full on first access regardless of how + * small the FUSE read is, so subsequent reads within the same block are served from disk. + */ +export const BLOCK_SIZE = 4 * 1024 * 1024; +export const BITS_PER_BYTE = 8; diff --git a/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts b/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts new file mode 100644 index 0000000000..e799cbd5cb --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts @@ -0,0 +1,42 @@ +import { HandleReadCallbackDeps } from '../handle-read-callback'; +import { writeChunkToDisk } from '../read-chunk-from-disk'; +import { type FileHydrationState, markBlocksInRangeDownloaded, startBlockDownload } from './hydration-state'; +import { type File } from '../../../../../context/virtual-drive/files/domain/File'; +import { downloadFileRange } from '../../../../../infra/environment/download-file/download-file'; +import { getStopwatch } from './hydration-stopwatch'; + +/** + * Downloads a block range, writes it to disk at the correct offset, and marks it as downloaded. + */ +export async function downloadAndCacheBlock( + deps: HandleReadCallbackDeps, + virtualFile: File, + filePath: string, + state: FileHydrationState, + blockStart: number, + blockLength: number, +): Promise { + const resolve = startBlockDownload(state, { position: blockStart, length: blockLength }); + try { + const buffer = await downloadFileRange({ + fileId: virtualFile.contentsId, + bucketId: deps.bucketId, + mnemonic: deps.mnemonic, + network: deps.network, + range: { position: blockStart, length: blockLength }, + signal: new AbortController(), + }); + await writeChunkToDisk(filePath, buffer, blockStart); + markBlocksInRangeDownloaded(state, { position: blockStart, length: blockLength }); + const elapsedTime = getStopwatch(virtualFile.contentsId)?.elapsedTime() ?? 0; + deps.onDownloadProgress( + virtualFile.name, + virtualFile.type, + blockStart + blockLength, + virtualFile.size, + elapsedTime, + ); + } finally { + resolve(); + } +} diff --git a/src/backend/features/fuse/on-read/download-cache/expand-to-block-boundaries.ts b/src/backend/features/fuse/on-read/download-cache/expand-to-block-boundaries.ts new file mode 100644 index 0000000000..bd4cd5c41f --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/expand-to-block-boundaries.ts @@ -0,0 +1,17 @@ +import { BLOCK_SIZE } from './constants'; + +/** + * Given a position and length, rounds up to 4MB block boundaries so that every + * request downloads complete blocks. Ensuring correct bitmap tracking, prefetching, + * and preventing double downloads. + */ +export function expandToBlockBoundaries( + position: number, + length: number, + fileSize: number, +): { blockStart: number; blockLength: number } { + const blockStart = Math.floor(position / BLOCK_SIZE) * BLOCK_SIZE; + const end = position + length; + const blockEnd = Math.min(Math.ceil(end / BLOCK_SIZE) * BLOCK_SIZE, fileSize); + return { blockStart, blockLength: blockEnd - blockStart }; +} diff --git a/src/backend/features/fuse/on-read/download-cache/file-exists-on-disk.ts b/src/backend/features/fuse/on-read/download-cache/file-exists-on-disk.ts new file mode 100644 index 0000000000..a8237fbd3a --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/file-exists-on-disk.ts @@ -0,0 +1,7 @@ +import fs from 'node:fs/promises'; +export async function fileExistsOnDisk(filePath: string): Promise { + return fs + .stat(filePath) + .then(() => true) + .catch(() => false); +} diff --git a/src/backend/features/fuse/on-read/download-cache/hydration-state.ts b/src/backend/features/fuse/on-read/download-cache/hydration-state.ts new file mode 100644 index 0000000000..a97699537f --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/hydration-state.ts @@ -0,0 +1,169 @@ +import { BITS_PER_BYTE, BLOCK_SIZE } from './constants'; + +/** + * Tracks which byte ranges of a file have been downloaded and written to disk. + * + * Uses a bitmap where each bit represents one 4MB block of the file. + * A set bit means that block has been FULLY downloaded and written to disk. + * An unset bit means that block contains pre-allocation zeros — not real data. + * + * This is necessary because files are pre-allocated to their full size before any + * data is downloaded, making it impossible to distinguish real bytes from zeros + * by inspecting the file alone. + * + * A block is only marked after its full write to disk succeeds — never partially. + * A hard kill mid-write is handled by wiping the download cache on startup. + * + * Race conditions: concurrent reads for the same block are tracked via an in-flight + * set. The second caller waits for the first to finish rather than double-downloading. + */ + +export type FileHydrationState = { + bitmap: Buffer; + totalBlocks: number; + downloadedBlocks: number; + blocksBeingDownloaded: Map>; +}; +type Range = { + position: number; + length: number; +}; +const hydrationState = new Map(); + +export function getOrInitHydrationState(contentsId: string, fileSize: number): FileHydrationState { + const existing = hydrationState.get(contentsId); + if (existing) return existing; + + const totalBlocks = Math.ceil(fileSize / BLOCK_SIZE); + const size = Math.ceil(totalBlocks / BITS_PER_BYTE); + const state: FileHydrationState = { + bitmap: Buffer.alloc(size, 0), + totalBlocks, + downloadedBlocks: 0, + blocksBeingDownloaded: new Map(), + }; + hydrationState.set(contentsId, state); + return state; +} + +function blockIndexForByte(byte: number): number { + return Math.floor(byte / BLOCK_SIZE); +} + +/** + * Creates a bitmask: a number where exactly ONE bit is turned on. + * + * Think of a byte as 8 switches: + * [bit7][bit6][bit5][bit4][bit3][bit2][bit1][bit0] + * + * The mask selects exactly one of those switches. + * + * Examples: + * bitIndexInByte = 0 is 0b00000001 (selects bit 0) + * bitIndexInByte = 2 is 0b00000100 (selects bit 2) + * bitIndexInByte = 7 is 0b10000000 (selects bit 7) + * + * Why we need this: + * - AND (&) with the mask → checks if that bit is set + * - OR (|) with the mask → sets that bit + * + * Implementation: + * Start with 1 (0b00000001) and shift it left N times. + */ +function bitMask(bitIndexInByte: number): number { + return 1 << bitIndexInByte; +} + +function getBit(bitmap: Buffer, blockIndex: number): boolean { + const byteIndex = Math.floor(blockIndex / BITS_PER_BYTE); + const bitIndexInByte = blockIndex % BITS_PER_BYTE; + return (bitmap[byteIndex] & bitMask(bitIndexInByte)) !== 0; +} + +function setBit(bitmap: Buffer, blockIndex: number): void { + const byteIndex = Math.floor(blockIndex / BITS_PER_BYTE); + const bitIndexInByte = blockIndex % BITS_PER_BYTE; + bitmap[byteIndex] = bitmap[byteIndex] | bitMask(bitIndexInByte); +} + +export function isFileHydrated(state: FileHydrationState): boolean { + return state.downloadedBlocks === state.totalBlocks; +} + +function blocksWithinRange({ position, length }: Range): Array { + const first = blockIndexForByte(position); + const last = blockIndexForByte(position + length - 1); + const blocks: number[] = []; + for (let block = first; block <= last; block++) { + blocks.push(block); + } + return blocks; +} + +export function isRangeHydrated(state: FileHydrationState, { position, length }: Range): boolean { + return blocksWithinRange({ position, length }).every((block) => getBit(state.bitmap, block)); +} + +export function markBlocksInRangeDownloaded(state: FileHydrationState, { position, length }: Range): void { + for (const block of blocksWithinRange({ position, length })) { + if (!getBit(state.bitmap, block)) { + setBit(state.bitmap, block); + state.downloadedBlocks++; + } + } +} + +/** + * Returns block indices within the range that are neither cached nor currently being downloaded. + * Use this after waiting for in-flight blocks to find what still needs downloading. + */ +export function getMissingBlocks(state: FileHydrationState, { position, length }: Range): number[] { + return blocksWithinRange({ position, length }).filter( + (block) => !getBit(state.bitmap, block) && !state.blocksBeingDownloaded.has(block), + ); +} + +export function getBlocksBeingDownloaded( + state: FileHydrationState, + { position, length }: Range, +): Map> { + const blocksBeingDownloadedWithinRange = new Map>(); + for (const block of blocksWithinRange({ position, length })) { + const existing = state.blocksBeingDownloaded.get(block); + if (existing) blocksBeingDownloadedWithinRange.set(block, existing); + } + return blocksBeingDownloadedWithinRange; +} + +/** + * Marks blocks as being downloaded. Call before starting a download. + * Returns a resolve function to call it when the download + write completes. + */ +export function startBlockDownload(state: FileHydrationState, { position, length }: Range): () => void { + let resolve: () => void = () => undefined; + const promise = new Promise((res) => { + resolve = res; + }); + + for (const block of blocksWithinRange({ position, length })) { + state.blocksBeingDownloaded.set(block, promise); + } + + return () => { + for (const block of blocksWithinRange({ position, length })) { + state.blocksBeingDownloaded.delete(block); + } + resolve(); + }; +} + +/** + * Removes the bitmap for a file — call when the file is deleted or cache is cleared. + */ +export function deleteHydrationState(contentsId: string): void { + hydrationState.delete(contentsId); +} + +export function clearHydrationState(): void { + hydrationState.clear(); +} diff --git a/src/backend/features/fuse/on-read/download-cache/hydration-stopwatch.ts b/src/backend/features/fuse/on-read/download-cache/hydration-stopwatch.ts new file mode 100644 index 0000000000..08516a4952 --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/hydration-stopwatch.ts @@ -0,0 +1,17 @@ +import { Stopwatch } from '../../../../../apps/shared/types/Stopwatch'; + +const stopwatches = new Map(); + +export function startStopwatch(contentsId: string): void { + const stopWatch = new Stopwatch(); + stopWatch.start(); + stopwatches.set(contentsId, stopWatch); +} + +export function getStopwatch(contentsId: string): Stopwatch | undefined { + return stopwatches.get(contentsId); +} + +export function deleteStopwatch(contentsId: string): void { + stopwatches.delete(contentsId); +} diff --git a/src/backend/features/fuse/on-read/handle-read-callback.ts b/src/backend/features/fuse/on-read/handle-read-callback.ts index ac69926395..2c6ce1b337 100644 --- a/src/backend/features/fuse/on-read/handle-read-callback.ts +++ b/src/backend/features/fuse/on-read/handle-read-callback.ts @@ -1,63 +1,113 @@ import { logger } from '@internxt/drive-desktop-core/build/backend'; -import { type Readable } from 'stream'; import { type TemporalFile } from '../../../../context/storage/TemporalFiles/domain/TemporalFile'; import { type File } from '../../../../context/virtual-drive/files/domain/File'; import { left, right, type Either } from '../../../../context/shared/domain/Either'; import { type FuseError, FuseNoSuchFileOrDirectoryError } from '../../../../apps/drive/fuse/callbacks/FuseErrors'; -import { tryCatch } from '../../../../shared/try-catch'; -import { createDownloadToDisk } from './create-download-to-disk'; -import { deleteHydration, getHydration, HydrationEntry, setHydration } from './hydration-registry'; import { readChunkFromDisk } from './read-chunk-from-disk'; import { shouldDownload } from '../on-open/open-flags-tracker'; import nodePath from 'node:path'; import { PATHS } from '../../../../core/electron/paths'; import { formatBytes } from '../../../../shared/format-bytes'; +import { allocateFile } from './download-cache/allocate-file'; +import { expandToBlockBoundaries } from './download-cache/expand-to-block-boundaries'; +import { BLOCK_SIZE } from './download-cache/constants'; +import { + type FileHydrationState, + getOrInitHydrationState, + isRangeHydrated, + isFileHydrated, + getBlocksBeingDownloaded, + getMissingBlocks, +} from './download-cache/hydration-state'; +import { type Network } from '@internxt/sdk'; +import { startStopwatch, deleteStopwatch } from './download-cache/hydration-stopwatch'; +import { fileExistsOnDisk } from './download-cache/file-exists-on-disk'; +import { downloadAndCacheBlock } from './download-cache/download-and-save-block'; +import { EMPTY } from './constants'; export type HandleReadCallbackDeps = { findVirtualFile: (path: string) => Promise; findTemporalFile: (path: string) => Promise; readTemporalFileChunk: (path: string, length: number, position: number) => Promise; - existsOnDisk: (contentsId: string) => Promise; - startDownload: (virtualFile: File) => Promise<{ stream: Readable; elapsedTime: () => number }>; - onDownloadProgress: (name: string, extension: string, progress: { percentage: number; elapsedTime: number }) => void; + onDownloadProgress: ( + name: string, + extension: string, + bytesDownloaded: number, + fileSize: number, + elapsedTime: number, + ) => void; saveToRepository: (contentsId: string, size: number, uuid: string, name: string, extension: string) => Promise; + bucketId: string; + mnemonic: string; + network: Network.Network; }; -const EMPTY = Buffer.alloc(0); +async function readFromTemporalFile( + deps: HandleReadCallbackDeps, + path: string, + length: number, + position: number, +): Promise> { + const temporalFile = await deps.findTemporalFile(path); + + if (!temporalFile) { + logger.error({ msg: '[ReadCallback] File not found', path }); + return left(new FuseNoSuchFileOrDirectoryError(path)); + } + + const chunk = await deps.readTemporalFileChunk(temporalFile.path.value, length, position); + return right(chunk ?? EMPTY); +} -async function startHydration( +async function ensureFileAllocated(filePath: string, virtualFile: File): Promise { + const allocated = await fileExistsOnDisk(filePath); + if (!allocated) { + await allocateFile(filePath, virtualFile.size); + startStopwatch(virtualFile.contentsId); + } + return getOrInitHydrationState(virtualFile.contentsId, virtualFile.size); +} + +async function ensureRangeDownloaded( deps: HandleReadCallbackDeps, virtualFile: File, filePath: string, -): Promise { - const { stream, elapsedTime } = await deps.startDownload(virtualFile); - const writer = createDownloadToDisk(stream, filePath, { - onProgress: (bytesWritten) => { - deps.onDownloadProgress(virtualFile.name, virtualFile.type, { - percentage: Math.min(bytesWritten / virtualFile.size, 1), - elapsedTime: elapsedTime(), - }); - }, - onFinished: () => { - deleteHydration(virtualFile.contentsId); - deps.saveToRepository( - virtualFile.contentsId, - virtualFile.size, - virtualFile.uuid, - virtualFile.name, - virtualFile.type, - ); - }, - onError: (err) => { - logger.error({ msg: '[startHydration] onError', error: err }); - tryCatch(() => writer.destroy()); - deleteHydration(virtualFile.contentsId); - }, - }); + state: FileHydrationState, + position: number, + length: number, +): Promise { + const { blockStart, blockLength } = expandToBlockBoundaries(position, length, virtualFile.size); + + const blocksBeingDownloaded = getBlocksBeingDownloaded(state, { position: blockStart, length: blockLength }); + if (blocksBeingDownloaded.size > 0) { + logger.debug({ msg: '[ReadCallback] waiting for blocks being downloaded', file: virtualFile.nameWithExtension }); + await Promise.all(blocksBeingDownloaded.values()); + } - setHydration(virtualFile.contentsId, { writer }); - return { writer }; + const missingBlocks = getMissingBlocks(state, { position: blockStart, length: blockLength }); + if (missingBlocks.length > 0) { + logger.debug({ msg: '[ReadCallback] downloading missing blocks', file: virtualFile.nameWithExtension, blocks: missingBlocks }); + await Promise.all( + missingBlocks.map((block) => { + const start = block * BLOCK_SIZE; + const end = Math.min(start + BLOCK_SIZE, virtualFile.size); + return downloadAndCacheBlock(deps, virtualFile, filePath, state, start, end - start); + }), + ); + } +} + +async function onFileFullyHydrated(deps: HandleReadCallbackDeps, virtualFile: File): Promise { + deleteStopwatch(virtualFile.contentsId); + await deps.saveToRepository( + virtualFile.contentsId, + virtualFile.size, + virtualFile.uuid, + virtualFile.name, + virtualFile.type, + ); } + export async function handleReadCallback( deps: HandleReadCallbackDeps, path: string, @@ -67,15 +117,7 @@ export async function handleReadCallback( const virtualFile = await deps.findVirtualFile(path); if (!virtualFile) { - const temporalFile = await deps.findTemporalFile(path); - - if (!temporalFile) { - logger.error({ msg: '[ReadCallback] File not found', path }); - return left(new FuseNoSuchFileOrDirectoryError(path)); - } - - const chunk = await deps.readTemporalFileChunk(temporalFile.path.value, length, position); - return right(chunk ?? EMPTY); + return readFromTemporalFile(deps, path, length, position); } if (!shouldDownload(path)) { @@ -83,46 +125,26 @@ export async function handleReadCallback( return right(EMPTY); } - const filePath = nodePath.join(PATHS.DOWNLOADED, virtualFile.contentsId); - logger.debug({ msg: '[ReadCallback] read request:', file: virtualFile.nameWithExtension, - position, - length, - targetByte: position + length, + position: formatBytes(position), + length: formatBytes(length), }); - if (await deps.existsOnDisk(virtualFile.contentsId)) { - const chunk = await readChunkFromDisk(filePath, length, position); - return right(chunk); - } + const filePath = nodePath.join(PATHS.DOWNLOADED, virtualFile.contentsId); + const state = await ensureFileAllocated(filePath, virtualFile); - const hydration = getHydration(virtualFile.contentsId) ?? (await startHydration(deps, virtualFile, filePath)); - const targetByte = position + length; - const bytesAvailable = hydration.writer.getBytesAvailable(); - const waitStart = Date.now(); - - if (bytesAvailable < targetByte) { - logger.debug({ - msg: '[ReadCallback] waiting for download to catch up', - file: virtualFile.nameWithExtension, - position: formatBytes(position), - targetByte: formatBytes(targetByte), - bytesAvailable: formatBytes(bytesAvailable), - bytesAhead: formatBytes(targetByte - bytesAvailable), - }); + if (isRangeHydrated(state, { position, length })) { + logger.debug({ msg: '[ReadCallback] serving from disk cache', file: virtualFile.nameWithExtension }); + return right(await readChunkFromDisk(filePath, length, position)); } - await hydration.writer.waitForBytes(position, length); + await ensureRangeDownloaded(deps, virtualFile, filePath, state, position, length); - logger.debug({ - msg: '[ReadCallback] wait resolved', - file: virtualFile.nameWithExtension, - position: formatBytes(position), - waitedMs: Date.now() - waitStart, - }); + if (isFileHydrated(state)) { + await onFileFullyHydrated(deps, virtualFile); + } - const chunk = await readChunkFromDisk(filePath, length, position); - return right(chunk); + return right(await readChunkFromDisk(filePath, length, position)); } diff --git a/src/backend/features/fuse/on-read/read-chunk-from-disk.ts b/src/backend/features/fuse/on-read/read-chunk-from-disk.ts index c8e41fd3b6..7a35678d7d 100644 --- a/src/backend/features/fuse/on-read/read-chunk-from-disk.ts +++ b/src/backend/features/fuse/on-read/read-chunk-from-disk.ts @@ -1,5 +1,14 @@ import fs from 'node:fs/promises'; +export async function writeChunkToDisk(filePath: string, buffer: Buffer, position: number): Promise { + const handle = await fs.open(filePath, 'r+'); + try { + await handle.write(new Uint8Array(buffer), 0, buffer.length, position); + } finally { + await handle.close(); + } +} + export async function readChunkFromDisk(filePath: string, length: number, position: number): Promise { const handle = await fs.open(filePath, 'r'); diff --git a/src/infra/environment/download-file/build-network-client.ts b/src/infra/environment/download-file/build-network-client.ts new file mode 100644 index 0000000000..43414b2308 --- /dev/null +++ b/src/infra/environment/download-file/build-network-client.ts @@ -0,0 +1,23 @@ +import { Network } from '@internxt/sdk'; +import { createHash } from 'node:crypto'; +import { INTERNXT_CLIENT, INTERNXT_VERSION } from '../../../core/utils/utils'; + +export type NetworkClientCredentials = { + bridgeUser: string; + userId: string; +}; + +export function buildNetworkClient(credentials: NetworkClientCredentials): Network.Network { + return Network.Network.client( + process.env.BRIDGE_URL as string, + { + clientName: INTERNXT_CLIENT, + clientVersion: INTERNXT_VERSION, + desktopHeader: process.env.INTERNXT_DESKTOP_HEADER_KEY, + }, + { + bridgeUser: credentials.bridgeUser, + userId: createHash('sha256').update(credentials.userId).digest('hex'), + }, + ); +} From ce21e10b44ef008f0f84bb28bf1aa6091fe6c72b Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Wed, 1 Apr 2026 18:17:45 +0200 Subject: [PATCH 03/10] fix:format --- src/apps/drive/index.ts | 2 +- src/backend/features/fuse/on-read/handle-read-callback.ts | 6 +++++- src/infra/environment/download-file/decrypt-at-offset.ts | 4 +--- src/infra/environment/download-file/download-file.ts | 2 -- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/apps/drive/index.ts b/src/apps/drive/index.ts index 55922fb7ab..2d951503b1 100644 --- a/src/apps/drive/index.ts +++ b/src/apps/drive/index.ts @@ -34,7 +34,7 @@ export async function startVirtualDrive() { * If a user abruptly quits the app, all the hydrated files will be orphaned. * Hence why we clear the cache before starting up the virtual drive. * To ensure that every time we get a fresh start. - */ + */ fuseApp.clearCache(); await fuseApp.start(); } diff --git a/src/backend/features/fuse/on-read/handle-read-callback.ts b/src/backend/features/fuse/on-read/handle-read-callback.ts index 2c6ce1b337..e3c2eab1a4 100644 --- a/src/backend/features/fuse/on-read/handle-read-callback.ts +++ b/src/backend/features/fuse/on-read/handle-read-callback.ts @@ -86,7 +86,11 @@ async function ensureRangeDownloaded( const missingBlocks = getMissingBlocks(state, { position: blockStart, length: blockLength }); if (missingBlocks.length > 0) { - logger.debug({ msg: '[ReadCallback] downloading missing blocks', file: virtualFile.nameWithExtension, blocks: missingBlocks }); + logger.debug({ + msg: '[ReadCallback] downloading missing blocks', + file: virtualFile.nameWithExtension, + blocks: missingBlocks, + }); await Promise.all( missingBlocks.map((block) => { const start = block * BLOCK_SIZE; diff --git a/src/infra/environment/download-file/decrypt-at-offset.ts b/src/infra/environment/download-file/decrypt-at-offset.ts index 2d38222108..22028ea459 100644 --- a/src/infra/environment/download-file/decrypt-at-offset.ts +++ b/src/infra/environment/download-file/decrypt-at-offset.ts @@ -24,9 +24,7 @@ export function decryptAtOffset(encryptedBytes: Buffer, key: Buffer, iv: Buffer, const startBlockNumber = (position - partialBlock) / AES_BLOCK_SIZE; // Compute the IV for the starting block by adding the block number to the original IV - const ivForRange = (BigInt('0x' + iv.toString('hex')) + BigInt(startBlockNumber)) - .toString(16) - .padStart(32, '0'); + const ivForRange = (BigInt('0x' + iv.toString('hex')) + BigInt(startBlockNumber)).toString(16).padStart(32, '0'); const offsetIv = Buffer.from(ivForRange, 'hex'); const decipher = createDecipheriv('aes-256-ctr', new Uint8Array(key), new Uint8Array(offsetIv)); diff --git a/src/infra/environment/download-file/download-file.ts b/src/infra/environment/download-file/download-file.ts index 6b702da1c1..65bed096e4 100644 --- a/src/infra/environment/download-file/download-file.ts +++ b/src/infra/environment/download-file/download-file.ts @@ -22,8 +22,6 @@ async function fetchEncryptedRange(url: string, position: number, length: number }); } - - export async function downloadFileRange({ signal, fileId, From 7517346220c76f7d668c5e822fdbc48e81f57bc7 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Fri, 1 May 2026 14:08:38 +0200 Subject: [PATCH 04/10] fix merge issues --- .../fuse/on-read/handle-read-callback.ts | 16 +++++----- .../services/operations/read.service.ts | 29 +++++++++---------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/src/backend/features/fuse/on-read/handle-read-callback.ts b/src/backend/features/fuse/on-read/handle-read-callback.ts index f568b12077..54d461c54b 100644 --- a/src/backend/features/fuse/on-read/handle-read-callback.ts +++ b/src/backend/features/fuse/on-read/handle-read-callback.ts @@ -28,7 +28,7 @@ import { EMPTY } from './constants'; export type HandleReadCallbackDeps = { findVirtualFile: (path: string) => Promise; findTemporalFile: (path: string) => Promise; - readTemporalFileChunk: (path: string, length: number, position: number) => Promise; + // readTemporalFileChunk: (path: string, length: number, position: number) => Promise; onDownloadProgress: ( name: string, extension: string, @@ -50,12 +50,12 @@ async function readFromTemporalFile( ): Promise> { const temporalFile = await deps.findTemporalFile(path); - if (!temporalFile) { + if (!temporalFile || !temporalFile.contentFilePath) { logger.error({ msg: '[ReadCallback] File not found', path }); return { error: new FuseNoSuchFileOrDirectoryError(path) }; } - const chunk = await deps.readTemporalFileChunk(temporalFile.path.value, length, position); + const chunk = await readChunkFromDisk(temporalFile.contentFilePath, length, position); return { data: chunk ?? EMPTY }; } @@ -131,7 +131,10 @@ export async function handleReadCallback( position: formatBytes(position), length: formatBytes(length), }); - + if (isBlocklistedProcess(processName)) { + logger.debug({ msg: '[ReadCallback] Download blocked - blocklisted process', path, processName }); + return { data: EMPTY }; + } const filePath = nodePath.join(PATHS.DOWNLOADED, virtualFile.contentsId); const state = await ensureFileAllocated(filePath, virtualFile); @@ -144,11 +147,6 @@ export async function handleReadCallback( logger.debug({ msg: '[ReadCallback] serving from disk cache', file: virtualFile.nameWithExtension }); return { data: await readChunkFromDisk(filePath, length, position) }; } - - if (isBlocklistedProcess(processName)) { - logger.debug({ msg: '[ReadCallback] Download blocked - blocklisted process', path, processName }); - return { data: EMPTY }; - } await ensureRangeDownloaded(deps, virtualFile, filePath, state, position, length); diff --git a/src/backend/features/virtual-drive/services/operations/read.service.ts b/src/backend/features/virtual-drive/services/operations/read.service.ts index f9d0426c3b..643cbb0a60 100644 --- a/src/backend/features/virtual-drive/services/operations/read.service.ts +++ b/src/backend/features/virtual-drive/services/operations/read.service.ts @@ -5,13 +5,13 @@ import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; import { FirstsFileSearcher } from '../../../../../context/virtual-drive/files/application/search/FirstsFileSearcher'; import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; import { StorageFilesRepository } from '../../../../../context/storage/StorageFiles/domain/StorageFilesRepository'; -import { StorageFileId } from '../../../../../context/storage/StorageFiles/domain/StorageFileId'; import { StorageFile } from '../../../../../context/storage/StorageFiles/domain/StorageFile'; -import { StorageFileDownloader } from '../../../../../context/storage/StorageFiles/application/download/StorageFileDownloader/StorageFileDownloader'; import { DownloadProgressTracker } from '../../../../../context/shared/domain/DownloadProgressTracker'; import { handleReadCallback } from '../../../../features/fuse/on-read/handle-read-callback'; import { logger } from '@internxt/drive-desktop-core/build/backend'; -import { type File } from '../../../../../context/virtual-drive/files/domain/File'; +import { getCredentials } from '../../../../../apps/main/auth/get-credentials'; +import { DependencyInjectionUserProvider } from '../../../../../apps/shared/dependency-injection/DependencyInjectionUserProvider'; +import { buildNetworkClient } from '../../../../../infra/environment/download-file/build-network-client'; export async function read( path: string, @@ -21,27 +21,21 @@ export async function read( container: Container, ): Promise> { try { + const { mnemonic } = getCredentials(); + const user = DependencyInjectionUserProvider.get(); + const network = buildNetworkClient({ bridgeUser: user.bridgeUser, userId: user.userId }); const repo = container.get(StorageFilesRepository); - const downloader = container.get(StorageFileDownloader); const tracker = container.get(DownloadProgressTracker); return await handleReadCallback( { findVirtualFile: (p) => container.get(FirstsFileSearcher).run({ path: p }), findTemporalFile: (p) => container.get(TemporalFileByPathFinder).run(p), - existsOnDisk: (contentsId) => repo.exists(new StorageFileId(contentsId)), - startDownload: async (virtualFile: File) => { - const storage = StorageFile.from({ - id: virtualFile.contentsId, - virtualId: virtualFile.uuid, - size: virtualFile.size, + onDownloadProgress: (name, extension, bytesDownloaded, fileSize, elapsedTime) => { + tracker.downloadUpdate(name, extension, { + percentage: Math.min(bytesDownloaded / fileSize, 1), + elapsedTime, }); - tracker.downloadStarted(virtualFile.name, virtualFile.type); - const { stream, handler } = await downloader.run(storage, virtualFile); - return { stream, elapsedTime: () => handler.elapsedTime() }; - }, - onDownloadProgress: (name, extension, progress) => { - tracker.downloadUpdate(name, extension, progress); }, saveToRepository: async (contentsId, size, uuid, name, extension) => { const storage = StorageFile.from({ @@ -52,6 +46,9 @@ export async function read( await repo.register(storage); tracker.downloadFinished(name, extension); }, + bucketId: user.bucket, + mnemonic, + network, }, path, length, From a9e7c08fe246b291d7cfd016a1f581bb22e87299 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Mon, 4 May 2026 13:47:26 +0200 Subject: [PATCH 05/10] feat: update eslint, update props from handleReadCallback, clearhydrationState before mounting fuse --- .eslintrc.js | 2 +- .gitignore | 1 + .../download-cache/download-and-save-block.ts | 46 +++-- .../fuse/on-read/handle-read-callback.ts | 165 +++++++++++------- .../services/operations/read.service.ts | 44 +++-- .../services/virtual-drive.service.ts | 5 +- 6 files changed, 158 insertions(+), 105 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index a81e34f4df..4970826224 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -14,7 +14,7 @@ module.exports = { ], rules: { 'no-await-in-loop': 'warn', - 'no-use-before-define': 'warn', + '@typescript-eslint/no-use-before-define': ['warn', { functions: false, classes: true, variables: true }], 'array-callback-return': 'warn', 'max-len': [ 'warn', // TODO: Change back to 'error' after fixing existing violations diff --git a/.gitignore b/.gitignore index a8204048d8..f938b9ca26 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,4 @@ enable-sso.sh .github/copilot-instructions.md .github/instructions/contributing.instructions.md .github/instructions/testing.instructions.md +.codex diff --git a/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts b/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts index e799cbd5cb..cfcf787e50 100644 --- a/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts +++ b/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts @@ -1,41 +1,49 @@ -import { HandleReadCallbackDeps } from '../handle-read-callback'; +import { type HandleReadCallbackProps } from '../handle-read-callback'; import { writeChunkToDisk } from '../read-chunk-from-disk'; import { type FileHydrationState, markBlocksInRangeDownloaded, startBlockDownload } from './hydration-state'; import { type File } from '../../../../../context/virtual-drive/files/domain/File'; import { downloadFileRange } from '../../../../../infra/environment/download-file/download-file'; import { getStopwatch } from './hydration-stopwatch'; +type Props = { + bucketId: HandleReadCallbackProps['bucketId']; + mnemonic: HandleReadCallbackProps['mnemonic']; + network: HandleReadCallbackProps['network']; + onDownloadProgress: HandleReadCallbackProps['onDownloadProgress']; + virtualFile: File; + filePath: string; + state: FileHydrationState; + blockStart: number; + blockLength: number; +}; /** * Downloads a block range, writes it to disk at the correct offset, and marks it as downloaded. */ -export async function downloadAndCacheBlock( - deps: HandleReadCallbackDeps, - virtualFile: File, - filePath: string, - state: FileHydrationState, - blockStart: number, - blockLength: number, -): Promise { +export async function downloadAndCacheBlock({ + bucketId, + mnemonic, + network, + onDownloadProgress, + virtualFile, + filePath, + state, + blockStart, + blockLength, +}: Props): Promise { const resolve = startBlockDownload(state, { position: blockStart, length: blockLength }); try { const buffer = await downloadFileRange({ fileId: virtualFile.contentsId, - bucketId: deps.bucketId, - mnemonic: deps.mnemonic, - network: deps.network, + bucketId, + mnemonic, + network, range: { position: blockStart, length: blockLength }, signal: new AbortController(), }); await writeChunkToDisk(filePath, buffer, blockStart); markBlocksInRangeDownloaded(state, { position: blockStart, length: blockLength }); const elapsedTime = getStopwatch(virtualFile.contentsId)?.elapsedTime() ?? 0; - deps.onDownloadProgress( - virtualFile.name, - virtualFile.type, - blockStart + blockLength, - virtualFile.size, - elapsedTime, - ); + onDownloadProgress(virtualFile.name, virtualFile.type, blockStart + blockLength, virtualFile.size, elapsedTime); } finally { resolve(); } diff --git a/src/backend/features/fuse/on-read/handle-read-callback.ts b/src/backend/features/fuse/on-read/handle-read-callback.ts index 54d461c54b..63bbae1935 100644 --- a/src/backend/features/fuse/on-read/handle-read-callback.ts +++ b/src/backend/features/fuse/on-read/handle-read-callback.ts @@ -24,8 +24,7 @@ import { startStopwatch, deleteStopwatch } from './download-cache/hydration-stop import { fileExistsOnDisk } from './download-cache/file-exists-on-disk'; import { downloadAndCacheBlock } from './download-cache/download-and-save-block'; import { EMPTY } from './constants'; - -export type HandleReadCallbackDeps = { +export type HandleReadCallbackProps = { findVirtualFile: (path: string) => Promise; findTemporalFile: (path: string) => Promise; // readTemporalFileChunk: (path: string, length: number, position: number) => Promise; @@ -40,15 +39,80 @@ export type HandleReadCallbackDeps = { bucketId: string; mnemonic: string; network: Network.Network; + path: string; + length: number; + position: number; + processName: string; }; +export async function handleReadCallback({ + findVirtualFile, + findTemporalFile, + onDownloadProgress, + saveToRepository, + bucketId, + mnemonic, + network, + path, + length, + position, + processName, +}: HandleReadCallbackProps): Promise> { + const virtualFile = await findVirtualFile(path); + + if (!virtualFile) { + return readFromTemporalFile(findTemporalFile, path, length, position); + } + + logger.debug({ + msg: '[ReadCallback] read request:', + file: virtualFile.nameWithExtension, + position: formatBytes(position), + length: formatBytes(length), + }); + if (isBlocklistedProcess(processName)) { + logger.debug({ msg: '[ReadCallback] Download blocked - blocklisted process', path, processName }); + return { data: EMPTY }; + } + const filePath = nodePath.join(PATHS.DOWNLOADED, virtualFile.contentsId); + const state = await ensureFileAllocated(filePath, virtualFile); + + if (isRangeHydrated(state, { position, length })) { + if (isBlocklistedProcess(processName)) { + logger.debug({ + msg: `[ReadCallback] Allowing read from disk for blocklisted process: ${processName} in ${path}`, + }); + } + logger.debug({ msg: '[ReadCallback] serving from disk cache', file: virtualFile.nameWithExtension }); + return { data: await readChunkFromDisk(filePath, length, position) }; + } + + await ensureRangeDownloaded({ + bucketId, + mnemonic, + network, + onDownloadProgress, + virtualFile, + filePath, + state, + position, + length, + }); + + if (isFileHydrated(state)) { + await onFileFullyHydrated(saveToRepository, virtualFile); + } + + return { data: await readChunkFromDisk(filePath, length, position) }; +} + async function readFromTemporalFile( - deps: HandleReadCallbackDeps, + findTemporalFile: HandleReadCallbackProps['findTemporalFile'], path: string, length: number, position: number, ): Promise> { - const temporalFile = await deps.findTemporalFile(path); + const temporalFile = await findTemporalFile(path); if (!temporalFile || !temporalFile.contentFilePath) { logger.error({ msg: '[ReadCallback] File not found', path }); @@ -68,14 +132,27 @@ async function ensureFileAllocated(filePath: string, virtualFile: File): Promise return getOrInitHydrationState(virtualFile.contentsId, virtualFile.size); } -async function ensureRangeDownloaded( - deps: HandleReadCallbackDeps, - virtualFile: File, - filePath: string, - state: FileHydrationState, - position: number, - length: number, -): Promise { +async function ensureRangeDownloaded({ + bucketId, + mnemonic, + network, + onDownloadProgress, + virtualFile, + filePath, + state, + position, + length, +}: { + bucketId: HandleReadCallbackProps['bucketId']; + mnemonic: HandleReadCallbackProps['mnemonic']; + network: HandleReadCallbackProps['network']; + onDownloadProgress: HandleReadCallbackProps['onDownloadProgress']; + virtualFile: File; + filePath: string; + state: FileHydrationState; + position: number; + length: number; +}): Promise { const { blockStart, blockLength } = expandToBlockBoundaries(position, length, virtualFile.size); const blocksBeingDownloaded = getBlocksBeingDownloaded(state, { position: blockStart, length: blockLength }); @@ -95,15 +172,28 @@ async function ensureRangeDownloaded( missingBlocks.map((block) => { const start = block * BLOCK_SIZE; const end = Math.min(start + BLOCK_SIZE, virtualFile.size); - return downloadAndCacheBlock(deps, virtualFile, filePath, state, start, end - start); + return downloadAndCacheBlock({ + bucketId, + mnemonic, + network, + onDownloadProgress, + virtualFile, + filePath, + state, + blockStart: start, + blockLength: end - start, + }); }), ); } } -async function onFileFullyHydrated(deps: HandleReadCallbackDeps, virtualFile: File): Promise { +async function onFileFullyHydrated( + saveToRepository: HandleReadCallbackProps['saveToRepository'], + virtualFile: File, +): Promise { deleteStopwatch(virtualFile.contentsId); - await deps.saveToRepository( + await saveToRepository( virtualFile.contentsId, virtualFile.size, virtualFile.uuid, @@ -111,48 +201,3 @@ async function onFileFullyHydrated(deps: HandleReadCallbackDeps, virtualFile: Fi virtualFile.type, ); } - -export async function handleReadCallback( - deps: HandleReadCallbackDeps, - path: string, - length: number, - position: number, - processName: string, -): Promise> { - const virtualFile = await deps.findVirtualFile(path); - - if (!virtualFile) { - return readFromTemporalFile(deps, path, length, position); - } - - logger.debug({ - msg: '[ReadCallback] read request:', - file: virtualFile.nameWithExtension, - position: formatBytes(position), - length: formatBytes(length), - }); - if (isBlocklistedProcess(processName)) { - logger.debug({ msg: '[ReadCallback] Download blocked - blocklisted process', path, processName }); - return { data: EMPTY }; - } - const filePath = nodePath.join(PATHS.DOWNLOADED, virtualFile.contentsId); - const state = await ensureFileAllocated(filePath, virtualFile); - - if (isRangeHydrated(state, { position, length })) { - if (isBlocklistedProcess(processName)) { - logger.debug({ - msg: `[ReadCallback] Allowing read from disk for blocklisted process: ${processName} in ${path}`, - }); - } - logger.debug({ msg: '[ReadCallback] serving from disk cache', file: virtualFile.nameWithExtension }); - return { data: await readChunkFromDisk(filePath, length, position) }; - } - - await ensureRangeDownloaded(deps, virtualFile, filePath, state, position, length); - - if (isFileHydrated(state)) { - await onFileFullyHydrated(deps, virtualFile); - } - - return { data: await readChunkFromDisk(filePath, length, position) }; -} diff --git a/src/backend/features/virtual-drive/services/operations/read.service.ts b/src/backend/features/virtual-drive/services/operations/read.service.ts index 643cbb0a60..018f6afdf7 100644 --- a/src/backend/features/virtual-drive/services/operations/read.service.ts +++ b/src/backend/features/virtual-drive/services/operations/read.service.ts @@ -27,34 +27,32 @@ export async function read( const repo = container.get(StorageFilesRepository); const tracker = container.get(DownloadProgressTracker); - return await handleReadCallback( - { - findVirtualFile: (p) => container.get(FirstsFileSearcher).run({ path: p }), - findTemporalFile: (p) => container.get(TemporalFileByPathFinder).run(p), - onDownloadProgress: (name, extension, bytesDownloaded, fileSize, elapsedTime) => { - tracker.downloadUpdate(name, extension, { - percentage: Math.min(bytesDownloaded / fileSize, 1), - elapsedTime, - }); - }, - saveToRepository: async (contentsId, size, uuid, name, extension) => { - const storage = StorageFile.from({ - id: contentsId, - virtualId: uuid, - size, - }); - await repo.register(storage); - tracker.downloadFinished(name, extension); - }, - bucketId: user.bucket, - mnemonic, - network, + return await handleReadCallback({ + findVirtualFile: (p) => container.get(FirstsFileSearcher).run({ path: p }), + findTemporalFile: (p) => container.get(TemporalFileByPathFinder).run(p), + onDownloadProgress: (name, extension, bytesDownloaded, fileSize, elapsedTime) => { + tracker.downloadUpdate(name, extension, { + percentage: Math.min(bytesDownloaded / fileSize, 1), + elapsedTime, + }); }, + saveToRepository: async (contentsId, size, uuid, name, extension) => { + const storage = StorageFile.from({ + id: contentsId, + virtualId: uuid, + size, + }); + await repo.register(storage); + tracker.downloadFinished(name, extension); + }, + bucketId: user.bucket, + mnemonic, + network, path, length, position, processName, - ); + }); } catch (err) { logger.error({ msg: '[FUSE - Read] Unexpected error', error: err, path }); return { error: new FuseError(FuseCodes.EIO, `[FUSE - Read] IO error: ${path}`) }; diff --git a/src/backend/features/virtual-drive/services/virtual-drive.service.ts b/src/backend/features/virtual-drive/services/virtual-drive.service.ts index 7865dcbb09..122225ebfc 100644 --- a/src/backend/features/virtual-drive/services/virtual-drive.service.ts +++ b/src/backend/features/virtual-drive/services/virtual-drive.service.ts @@ -7,7 +7,7 @@ import { startFuseDaemonServer, stopFuseDaemonServer } from './server.service'; import { updateVirtualDriveContainer } from './update-virtual-drive-container.service'; import { DependencyInjectionUserProvider } from '../../../../apps/shared/dependency-injection/DependencyInjectionUserProvider'; import { StorageClearer } from '../../../../context/storage/StorageFiles/application/delete/StorageClearer'; -import { destroyAllHydrations } from '../../fuse/on-read/hydration-registry'; +import { clearHydrationState } from '../../fuse/on-read/download-cache/hydration-state'; let container: Container | undefined; @@ -26,6 +26,7 @@ export async function startVirtualDrive() { * Hence why we clear the cache before starting up the virtual drive. * To ensure that every time we get a fresh start. */ + clearHydrationState(); await container.get(StorageClearer).run(); await startFuseDaemonServer(container); await startDaemon(localRoot); @@ -35,7 +36,7 @@ export async function stopVirtualDrive() { logger.debug({ msg: '[VIRTUAL DRIVE] stopping daemon...' }); await stopDaemon(); logger.debug({ msg: '[VIRTUAL DRIVE] clearing storage cache...' }); - await destroyAllHydrations(); + clearHydrationState(); if (container) { await container.get(StorageClearer).run(); } From 4a92c52d0a5fd839c69a93bf28e185f5396e0c51 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Mon, 4 May 2026 13:47:58 +0200 Subject: [PATCH 06/10] chore: change legacy fuseApp code so ts does not complain --- src/apps/drive/fuse/FuseApp.test.ts | 9 ++++----- src/apps/drive/fuse/FuseApp.ts | 4 ++-- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/apps/drive/fuse/FuseApp.test.ts b/src/apps/drive/fuse/FuseApp.test.ts index bfa9c945ef..7823e023c5 100644 --- a/src/apps/drive/fuse/FuseApp.test.ts +++ b/src/apps/drive/fuse/FuseApp.test.ts @@ -7,7 +7,7 @@ import { FolderRepositorySynchronizer } from '../../../context/virtual-drive/fol import { RemoteTreeBuilder } from '../../../context/virtual-drive/remoteTree/application/RemoteTreeBuilder'; import { StorageRemoteChangesSyncher } from '../../../context/storage/StorageFiles/application/sync/StorageRemoteChangesSyncher'; import * as helpersModule from './helpers'; -import * as hydrationModule from '../../../backend/features/fuse/on-read/hydration-registry'; +import * as hydrationStateModule from '../../../backend/features/fuse/on-read/download-cache/hydration-state'; import * as childProcess from 'child_process'; import { partialSpyOn } from 'tests/vitest/utils.helper'; import { loggerMock } from 'tests/vitest/mocks.helper'; @@ -21,7 +21,7 @@ vi.mock('child_process', () => ({ })); const mountPromiseMock = partialSpyOn(helpersModule, 'mountPromise'); -const destroyAllHydrationsMock = partialSpyOn(hydrationModule, 'destroyAllHydrations'); +const clearHydrationStateMock = partialSpyOn(hydrationStateModule, 'clearHydrationState'); const execFileMock = vi.mocked(childProcess.execFile); function createMockContainer() { @@ -208,14 +208,13 @@ describe.skip('FuseApp', () => { }); describe('clearCache', () => { - it('should destroy hydrations and clear storage', async () => { + it('should clear hydration state and storage', async () => { const storageClearer = { run: vi.fn().mockResolvedValue(undefined) }; register(StorageClearer, storageClearer); - destroyAllHydrationsMock.mockResolvedValue(undefined); await fuseApp.clearCache(); - expect(destroyAllHydrationsMock).toHaveBeenCalled(); + expect(clearHydrationStateMock).toHaveBeenCalled(); expect(storageClearer.run).toHaveBeenCalled(); }); }); diff --git a/src/apps/drive/fuse/FuseApp.ts b/src/apps/drive/fuse/FuseApp.ts index b4bfe9b6e4..09ff0731de 100644 --- a/src/apps/drive/fuse/FuseApp.ts +++ b/src/apps/drive/fuse/FuseApp.ts @@ -1,7 +1,7 @@ import { Container } from 'diod'; import { logger } from '@internxt/drive-desktop-core/build/backend'; import { StorageClearer } from '../../../context/storage/StorageFiles/application/delete/StorageClearer'; -import { destroyAllHydrations } from '../../../backend/features/fuse/on-read/hydration-registry'; +import { clearHydrationState } from '../../../backend/features/fuse/on-read/download-cache/hydration-state'; import { VirtualDrive } from '../virtual-drive/VirtualDrive'; import { FuseDriveStatus } from './FuseDriveStatus'; import { CreateCallback } from './callbacks/CreateCallback'; @@ -120,7 +120,7 @@ export class FuseApp extends EventEmitter { } async clearCache(): Promise { - await destroyAllHydrations(); + clearHydrationState(); await this.container.get(StorageClearer).run(); } From 6b79c9559cbe535b3e3dc4d9f09e3971b610d1f7 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Tue, 5 May 2026 19:30:26 +0200 Subject: [PATCH 07/10] feat: allow ranged requests --- .../download-cache/allocate-file.test.ts | 58 +++ .../download-and-save-block.test.ts | 215 ++++++++ .../download-cache/download-and-save-block.ts | 50 +- .../expand-to-block-boundaries.test.ts | 43 ++ .../expand-to-block-boundaries.ts | 14 +- .../file-exists-on-disk.test.ts | 30 ++ .../download-cache/hydration-state.test.ts | 233 +++++++++ .../on-read/download-cache/hydration-state.ts | 154 ++++-- .../download-cache/read-if-hydrated.test.ts | 54 ++ .../download-cache/read-if-hydrated.ts | 19 + .../fuse/on-read/handle-read-callback.test.ts | 157 +++--- .../fuse/on-read/handle-read-callback.ts | 168 +----- .../fuse/on-read/read-chunk-from-disk.ts | 2 +- .../fuse/on-read/read-or-hydrate.test.ts | 493 ++++++++++++++++++ .../features/fuse/on-read/read-or-hydrate.ts | 198 +++++++ src/backend/features/fuse/on-read/types.ts | 20 + .../services/operations/read.service.test.ts | 32 +- .../services/operations/read.service.ts | 6 +- .../services/virtual-drive.service.test.ts | 48 ++ .../services/virtual-drive.service.ts | 10 +- .../download-file/build-crypto-lib.test.ts | 65 +++ .../build-network-client.test.ts | 43 ++ .../download-file/decrypt-at-offset.test.ts | 40 ++ .../download-file/download-file.test.ts | 71 +++ .../download-file/download-file.ts | 93 ++-- src/infra/environment/download-file/types.ts | 2 +- 26 files changed, 2008 insertions(+), 310 deletions(-) create mode 100644 src/backend/features/fuse/on-read/download-cache/allocate-file.test.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/download-and-save-block.test.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/expand-to-block-boundaries.test.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/file-exists-on-disk.test.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/hydration-state.test.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/read-if-hydrated.test.ts create mode 100644 src/backend/features/fuse/on-read/download-cache/read-if-hydrated.ts create mode 100644 src/backend/features/fuse/on-read/read-or-hydrate.test.ts create mode 100644 src/backend/features/fuse/on-read/read-or-hydrate.ts create mode 100644 src/backend/features/fuse/on-read/types.ts create mode 100644 src/backend/features/virtual-drive/services/virtual-drive.service.test.ts create mode 100644 src/infra/environment/download-file/build-crypto-lib.test.ts create mode 100644 src/infra/environment/download-file/build-network-client.test.ts create mode 100644 src/infra/environment/download-file/decrypt-at-offset.test.ts create mode 100644 src/infra/environment/download-file/download-file.test.ts diff --git a/src/backend/features/fuse/on-read/download-cache/allocate-file.test.ts b/src/backend/features/fuse/on-read/download-cache/allocate-file.test.ts new file mode 100644 index 0000000000..c4887aa787 --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/allocate-file.test.ts @@ -0,0 +1,58 @@ +import fs from 'node:fs/promises'; +import { allocateFile } from './allocate-file'; + +vi.mock('node:fs/promises', () => ({ + default: { + open: vi.fn(), + }, +})); + +const fsMock = vi.mocked(fs); + +function createHandle() { + return { + truncate: vi.fn().mockResolvedValue(undefined), + close: vi.fn().mockResolvedValue(undefined), + }; +} + +describe('allocateFile', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('opens the file for writing and truncates it to the requested size', async () => { + const handle = createHandle(); + fsMock.open.mockResolvedValue(handle as unknown as Awaited>); + + await allocateFile('/tmp/cache-file', 1024); + + expect(fsMock.open).toHaveBeenCalledWith('/tmp/cache-file', 'w'); + expect(handle.truncate).toHaveBeenCalledWith(1024); + }); + + it('closes the file handle after successful allocation', async () => { + const handle = createHandle(); + fsMock.open.mockResolvedValue(handle as unknown as Awaited>); + + await allocateFile('/tmp/cache-file', 1024); + + expect(handle.close).toHaveBeenCalledOnce(); + }); + + it('closes the file handle when truncate fails', async () => { + const handle = createHandle(); + handle.truncate.mockRejectedValue(new Error('truncate failed')); + fsMock.open.mockResolvedValue(handle as unknown as Awaited>); + + await expect(allocateFile('/tmp/cache-file', 1024)).rejects.toThrow('truncate failed'); + + expect(handle.close).toHaveBeenCalledOnce(); + }); + + it('propagates open failures', async () => { + fsMock.open.mockRejectedValue(new Error('open failed')); + + await expect(allocateFile('/tmp/cache-file', 1024)).rejects.toThrow('open failed'); + }); +}); diff --git a/src/backend/features/fuse/on-read/download-cache/download-and-save-block.test.ts b/src/backend/features/fuse/on-read/download-cache/download-and-save-block.test.ts new file mode 100644 index 0000000000..dfe0c025e3 --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/download-and-save-block.test.ts @@ -0,0 +1,215 @@ +import { type File } from '../../../../../context/virtual-drive/files/domain/File'; +import { downloadFileRange } from '../../../../../infra/environment/download-file/download-file'; +import { writeChunkToDisk } from '../read-chunk-from-disk'; +import { BLOCK_SIZE } from './constants'; +import { downloadAndCacheBlock } from './download-and-save-block'; +import { + clearHydrationState, + getOrCreateHydrationState, + isRangeHydrated, + markBlocksInRangeDownloaded, + type FileHydrationState, +} from './hydration-state'; + +vi.mock('../../../../../infra/environment/download-file/download-file', () => ({ + downloadFileRange: vi.fn(), +})); + +vi.mock('../read-chunk-from-disk', () => ({ + writeChunkToDisk: vi.fn(), +})); + +const downloadFileRangeMock = vi.mocked(downloadFileRange); +const writeChunkToDiskMock = vi.mocked(writeChunkToDisk); + +const virtualFile = { + contentsId: 'contents-id', + name: 'video', + nameWithExtension: 'video.mp4', + type: 'mp4', + uuid: 'uuid', + size: 1024, +} as unknown as File; + +function createState(): FileHydrationState { + return getOrCreateHydrationState(virtualFile.contentsId, virtualFile.size); +} + +function createVirtualFile(overrides: Partial = {}): File { + return { + ...virtualFile, + ...overrides, + } as File; +} + +function createProps(overrides: Partial[0]> = {}) { + return { + bucketId: 'bucket-id', + mnemonic: 'mnemonic', + network: {} as Parameters[0]['network'], + onDownloadProgress: vi.fn(), + virtualFile, + filePath: '/tmp/cache-file', + state: createState(), + blockStart: 100, + blockLength: 50, + ...overrides, + }; +} + +describe('downloadAndCacheBlock', () => { + beforeEach(() => { + clearHydrationState(); + vi.clearAllMocks(); + downloadFileRangeMock.mockResolvedValue({ data: Buffer.from('downloaded') }); + writeChunkToDiskMock.mockResolvedValue(undefined); + }); + + it('downloads the requested range and writes it to the cache file offset', async () => { + const props = createProps(); + + await downloadAndCacheBlock(props); + + expect(downloadFileRangeMock).toHaveBeenCalledWith({ + fileId: virtualFile.contentsId, + bucketId: props.bucketId, + mnemonic: props.mnemonic, + network: props.network, + range: { position: props.blockStart, length: props.blockLength }, + signal: props.state.abortController.signal, + }); + expect(writeChunkToDiskMock).toHaveBeenCalledWith('/tmp/cache-file', Buffer.from('downloaded'), 100); + }); + + it('marks the block hydrated only after download and disk write succeed', async () => { + const props = createProps(); + + await downloadAndCacheBlock(props); + + expect(isRangeHydrated(props.state, { position: props.blockStart, length: props.blockLength })).toBe(true); + }); + + it('emits progress from hydrated bytes after the block is written and marked hydrated', async () => { + const onDownloadProgress = vi.fn(); + const hydratedFile = createVirtualFile({ contentsId: 'first-block-file', size: BLOCK_SIZE * 2 }); + const state = getOrCreateHydrationState(hydratedFile.contentsId, hydratedFile.size); + state.stopwatch = { elapsedTime: vi.fn(() => 123) } as unknown as FileHydrationState['stopwatch']; + + await downloadAndCacheBlock( + createProps({ + state, + onDownloadProgress, + virtualFile: hydratedFile, + blockStart: 0, + blockLength: BLOCK_SIZE, + }), + ); + + expect(onDownloadProgress).toHaveBeenCalledWith('video', 'mp4', BLOCK_SIZE, hydratedFile.size, 123); + }); + + it('does not report full progress for a random EOF block when earlier blocks are missing', async () => { + const onDownloadProgress = vi.fn(); + const eofFile = createVirtualFile({ contentsId: 'eof-file', size: BLOCK_SIZE * 3 + 123 }); + const state = getOrCreateHydrationState(eofFile.contentsId, eofFile.size); + + await downloadAndCacheBlock( + createProps({ + state, + onDownloadProgress, + virtualFile: eofFile, + blockStart: BLOCK_SIZE * 3, + blockLength: 123, + }), + ); + + expect(onDownloadProgress).toHaveBeenCalledWith('video', 'mp4', 123, eofFile.size, 0); + }); + + it('counts the final block by its actual length', async () => { + const onDownloadProgress = vi.fn(); + const fileWithPartialFinalBlock = createVirtualFile({ + contentsId: 'partial-final-block-file', + size: BLOCK_SIZE + 123, + }); + const state = getOrCreateHydrationState(fileWithPartialFinalBlock.contentsId, fileWithPartialFinalBlock.size); + + await downloadAndCacheBlock( + createProps({ + state, + onDownloadProgress, + virtualFile: fileWithPartialFinalBlock, + blockStart: BLOCK_SIZE, + blockLength: 123, + }), + ); + + expect(onDownloadProgress).toHaveBeenCalledWith('video', 'mp4', 123, fileWithPartialFinalBlock.size, 0); + }); + + it('reports 100% progress when every block is hydrated', async () => { + const onDownloadProgress = vi.fn(); + const fullyHydratedFile = createVirtualFile({ contentsId: 'fully-hydrated-file', size: BLOCK_SIZE + 123 }); + const state = getOrCreateHydrationState(fullyHydratedFile.contentsId, fullyHydratedFile.size); + markBlocksInRangeDownloaded(state, { position: 0, length: BLOCK_SIZE }); + + await downloadAndCacheBlock( + createProps({ + state, + onDownloadProgress, + virtualFile: fullyHydratedFile, + blockStart: BLOCK_SIZE, + blockLength: 123, + }), + ); + + expect(onDownloadProgress).toHaveBeenCalledWith('video', 'mp4', fullyHydratedFile.size, fullyHydratedFile.size, 0); + }); + + it('does not write, mark hydrated, or emit progress when the range download fails', async () => { + const props = createProps(); + downloadFileRangeMock.mockResolvedValue({ error: new Error('network failed') }); + + await expect(downloadAndCacheBlock(props)).resolves.toStrictEqual({ error: new Error('network failed') }); + + expect(writeChunkToDiskMock).not.toHaveBeenCalled(); + expect(isRangeHydrated(props.state, { position: props.blockStart, length: props.blockLength })).toBe(false); + expect(props.onDownloadProgress).not.toHaveBeenCalled(); + }); + + it('does not mark hydrated or emit progress when the disk write fails', async () => { + const props = createProps(); + writeChunkToDiskMock.mockRejectedValue(new Error('write failed')); + + await expect(downloadAndCacheBlock(props)).resolves.toStrictEqual({ error: new Error('write failed') }); + + expect(isRangeHydrated(props.state, { position: props.blockStart, length: props.blockLength })).toBe(false); + expect(props.onDownloadProgress).not.toHaveBeenCalled(); + }); + + it('does not start a download when hydration is already aborted', async () => { + const props = createProps(); + props.state.abortController.abort(); + + await expect(downloadAndCacheBlock(props)).resolves.toStrictEqual({ data: undefined }); + + expect(downloadFileRangeMock).not.toHaveBeenCalled(); + expect(writeChunkToDiskMock).not.toHaveBeenCalled(); + expect(isRangeHydrated(props.state, { position: props.blockStart, length: props.blockLength })).toBe(false); + expect(props.onDownloadProgress).not.toHaveBeenCalled(); + }); + + it('does not write, mark hydrated, or emit progress when hydration aborts after download', async () => { + const props = createProps(); + downloadFileRangeMock.mockImplementation(async () => { + props.state.abortController.abort(); + return { data: Buffer.from('downloaded') }; + }); + + await expect(downloadAndCacheBlock(props)).resolves.toStrictEqual({ data: undefined }); + + expect(writeChunkToDiskMock).not.toHaveBeenCalled(); + expect(isRangeHydrated(props.state, { position: props.blockStart, length: props.blockLength })).toBe(false); + expect(props.onDownloadProgress).not.toHaveBeenCalled(); + }); +}); diff --git a/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts b/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts index cfcf787e50..ecbc769b82 100644 --- a/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts +++ b/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts @@ -1,14 +1,14 @@ -import { type HandleReadCallbackProps } from '../handle-read-callback'; +import { type HandleReadDeps } from '../types'; import { writeChunkToDisk } from '../read-chunk-from-disk'; -import { type FileHydrationState, markBlocksInRangeDownloaded, startBlockDownload } from './hydration-state'; +import { getHydratedBytes, type FileHydrationState, markBlocksInRangeDownloaded } from './hydration-state'; import { type File } from '../../../../../context/virtual-drive/files/domain/File'; import { downloadFileRange } from '../../../../../infra/environment/download-file/download-file'; -import { getStopwatch } from './hydration-stopwatch'; +import { type Result } from '../../../../../context/shared/domain/Result'; type Props = { - bucketId: HandleReadCallbackProps['bucketId']; - mnemonic: HandleReadCallbackProps['mnemonic']; - network: HandleReadCallbackProps['network']; - onDownloadProgress: HandleReadCallbackProps['onDownloadProgress']; + bucketId: HandleReadDeps['bucketId']; + mnemonic: HandleReadDeps['mnemonic']; + network: HandleReadDeps['network']; + onDownloadProgress: HandleReadDeps['onDownloadProgress']; virtualFile: File; filePath: string; state: FileHydrationState; @@ -29,22 +29,40 @@ export async function downloadAndCacheBlock({ state, blockStart, blockLength, -}: Props): Promise { - const resolve = startBlockDownload(state, { position: blockStart, length: blockLength }); +}: Props): Promise> { + if (isAborted(state)) return { data: undefined }; + try { - const buffer = await downloadFileRange({ + const download = await downloadFileRange({ fileId: virtualFile.contentsId, bucketId, mnemonic, network, range: { position: blockStart, length: blockLength }, - signal: new AbortController(), + signal: state.abortController.signal, }); - await writeChunkToDisk(filePath, buffer, blockStart); + if (isAborted(state)) return { data: undefined }; + if (download.error) return { error: download.error }; + + await writeChunkToDisk(filePath, download.data, blockStart); + if (isAborted(state)) return { data: undefined }; + markBlocksInRangeDownloaded(state, { position: blockStart, length: blockLength }); - const elapsedTime = getStopwatch(virtualFile.contentsId)?.elapsedTime() ?? 0; - onDownloadProgress(virtualFile.name, virtualFile.type, blockStart + blockLength, virtualFile.size, elapsedTime); - } finally { - resolve(); + const elapsedTime = state.stopwatch?.elapsedTime() ?? 0; + onDownloadProgress( + virtualFile.name, + virtualFile.type, + getHydratedBytes(state), + virtualFile.size, + elapsedTime, + ); + return { data: undefined }; + } catch (error) { + if (isAborted(state)) return { data: undefined }; + return { error: error instanceof Error ? error : new Error('Unknown error occurred') }; } } + +function isAborted(state: FileHydrationState): boolean { + return state.abortController.signal.aborted; +} diff --git a/src/backend/features/fuse/on-read/download-cache/expand-to-block-boundaries.test.ts b/src/backend/features/fuse/on-read/download-cache/expand-to-block-boundaries.test.ts new file mode 100644 index 0000000000..8c7e5f9600 --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/expand-to-block-boundaries.test.ts @@ -0,0 +1,43 @@ +import { BLOCK_SIZE } from './constants'; +import { expandToBlockBoundaries } from './expand-to-block-boundaries'; + +describe('expandToBlockBoundaries', () => { + it('expands a small read inside the first block to the full first block', () => { + const result = expandToBlockBoundaries({ + range: { position: 100, length: 4096 }, + fileSize: BLOCK_SIZE * 3, + }); + + expect(result).toStrictEqual({ blockStart: 0, blockLength: BLOCK_SIZE }); + }); + + it('starts at the containing block boundary for reads after the first block', () => { + const result = expandToBlockBoundaries({ + range: { position: BLOCK_SIZE + 100, length: 4096 }, + fileSize: BLOCK_SIZE * 3, + }); + + expect(result).toStrictEqual({ blockStart: BLOCK_SIZE, blockLength: BLOCK_SIZE }); + }); + + it('expands reads crossing a block boundary to cover every touched block', () => { + const result = expandToBlockBoundaries({ + range: { position: BLOCK_SIZE - 100, length: 200 }, + fileSize: BLOCK_SIZE * 3, + }); + + expect(result).toStrictEqual({ blockStart: 0, blockLength: BLOCK_SIZE * 2 }); + }); + + it('expands a read inside a partial last block to that whole partial block', () => { + const partialLastBlockLength = 500; + const fileSize = BLOCK_SIZE + partialLastBlockLength; + + const result = expandToBlockBoundaries({ + range: { position: BLOCK_SIZE + 100, length: 100 }, + fileSize, + }); + + expect(result).toStrictEqual({ blockStart: BLOCK_SIZE, blockLength: partialLastBlockLength }); + }); +}); diff --git a/src/backend/features/fuse/on-read/download-cache/expand-to-block-boundaries.ts b/src/backend/features/fuse/on-read/download-cache/expand-to-block-boundaries.ts index bd4cd5c41f..ebfc009bd5 100644 --- a/src/backend/features/fuse/on-read/download-cache/expand-to-block-boundaries.ts +++ b/src/backend/features/fuse/on-read/download-cache/expand-to-block-boundaries.ts @@ -1,3 +1,4 @@ +import { ReadRange } from '../types'; import { BLOCK_SIZE } from './constants'; /** @@ -5,13 +6,12 @@ import { BLOCK_SIZE } from './constants'; * request downloads complete blocks. Ensuring correct bitmap tracking, prefetching, * and preventing double downloads. */ -export function expandToBlockBoundaries( - position: number, - length: number, - fileSize: number, -): { blockStart: number; blockLength: number } { - const blockStart = Math.floor(position / BLOCK_SIZE) * BLOCK_SIZE; - const end = position + length; +export function expandToBlockBoundaries({ range, fileSize }: { range: ReadRange; fileSize: number }): { + blockStart: number; + blockLength: number; +} { + const blockStart = Math.floor(range.position / BLOCK_SIZE) * BLOCK_SIZE; + const end = range.position + range.length; const blockEnd = Math.min(Math.ceil(end / BLOCK_SIZE) * BLOCK_SIZE, fileSize); return { blockStart, blockLength: blockEnd - blockStart }; } diff --git a/src/backend/features/fuse/on-read/download-cache/file-exists-on-disk.test.ts b/src/backend/features/fuse/on-read/download-cache/file-exists-on-disk.test.ts new file mode 100644 index 0000000000..164b58b73f --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/file-exists-on-disk.test.ts @@ -0,0 +1,30 @@ +import fs from 'node:fs/promises'; +import { fileExistsOnDisk } from './file-exists-on-disk'; + +vi.mock('node:fs/promises', () => ({ + default: { + stat: vi.fn(), + }, +})); + +const fsMock = vi.mocked(fs); + +describe('fileExistsOnDisk', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('returns true when fs.stat succeeds', async () => { + fsMock.stat.mockResolvedValue({} as Awaited>); + + await expect(fileExistsOnDisk('/tmp/cache-file')).resolves.toBe(true); + + expect(fsMock.stat).toHaveBeenCalledWith('/tmp/cache-file'); + }); + + it('returns false when fs.stat rejects', async () => { + fsMock.stat.mockRejectedValue(new Error('missing')); + + await expect(fileExistsOnDisk('/tmp/cache-file')).resolves.toBe(false); + }); +}); diff --git a/src/backend/features/fuse/on-read/download-cache/hydration-state.test.ts b/src/backend/features/fuse/on-read/download-cache/hydration-state.test.ts new file mode 100644 index 0000000000..255461e2ba --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/hydration-state.test.ts @@ -0,0 +1,233 @@ +import { + abortAllHydrations, + abortHydrationState, + clearHydrationState, + ensureAllocatedOnce, + finalizeIfNeeded, + getExistingHydrationState, + getHydratedBytes, + getOrCreateHydrationState, + isFileHydrated, + markBlocksInRangeDownloaded, + markFinalized, +} from './hydration-state'; +import { allocateFile } from './allocate-file'; +import { BLOCK_SIZE } from './constants'; + +vi.mock('./allocate-file', () => ({ + allocateFile: vi.fn(), +})); + +const allocateFileMock = vi.mocked(allocateFile); + +describe('hydration-state lifecycle', () => { + beforeEach(() => { + clearHydrationState(); + }); + + it('reads an existing state without creating a new one', () => { + const created = getOrCreateHydrationState('contents-id', 1024); + + const existing = getExistingHydrationState('contents-id'); + + expect(existing).toBe(created); + }); + + it('does not create state when reading a missing contents id', () => { + const missing = getExistingHydrationState('missing'); + + expect(missing).toBeUndefined(); + expect(getExistingHydrationState('missing')).toBeUndefined(); + }); + + it('creates state once per contents id', () => { + const first = getOrCreateHydrationState('contents-id', 1024); + const second = getOrCreateHydrationState('contents-id', 2048); + + expect(second).toBe(first); + }); + + it('creates new states with a fresh AbortController and unfinished finalization state', () => { + const first = getOrCreateHydrationState('first', 1024); + const second = getOrCreateHydrationState('second', 1024); + + expect(first.abortController).toBeInstanceOf(AbortController); + expect(second.abortController).toBeInstanceOf(AbortController); + expect(first.abortController).not.toBe(second.abortController); + expect(first.fileSize).toBe(1024); + expect(first.hydratedBytes).toBe(0); + expect(first.finalized).toBe(false); + expect(first.finalization).toBeUndefined(); + }); + + it('aborts one hydration state without aborting another', () => { + const first = getOrCreateHydrationState('first', 1024); + const second = getOrCreateHydrationState('second', 1024); + + abortHydrationState(first); + + expect(first.abortController.signal.aborted).toBe(true); + expect(second.abortController.signal.aborted).toBe(false); + }); + + it('aborts every hydration state', () => { + const first = getOrCreateHydrationState('first', 1024); + const second = getOrCreateHydrationState('second', 1024); + + expect(first.abortController.signal.aborted).toBe(false); + expect(second.abortController.signal.aborted).toBe(false); + abortAllHydrations(); + + expect(first.abortController.signal.aborted).toBe(true); + expect(second.abortController.signal.aborted).toBe(true); + }); + + it('reuses the in-flight repository registration for concurrent finalization attempts', () => { + const state = getOrCreateHydrationState('contents-id', 1024); + const promise = new Promise(() => undefined); + + const first = finalizeIfNeeded(state, () => promise); + const second = finalizeIfNeeded(state, () => Promise.resolve()); + + expect(second).toBe(first); + expect(state.finalization).toBe(first); + expect(state.finalized).toBe(false); + }); + + it('allows failed finalization to be retried', async () => { + const state = getOrCreateHydrationState('contents-id', 1024); + + await expect(finalizeIfNeeded(state, () => Promise.reject(new Error('register failed')))).rejects.toThrow( + 'register failed', + ); + + expect(state.finalization).toBeUndefined(); + expect(state.finalized).toBe(false); + + await finalizeIfNeeded(state, () => Promise.resolve()); + + expect(state.finalized).toBe(true); + }); + + it('marks successful finalization as finalized', async () => { + const state = getOrCreateHydrationState('contents-id', 1024); + + await finalizeIfNeeded(state, () => Promise.resolve()); + + expect(state.finalized).toBe(true); + expect(state.finalization).toBeUndefined(); + }); + + it('can mark a state as finalized directly', () => { + const state = getOrCreateHydrationState('contents-id', 1024); + + markFinalized(state); + + expect(state.finalized).toBe(true); + }); + + it('reports hydrated bytes from completed blocks only', () => { + const fileSize = BLOCK_SIZE * 2 + 123; + const state = getOrCreateHydrationState('contents-id', fileSize); + + markBlocksInRangeDownloaded(state, { position: 0, length: BLOCK_SIZE }); + + expect(getHydratedBytes(state)).toBe(BLOCK_SIZE); + }); + + it('counts the final block by its actual byte length', () => { + const fileSize = BLOCK_SIZE * 2 + 123; + const state = getOrCreateHydrationState('contents-id', fileSize); + + markBlocksInRangeDownloaded(state, { position: BLOCK_SIZE * 2, length: 123 }); + + expect(getHydratedBytes(state)).toBe(123); + }); + + it('reports full file size when every block is hydrated', () => { + const fileSize = BLOCK_SIZE + 123; + const state = getOrCreateHydrationState('contents-id', fileSize); + + markBlocksInRangeDownloaded(state, { position: 0, length: BLOCK_SIZE }); + markBlocksInRangeDownloaded(state, { position: BLOCK_SIZE, length: 123 }); + + expect(getHydratedBytes(state)).toBe(fileSize); + }); + + it('counts hydrated bytes only once when the same block is marked again', () => { + const fileSize = BLOCK_SIZE + 123; + const state = getOrCreateHydrationState('contents-id', fileSize); + + markBlocksInRangeDownloaded(state, { position: 0, length: BLOCK_SIZE }); + markBlocksInRangeDownloaded(state, { position: 10, length: 10 }); + + expect(getHydratedBytes(state)).toBe(BLOCK_SIZE); + }); + + it('treats an empty file as fully hydrated without marking any blocks', () => { + const state = getOrCreateHydrationState('empty-contents-id', 0); + + expect(isFileHydrated(state)).toBe(true); + expect(getHydratedBytes(state)).toBe(0); + }); + + describe('file allocation', () => { + it('allocates a file only once for concurrent callers', async () => { + const state = getOrCreateHydrationState('contents-id', 1024); + let resolveAllocation: () => void = () => undefined; + allocateFileMock.mockReturnValue( + new Promise((resolve) => { + resolveAllocation = resolve; + }), + ); + + const first = ensureAllocatedOnce(state, '/tmp/cache-file', 1024); + const second = ensureAllocatedOnce(state, '/tmp/cache-file', 1024); + + expect(first).toBe(second); + expect(allocateFileMock).toHaveBeenCalledOnce(); + expect(allocateFileMock).toHaveBeenCalledWith('/tmp/cache-file', 1024); + + resolveAllocation(); + await expect(first).resolves.toStrictEqual({ data: undefined }); + await expect(second).resolves.toStrictEqual({ data: undefined }); + }); + + it('keeps successful allocation in state so later callers reuse it', async () => { + const state = getOrCreateHydrationState('contents-id', 1024); + allocateFileMock.mockResolvedValue(undefined); + + const first = ensureAllocatedOnce(state, '/tmp/cache-file', 1024); + await expect(first).resolves.toStrictEqual({ data: undefined }); + const second = ensureAllocatedOnce(state, '/tmp/cache-file', 1024); + + expect(second).toBe(first); + expect(allocateFileMock).toHaveBeenCalledOnce(); + }); + + it('allows failed allocation to be retried', async () => { + const state = getOrCreateHydrationState('contents-id', 1024); + allocateFileMock.mockRejectedValueOnce(new Error('allocation failed')).mockResolvedValueOnce(undefined); + + await expect(ensureAllocatedOnce(state, '/tmp/cache-file', 1024)).resolves.toStrictEqual({ + error: new Error('allocation failed'), + }); + + expect(state.allocation).toBeUndefined(); + + await expect(ensureAllocatedOnce(state, '/tmp/cache-file', 1024)).resolves.toStrictEqual({ data: undefined }); + + expect(allocateFileMock).toHaveBeenCalledTimes(2); + }); + + it('starts the state stopwatch when allocation begins', async () => { + const state = getOrCreateHydrationState('contents-id', 1024); + allocateFileMock.mockResolvedValue(undefined); + + await expect(ensureAllocatedOnce(state, '/tmp/cache-file', 1024)).resolves.toStrictEqual({ data: undefined }); + + expect(state.stopwatch).toBeDefined(); + expect(state.stopwatch?.elapsedTime()).not.toBe(-1); + }); + }); +}); diff --git a/src/backend/features/fuse/on-read/download-cache/hydration-state.ts b/src/backend/features/fuse/on-read/download-cache/hydration-state.ts index a97699537f..51dc4e9281 100644 --- a/src/backend/features/fuse/on-read/download-cache/hydration-state.ts +++ b/src/backend/features/fuse/on-read/download-cache/hydration-state.ts @@ -1,4 +1,8 @@ import { BITS_PER_BYTE, BLOCK_SIZE } from './constants'; +import { allocateFile } from './allocate-file'; +import { Stopwatch } from '../../../../../apps/shared/types/Stopwatch'; +import { type Result } from '../../../../../context/shared/domain/Result'; +import { ReadRange } from '../types'; /** * Tracks which byte ranges of a file have been downloaded and written to disk. @@ -14,38 +18,73 @@ import { BITS_PER_BYTE, BLOCK_SIZE } from './constants'; * A block is only marked after its full write to disk succeeds — never partially. * A hard kill mid-write is handled by wiping the download cache on startup. * - * Race conditions: concurrent reads for the same block are tracked via an in-flight - * set. The second caller waits for the first to finish rather than double-downloading. + * Concurrent reads for the same block share the in-flight block download promise + * instead of starting duplicate downloads. */ export type FileHydrationState = { bitmap: Buffer; + fileSize: number; totalBlocks: number; - downloadedBlocks: number; - blocksBeingDownloaded: Map>; -}; -type Range = { - position: number; - length: number; + hydratedBytes: number; + blocksBeingDownloaded: Map>>; + allocation?: Promise>; + stopwatch?: Stopwatch; + finalized: boolean; + finalization?: Promise; + abortController: AbortController; }; + const hydrationState = new Map(); -export function getOrInitHydrationState(contentsId: string, fileSize: number): FileHydrationState { - const existing = hydrationState.get(contentsId); +export function getExistingHydrationState(contentsId: string): FileHydrationState | undefined { + return hydrationState.get(contentsId); +} + +export function getOrCreateHydrationState(contentsId: string, fileSize: number): FileHydrationState { + const existing = getExistingHydrationState(contentsId); if (existing) return existing; const totalBlocks = Math.ceil(fileSize / BLOCK_SIZE); const size = Math.ceil(totalBlocks / BITS_PER_BYTE); const state: FileHydrationState = { bitmap: Buffer.alloc(size, 0), + fileSize, totalBlocks, - downloadedBlocks: 0, + hydratedBytes: 0, blocksBeingDownloaded: new Map(), + finalized: false, + abortController: new AbortController(), }; hydrationState.set(contentsId, state); return state; } +export function ensureAllocatedOnce( + state: FileHydrationState, + filePath: string, + fileSize: number, +): Promise> { + if (state.allocation) return state.allocation; + + state.stopwatch = new Stopwatch(); + state.stopwatch.start(); + + const allocation = allocateFile(filePath, fileSize).then( + (): Result => ({ data: undefined }), + (error): Result => { + if (state.allocation === allocation) { + state.allocation = undefined; + state.stopwatch = undefined; + } + return { error: error instanceof Error ? error : new Error('Unknown error occurred') }; + }, + ); + + state.allocation = allocation; + return allocation; +} + function blockIndexForByte(byte: number): number { return Math.floor(byte / BLOCK_SIZE); } @@ -87,10 +126,14 @@ function setBit(bitmap: Buffer, blockIndex: number): void { } export function isFileHydrated(state: FileHydrationState): boolean { - return state.downloadedBlocks === state.totalBlocks; + return state.hydratedBytes === state.fileSize; +} + +export function getHydratedBytes(state: FileHydrationState): number { + return state.hydratedBytes; } -function blocksWithinRange({ position, length }: Range): Array { +function blocksWithinRange({ position, length }: ReadRange): Array { const first = blockIndexForByte(position); const last = blockIndexForByte(position + length - 1); const blocks: number[] = []; @@ -100,24 +143,29 @@ function blocksWithinRange({ position, length }: Range): Array { return blocks; } -export function isRangeHydrated(state: FileHydrationState, { position, length }: Range): boolean { +export function isRangeHydrated(state: FileHydrationState, { position, length }: ReadRange): boolean { return blocksWithinRange({ position, length }).every((block) => getBit(state.bitmap, block)); } -export function markBlocksInRangeDownloaded(state: FileHydrationState, { position, length }: Range): void { +export function markBlocksInRangeDownloaded(state: FileHydrationState, { position, length }: ReadRange): void { for (const block of blocksWithinRange({ position, length })) { if (!getBit(state.bitmap, block)) { setBit(state.bitmap, block); - state.downloadedBlocks++; + state.hydratedBytes += blockByteLength(state, block); } } } +function blockByteLength(state: FileHydrationState, block: number): number { + const blockStart = block * BLOCK_SIZE; + return Math.min(BLOCK_SIZE, state.fileSize - blockStart); +} + /** - * Returns block indices within the range that are neither cached nor currently being downloaded. - * Use this after waiting for in-flight blocks to find what still needs downloading. + * Returns block indices within the range that are neither hydrated nor already downloading. + * Call after waiting for existing in-flight blocks to identify the remaining work. */ -export function getMissingBlocks(state: FileHydrationState, { position, length }: Range): number[] { +export function getMissingBlocks(state: FileHydrationState, { position, length }: ReadRange): number[] { return blocksWithinRange({ position, length }).filter( (block) => !getBit(state.bitmap, block) && !state.blocksBeingDownloaded.has(block), ); @@ -125,9 +173,9 @@ export function getMissingBlocks(state: FileHydrationState, { position, length } export function getBlocksBeingDownloaded( state: FileHydrationState, - { position, length }: Range, -): Map> { - const blocksBeingDownloadedWithinRange = new Map>(); + { position, length }: ReadRange, +): Map>> { + const blocksBeingDownloadedWithinRange = new Map>>(); for (const block of blocksWithinRange({ position, length })) { const existing = state.blocksBeingDownloaded.get(block); if (existing) blocksBeingDownloadedWithinRange.set(block, existing); @@ -135,26 +183,54 @@ export function getBlocksBeingDownloaded( return blocksBeingDownloadedWithinRange; } -/** - * Marks blocks as being downloaded. Call before starting a download. - * Returns a resolve function to call it when the download + write completes. - */ -export function startBlockDownload(state: FileHydrationState, { position, length }: Range): () => void { - let resolve: () => void = () => undefined; - const promise = new Promise((res) => { - resolve = res; - }); +export function setBlockDownloadInFlight( + state: FileHydrationState, + block: number, + promise: Promise>, +): void { + state.blocksBeingDownloaded.set(block, promise); +} - for (const block of blocksWithinRange({ position, length })) { - state.blocksBeingDownloaded.set(block, promise); +export function clearBlockDownloadInFlight( + state: FileHydrationState, + block: number, + promise: Promise>, +): void { + if (state.blocksBeingDownloaded.get(block) === promise) { + state.blocksBeingDownloaded.delete(block); } +} - return () => { - for (const block of blocksWithinRange({ position, length })) { - state.blocksBeingDownloaded.delete(block); - } - resolve(); - }; +export function finalizeIfNeeded(state: FileHydrationState, finalize: () => Promise): Promise { + if (state.finalized) return Promise.resolve(); + if (state.finalization) return state.finalization; + + const finalization = Promise.resolve() + .then(finalize) + .then(() => { + markFinalized(state); + }) + .finally(() => { + if (state.finalization === finalization) { + state.finalization = undefined; + } + }); + state.finalization = finalization; + return finalization; +} + +export function markFinalized(state: FileHydrationState): void { + state.finalized = true; +} + +export function abortHydrationState(state: FileHydrationState): void { + state.abortController.abort(); +} + +export function abortAllHydrations(): void { + for (const state of hydrationState.values()) { + abortHydrationState(state); + } } /** diff --git a/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.test.ts b/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.test.ts new file mode 100644 index 0000000000..6e9b203f7f --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.test.ts @@ -0,0 +1,54 @@ +import { readIfHydrated } from './read-if-hydrated'; +import { + clearHydrationState, + getExistingHydrationState, + getOrCreateHydrationState, + markBlocksInRangeDownloaded, +} from './hydration-state'; +import { readChunkFromDisk } from '../read-chunk-from-disk'; + +vi.mock('../read-chunk-from-disk', () => ({ + readChunkFromDisk: vi.fn(), +})); + +const readChunkFromDiskMock = vi.mocked(readChunkFromDisk); + +describe('readIfHydrated', () => { + beforeEach(() => { + clearHydrationState(); + vi.clearAllMocks(); + }); + + it('returns undefined when no hydration state exists', async () => { + const result = await readIfHydrated('/tmp/cache-file', 'contents-id', { position: 0, length: 10 }); + + expect(result).toBeUndefined(); + }); + + it('does not create hydration state when no hydration state exists', async () => { + await readIfHydrated('/tmp/cache-file', 'contents-id', { position: 0, length: 10 }); + + expect(getExistingHydrationState('contents-id')).toBeUndefined(); + }); + + it('returns undefined when the requested range is not hydrated', async () => { + getOrCreateHydrationState('contents-id', 1024); + + const result = await readIfHydrated('/tmp/cache-file', 'contents-id', { position: 0, length: 10 }); + + expect(result).toBeUndefined(); + expect(readChunkFromDiskMock).not.toHaveBeenCalled(); + }); + + it('reads bytes from disk when the requested range is hydrated', async () => { + const chunk = Buffer.from('cached'); + const state = getOrCreateHydrationState('contents-id', 1024); + markBlocksInRangeDownloaded(state, { position: 0, length: 10 }); + readChunkFromDiskMock.mockResolvedValue(chunk); + + const result = await readIfHydrated('/tmp/cache-file', 'contents-id', { position: 0, length: 10 }); + + expect(result).toBe(chunk); + expect(readChunkFromDiskMock).toHaveBeenCalledWith('/tmp/cache-file', 10, 0); + }); +}); diff --git a/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.ts b/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.ts new file mode 100644 index 0000000000..13c41497c2 --- /dev/null +++ b/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.ts @@ -0,0 +1,19 @@ +import { readChunkFromDisk } from '../read-chunk-from-disk'; +import { getExistingHydrationState, isRangeHydrated } from './hydration-state'; + +type Range = { + position: number; + length: number; +}; + +export async function readIfHydrated( + filePath: string, + contentsId: string, + range: Range, +): Promise { + const state = getExistingHydrationState(contentsId); + if (!state) return undefined; + if (!isRangeHydrated(state, range)) return undefined; + + return readChunkFromDisk(filePath, range.length, range.position); +} diff --git a/src/backend/features/fuse/on-read/handle-read-callback.test.ts b/src/backend/features/fuse/on-read/handle-read-callback.test.ts index 5d49a69349..21d32d2f9a 100644 --- a/src/backend/features/fuse/on-read/handle-read-callback.test.ts +++ b/src/backend/features/fuse/on-read/handle-read-callback.test.ts @@ -1,18 +1,24 @@ -import { PassThrough } from 'node:stream'; -import { handleReadCallback, type HandleReadCallbackDeps } from './handle-read-callback'; +import { handleReadCallback, type HandleReadCallbackProps } from './handle-read-callback'; import * as readChunkModule from './read-chunk-from-disk'; -import * as createDownloadModule from './create-download-to-disk'; -import * as hydrationRegistryModule from './hydration-registry'; import * as processBlocklistModule from '../../../features/virtual-drive/utils/process-blocklist'; +import * as fileExistsModule from './download-cache/file-exists-on-disk'; +import * as allocateFileModule from './download-cache/allocate-file'; +import * as downloadAndSaveBlockModule from './download-cache/download-and-save-block'; +import { + clearHydrationState, + getExistingHydrationState, + getOrCreateHydrationState, + markBlocksInRangeDownloaded, +} from './download-cache/hydration-state'; import { partialSpyOn, call } from '../../../../../tests/vitest/utils.helper'; import { type File } from '../../../../context/virtual-drive/files/domain/File'; -import { FuseNoSuchFileOrDirectoryError } from '../../../../apps/drive/fuse/callbacks/FuseErrors'; +import { FuseIOError, FuseNoSuchFileOrDirectoryError } from '../../../../apps/drive/fuse/callbacks/FuseErrors'; const readChunkFromDiskMock = partialSpyOn(readChunkModule, 'readChunkFromDisk'); -const createDownloadToDiskMock = partialSpyOn(createDownloadModule, 'createDownloadToDisk'); -const getHydrationMock = partialSpyOn(hydrationRegistryModule, 'getHydration'); -const setHydrationMock = partialSpyOn(hydrationRegistryModule, 'setHydration'); const isBlocklistedProcessMock = partialSpyOn(processBlocklistModule, 'isBlocklistedProcess'); +const fileExistsOnDiskMock = partialSpyOn(fileExistsModule, 'fileExistsOnDisk'); +const allocateFileMock = partialSpyOn(allocateFileModule, 'allocateFile'); +const downloadAndCacheBlockMock = partialSpyOn(downloadAndSaveBlockModule, 'downloadAndCacheBlock'); const virtualFile = { contentsId: 'contents-123', @@ -23,32 +29,31 @@ const virtualFile = { size: 1000, } as unknown as File; -function createDeps(overrides: Partial = {}): HandleReadCallbackDeps { +function createDeps(overrides: Partial = {}): HandleReadCallbackProps { return { findVirtualFile: vi.fn().mockResolvedValue(virtualFile), findTemporalFile: vi.fn().mockResolvedValue(undefined), - existsOnDisk: vi.fn().mockResolvedValue(false), - startDownload: vi.fn().mockResolvedValue({ stream: new PassThrough(), elapsedTime: () => 0 }), onDownloadProgress: vi.fn(), saveToRepository: vi.fn().mockResolvedValue(undefined), + bucketId: 'bucket-id', + mnemonic: 'mnemonic', + network: {} as HandleReadCallbackProps['network'], + path: '/file.mp4', + range: { position: 0, length: 10 }, + processName: 'vlc', ...overrides, }; } -function createWriterMock(bytesAvailable = 0) { - return { - waitForBytes: vi.fn().mockResolvedValue(undefined), - getBytesAvailable: vi.fn().mockReturnValue(bytesAvailable), - destroy: vi.fn().mockResolvedValue(undefined), - }; -} - describe('handleReadCallback', () => { beforeEach(() => { + clearHydrationState(); + vi.clearAllMocks(); isBlocklistedProcessMock.mockReturnValue(false); - getHydrationMock.mockReturnValue(undefined); + fileExistsOnDiskMock.mockResolvedValue(true); + allocateFileMock.mockResolvedValue(undefined); + downloadAndCacheBlockMock.mockResolvedValue({ data: undefined }); readChunkFromDiskMock.mockResolvedValue(Buffer.from('data')); - createDownloadToDiskMock.mockReturnValue(createWriterMock()); }); describe('when virtual file is not found', () => { @@ -58,7 +63,7 @@ describe('handleReadCallback', () => { findTemporalFile: vi.fn().mockResolvedValue(undefined), }); - const result = await handleReadCallback(deps, '/file.txt', 10, 0, 'vlc'); + const result = await handleReadCallback({ ...deps, path: '/file.txt' }); expect(result.error).toBeInstanceOf(FuseNoSuchFileOrDirectoryError); }); @@ -74,7 +79,7 @@ describe('handleReadCallback', () => { }), }); - const result = await handleReadCallback(deps, '/file.txt', 13, 0, 'vlc'); + const result = await handleReadCallback({ ...deps, path: '/file.txt', range: { position: 0, length: 13 } }); expect(result.data).toBe(chunk); call(readChunkFromDiskMock).toStrictEqual(['/tmp/internxt-drive-tmp/uuid', 13, 0]); @@ -86,78 +91,108 @@ describe('handleReadCallback', () => { findTemporalFile: vi.fn().mockResolvedValue({ path: { value: '/virtual/file.txt' } }), }); - const result = await handleReadCallback(deps, '/file.txt', 10, 0, 'vlc'); + const result = await handleReadCallback({ ...deps, path: '/file.txt' }); expect(result.error).toBeInstanceOf(FuseNoSuchFileOrDirectoryError); }); }); describe('when process is blocklisted', () => { - it('should return empty buffer when file is not on disk', async () => { + it('should return empty buffer without side effects when the requested range is not cached', async () => { isBlocklistedProcessMock.mockReturnValue(true); - const deps = createDeps({ existsOnDisk: vi.fn().mockResolvedValue(false) }); + fileExistsOnDiskMock.mockResolvedValue(false); + const deps = createDeps({ processName: 'pool-org.gnome.' }); - const result = await handleReadCallback(deps, '/file.mp4', 10, 0, 'pool-org.gnome.'); + const result = await handleReadCallback(deps); expect(result.data).toHaveLength(0); - expect(deps.startDownload).not.toHaveBeenCalled(); + expect(getExistingHydrationState(virtualFile.contentsId)).toBeUndefined(); + expect(fileExistsOnDiskMock).not.toHaveBeenCalled(); + expect(allocateFileMock).not.toHaveBeenCalled(); + expect(downloadAndCacheBlockMock).not.toHaveBeenCalled(); + expect(deps.onDownloadProgress).not.toHaveBeenCalled(); + expect(deps.saveToRepository).not.toHaveBeenCalled(); + expect(readChunkFromDiskMock).not.toHaveBeenCalled(); }); - it('should serve from disk when file is already downloaded', async () => { + it('should return empty buffer when hydration state exists but the requested range is not cached', async () => { isBlocklistedProcessMock.mockReturnValue(true); - const chunk = Buffer.from('cached'); - readChunkFromDiskMock.mockResolvedValue(chunk); - const deps = createDeps({ existsOnDisk: vi.fn().mockResolvedValue(true) }); + getOrCreateHydrationState(virtualFile.contentsId, virtualFile.size); + const deps = createDeps({ processName: 'pool-org.gnome.' }); - const result = await handleReadCallback(deps, '/file.mp4', 6, 0, 'pool-org.gnome.'); + const result = await handleReadCallback(deps); - expect(result.data).toBe(chunk); - expect(deps.startDownload).not.toHaveBeenCalled(); + expect(result.data).toHaveLength(0); + expect(allocateFileMock).not.toHaveBeenCalled(); + expect(readChunkFromDiskMock).not.toHaveBeenCalled(); + expect(downloadAndCacheBlockMock).not.toHaveBeenCalled(); + expect(deps.onDownloadProgress).not.toHaveBeenCalled(); + expect(deps.saveToRepository).not.toHaveBeenCalled(); + }); + + it('should return requested bytes when the range is already cached', async () => { + isBlocklistedProcessMock.mockReturnValue(true); + const state = getOrCreateHydrationState(virtualFile.contentsId, virtualFile.size); + markBlocksInRangeDownloaded(state, { position: 0, length: 10 }); + const cached = Buffer.from('cached'); + readChunkFromDiskMock.mockResolvedValue(cached); + const deps = createDeps({ processName: 'pool-org.gnome.' }); + + const result = await handleReadCallback(deps); + + expect(result.data).toBe(cached); + expect(fileExistsOnDiskMock).not.toHaveBeenCalled(); + expect(allocateFileMock).not.toHaveBeenCalled(); + expect(downloadAndCacheBlockMock).not.toHaveBeenCalled(); + expect(deps.onDownloadProgress).not.toHaveBeenCalled(); + expect(deps.saveToRepository).not.toHaveBeenCalled(); + expect(readChunkFromDiskMock).toHaveBeenCalledWith(expect.stringContaining(virtualFile.contentsId), 10, 0); }); }); - describe('when file needs to be downloaded', () => { - it('should start a new hydration when none exists', async () => { - const writer = createWriterMock(); - createDownloadToDiskMock.mockReturnValue(writer); + describe('when allocating the cache file', () => { + it('returns EIO and does not download when allocation fails', async () => { + fileExistsOnDiskMock.mockResolvedValue(false); + allocateFileMock.mockRejectedValueOnce(new Error('disk full')); const deps = createDeps(); - await handleReadCallback(deps, '/file.mp4', 10, 50, 'vlc'); + const result = await handleReadCallback(deps); - expect(deps.startDownload).toHaveBeenCalledWith(virtualFile); - expect(setHydrationMock).toHaveBeenCalledOnce(); - expect(writer.waitForBytes).toHaveBeenCalledWith(50, 10); + expect(result.error).toBeInstanceOf(FuseIOError); + expect(downloadAndCacheBlockMock).not.toHaveBeenCalled(); + expect(readChunkFromDiskMock).not.toHaveBeenCalled(); + expect(deps.onDownloadProgress).not.toHaveBeenCalled(); + expect(deps.saveToRepository).not.toHaveBeenCalled(); }); - it('should reuse existing hydration when one exists', async () => { - const writer = createWriterMock(); - getHydrationMock.mockReturnValue({ writer }); + it('retries allocation on a later read after allocation fails', async () => { + fileExistsOnDiskMock.mockResolvedValue(false); + allocateFileMock.mockRejectedValueOnce(new Error('disk full')).mockResolvedValueOnce(undefined); const deps = createDeps(); - await handleReadCallback(deps, '/file.mp4', 10, 50, 'vlc'); + const first = await handleReadCallback(deps); + const second = await handleReadCallback(deps); - expect(deps.startDownload).not.toHaveBeenCalled(); - expect(writer.waitForBytes).toHaveBeenCalledWith(50, 10); + expect(first.error).toBeInstanceOf(FuseIOError); + expect(second.data).toStrictEqual(Buffer.from('data')); + expect(allocateFileMock).toHaveBeenCalledTimes(2); + expect(downloadAndCacheBlockMock).toHaveBeenCalledOnce(); }); + }); - it('should read chunk from disk after waitForBytes resolves', async () => { + describe('when file needs to be downloaded', () => { + it('allocates missing cache file, downloads missing blocks, then reads from disk', async () => { + fileExistsOnDiskMock.mockResolvedValue(false); const chunk = Buffer.from('downloaded'); readChunkFromDiskMock.mockResolvedValue(chunk); const deps = createDeps(); - const result = await handleReadCallback(deps, '/file.mp4', 10, 0, 'vlc'); + const result = await handleReadCallback(deps); expect(result.data).toBe(chunk); - }); - - it('should skip waitForBytes when bytes are already available', async () => { - const writer = createWriterMock(1000); - getHydrationMock.mockReturnValue({ writer }); - const deps = createDeps(); - - await handleReadCallback(deps, '/file.mp4', 10, 50, 'vlc'); - - expect(writer.waitForBytes).toHaveBeenCalledWith(50, 10); + expect(allocateFileMock).toHaveBeenCalledWith(expect.stringContaining(virtualFile.contentsId), virtualFile.size); + expect(downloadAndCacheBlockMock).toHaveBeenCalledOnce(); + expect(readChunkFromDiskMock).toHaveBeenCalledWith(expect.stringContaining(virtualFile.contentsId), 10, 0); }); }); }); diff --git a/src/backend/features/fuse/on-read/handle-read-callback.ts b/src/backend/features/fuse/on-read/handle-read-callback.ts index 63bbae1935..07d9a8f69d 100644 --- a/src/backend/features/fuse/on-read/handle-read-callback.ts +++ b/src/backend/features/fuse/on-read/handle-read-callback.ts @@ -7,44 +7,25 @@ import { readChunkFromDisk } from './read-chunk-from-disk'; import { isBlocklistedProcess } from '../../virtual-drive/utils/process-blocklist'; import nodePath from 'node:path'; import { PATHS } from '../../../../core/electron/paths'; -import { formatBytes } from '../../../../shared/format-bytes'; -import { allocateFile } from './download-cache/allocate-file'; -import { expandToBlockBoundaries } from './download-cache/expand-to-block-boundaries'; -import { BLOCK_SIZE } from './download-cache/constants'; -import { - type FileHydrationState, - getOrInitHydrationState, - isRangeHydrated, - isFileHydrated, - getBlocksBeingDownloaded, - getMissingBlocks, -} from './download-cache/hydration-state'; -import { type Network } from '@internxt/sdk'; -import { startStopwatch, deleteStopwatch } from './download-cache/hydration-stopwatch'; -import { fileExistsOnDisk } from './download-cache/file-exists-on-disk'; -import { downloadAndCacheBlock } from './download-cache/download-and-save-block'; import { EMPTY } from './constants'; -export type HandleReadCallbackProps = { +import { readIfHydrated } from './download-cache/read-if-hydrated'; +import { readOrHydrate } from './read-or-hydrate'; +import { type HandleReadDeps, type ReadRange } from './types'; +export type HandleReadCallbackProps = HandleReadDeps & { findVirtualFile: (path: string) => Promise; findTemporalFile: (path: string) => Promise; - // readTemporalFileChunk: (path: string, length: number, position: number) => Promise; - onDownloadProgress: ( - name: string, - extension: string, - bytesDownloaded: number, - fileSize: number, - elapsedTime: number, - ) => void; - saveToRepository: (contentsId: string, size: number, uuid: string, name: string, extension: string) => Promise; - bucketId: string; - mnemonic: string; - network: Network.Network; path: string; - length: number; - position: number; + range: ReadRange; processName: string; }; +/** + * Routes reads between virtual-drive files and temporal local files. + * + * Virtual-file reads enforce process policy: blocklisted processes are cache-only + * readers, while normal processes may hydrate missing cache blocks and finalize the + * file once the full contents are available. + */ export async function handleReadCallback({ findVirtualFile, findTemporalFile, @@ -54,56 +35,38 @@ export async function handleReadCallback({ mnemonic, network, path, - length, - position, + range, processName, }: HandleReadCallbackProps): Promise> { const virtualFile = await findVirtualFile(path); if (!virtualFile) { - return readFromTemporalFile(findTemporalFile, path, length, position); + return readFromTemporalFile(findTemporalFile, path, range.length, range.position); } - logger.debug({ - msg: '[ReadCallback] read request:', - file: virtualFile.nameWithExtension, - position: formatBytes(position), - length: formatBytes(length), - }); - if (isBlocklistedProcess(processName)) { - logger.debug({ msg: '[ReadCallback] Download blocked - blocklisted process', path, processName }); - return { data: EMPTY }; - } const filePath = nodePath.join(PATHS.DOWNLOADED, virtualFile.contentsId); - const state = await ensureFileAllocated(filePath, virtualFile); - - if (isRangeHydrated(state, { position, length })) { - if (isBlocklistedProcess(processName)) { - logger.debug({ - msg: `[ReadCallback] Allowing read from disk for blocklisted process: ${processName} in ${path}`, - }); + if (isBlocklistedProcess(processName)) { + const cached = await readIfHydrated(filePath, virtualFile.contentsId, { + position: range.position, + length: range.length, + }); + if (cached) { + return { data: cached }; } - logger.debug({ msg: '[ReadCallback] serving from disk cache', file: virtualFile.nameWithExtension }); - return { data: await readChunkFromDisk(filePath, length, position) }; + logger.debug({ msg: '[ReadCallback] Download blocked for blocklisted process:', path, processName }); + return { data: EMPTY }; } - await ensureRangeDownloaded({ + return readOrHydrate({ bucketId, mnemonic, network, onDownloadProgress, + saveToRepository, virtualFile, filePath, - state, - position, - length, + range, }); - - if (isFileHydrated(state)) { - await onFileFullyHydrated(saveToRepository, virtualFile); - } - - return { data: await readChunkFromDisk(filePath, length, position) }; } async function readFromTemporalFile( @@ -122,82 +85,3 @@ async function readFromTemporalFile( const chunk = await readChunkFromDisk(temporalFile.contentFilePath, length, position); return { data: chunk ?? EMPTY }; } - -async function ensureFileAllocated(filePath: string, virtualFile: File): Promise { - const allocated = await fileExistsOnDisk(filePath); - if (!allocated) { - await allocateFile(filePath, virtualFile.size); - startStopwatch(virtualFile.contentsId); - } - return getOrInitHydrationState(virtualFile.contentsId, virtualFile.size); -} - -async function ensureRangeDownloaded({ - bucketId, - mnemonic, - network, - onDownloadProgress, - virtualFile, - filePath, - state, - position, - length, -}: { - bucketId: HandleReadCallbackProps['bucketId']; - mnemonic: HandleReadCallbackProps['mnemonic']; - network: HandleReadCallbackProps['network']; - onDownloadProgress: HandleReadCallbackProps['onDownloadProgress']; - virtualFile: File; - filePath: string; - state: FileHydrationState; - position: number; - length: number; -}): Promise { - const { blockStart, blockLength } = expandToBlockBoundaries(position, length, virtualFile.size); - - const blocksBeingDownloaded = getBlocksBeingDownloaded(state, { position: blockStart, length: blockLength }); - if (blocksBeingDownloaded.size > 0) { - logger.debug({ msg: '[ReadCallback] waiting for blocks being downloaded', file: virtualFile.nameWithExtension }); - await Promise.all(blocksBeingDownloaded.values()); - } - - const missingBlocks = getMissingBlocks(state, { position: blockStart, length: blockLength }); - if (missingBlocks.length > 0) { - logger.debug({ - msg: '[ReadCallback] downloading missing blocks', - file: virtualFile.nameWithExtension, - blocks: missingBlocks, - }); - await Promise.all( - missingBlocks.map((block) => { - const start = block * BLOCK_SIZE; - const end = Math.min(start + BLOCK_SIZE, virtualFile.size); - return downloadAndCacheBlock({ - bucketId, - mnemonic, - network, - onDownloadProgress, - virtualFile, - filePath, - state, - blockStart: start, - blockLength: end - start, - }); - }), - ); - } -} - -async function onFileFullyHydrated( - saveToRepository: HandleReadCallbackProps['saveToRepository'], - virtualFile: File, -): Promise { - deleteStopwatch(virtualFile.contentsId); - await saveToRepository( - virtualFile.contentsId, - virtualFile.size, - virtualFile.uuid, - virtualFile.name, - virtualFile.type, - ); -} diff --git a/src/backend/features/fuse/on-read/read-chunk-from-disk.ts b/src/backend/features/fuse/on-read/read-chunk-from-disk.ts index 7a35678d7d..d2e206834a 100644 --- a/src/backend/features/fuse/on-read/read-chunk-from-disk.ts +++ b/src/backend/features/fuse/on-read/read-chunk-from-disk.ts @@ -1,5 +1,5 @@ import fs from 'node:fs/promises'; - +// TODO: Rename chunk -> block export async function writeChunkToDisk(filePath: string, buffer: Buffer, position: number): Promise { const handle = await fs.open(filePath, 'r+'); try { diff --git a/src/backend/features/fuse/on-read/read-or-hydrate.test.ts b/src/backend/features/fuse/on-read/read-or-hydrate.test.ts new file mode 100644 index 0000000000..21092ee453 --- /dev/null +++ b/src/backend/features/fuse/on-read/read-or-hydrate.test.ts @@ -0,0 +1,493 @@ +import { type File } from '../../../../context/virtual-drive/files/domain/File'; +import { FuseIOError } from '../../../../apps/drive/fuse/callbacks/FuseErrors'; +import { call, partialSpyOn, testSleep } from '../../../../../tests/vitest/utils.helper'; +import * as readChunkModule from './read-chunk-from-disk'; +import * as fileExistsModule from './download-cache/file-exists-on-disk'; +import * as allocateFileModule from './download-cache/allocate-file'; +import * as downloadAndSaveBlockModule from './download-cache/download-and-save-block'; +import { + clearHydrationState, + getExistingHydrationState, + getOrCreateHydrationState, + isRangeHydrated, + markBlocksInRangeDownloaded, +} from './download-cache/hydration-state'; +import { BLOCK_SIZE } from './download-cache/constants'; +import { readOrHydrate, type ReadOrHydrateDeps } from './read-or-hydrate'; + +const readChunkFromDiskMock = partialSpyOn(readChunkModule, 'readChunkFromDisk'); +const fileExistsOnDiskMock = partialSpyOn(fileExistsModule, 'fileExistsOnDisk'); +const allocateFileMock = partialSpyOn(allocateFileModule, 'allocateFile'); +const downloadAndCacheBlockMock = partialSpyOn(downloadAndSaveBlockModule, 'downloadAndCacheBlock'); + +const virtualFile = { + contentsId: 'contents-123', + name: 'video', + nameWithExtension: 'video.mp4', + type: 'mp4', + uuid: 'uuid-123', + size: 1000, +} as unknown as File; + +function createDeps(overrides: Partial = {}): ReadOrHydrateDeps { + return { + onDownloadProgress: vi.fn(), + saveToRepository: vi.fn().mockResolvedValue(undefined), + bucketId: 'bucket-id', + mnemonic: 'mnemonic', + network: {} as ReadOrHydrateDeps['network'], + ...overrides, + }; +} + +describe('readOrHydrate', () => { + beforeEach(() => { + clearHydrationState(); + vi.clearAllMocks(); + fileExistsOnDiskMock.mockResolvedValue(true); + allocateFileMock.mockResolvedValue(undefined); + downloadAndCacheBlockMock.mockResolvedValue({ data: undefined }); + readChunkFromDiskMock.mockResolvedValue(Buffer.from('data')); + }); + + it('reads an already hydrated range from disk without downloading', async () => { + const chunk = Buffer.from('cached'); + const state = getOrCreateHydrationState(virtualFile.contentsId, virtualFile.size); + markBlocksInRangeDownloaded(state, { position: 0, length: 10 }); + readChunkFromDiskMock.mockResolvedValue(chunk); + + const result = await readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(result.data).toBe(chunk); + expect(downloadAndCacheBlockMock).not.toHaveBeenCalled(); + expect(readChunkFromDiskMock).toHaveBeenCalledWith('/tmp/cache-file', 10, 0); + }); + + it('downloads a missing range then reads requested bytes from disk', async () => { + const chunk = Buffer.from('downloaded'); + readChunkFromDiskMock.mockResolvedValue(chunk); + const deps = createDeps(); + + const result = await readOrHydrate({ + ...deps, + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(result.data).toBe(chunk); + expect(downloadAndCacheBlockMock).toHaveBeenCalledOnce(); + expect(readChunkFromDiskMock).toHaveBeenCalledWith('/tmp/cache-file', 10, 0); + }); + + it('creates hydration state for normal reads', async () => { + await readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(getExistingHydrationState(virtualFile.contentsId)).toBeDefined(); + }); + + it('allocates the cache file when it is missing', async () => { + fileExistsOnDiskMock.mockResolvedValue(false); + + await readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(allocateFileMock).toHaveBeenCalledWith('/tmp/cache-file', virtualFile.size); + }); + + it('passes progress reporting through to block hydration', async () => { + const deps = createDeps(); + downloadAndCacheBlockMock.mockImplementation(async ({ onDownloadProgress }) => { + onDownloadProgress(virtualFile.name, virtualFile.type, 10, virtualFile.size, 1); + return { data: undefined }; + }); + + await readOrHydrate({ + ...deps, + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + call(deps.onDownloadProgress).toStrictEqual([virtualFile.name, virtualFile.type, 10, virtualFile.size, 1]); + }); + + it('returns empty when in-flight hydration is aborted before the read resolves', async () => { + downloadAndCacheBlockMock.mockImplementation(async () => { + const state = getExistingHydrationState(virtualFile.contentsId); + state?.abortController.abort(); + return { data: undefined }; + }); + + const result = await readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(result.data).toStrictEqual(Buffer.alloc(0)); + expect(readChunkFromDiskMock).not.toHaveBeenCalled(); + }); + + it('returns non-abort download errors', async () => { + downloadAndCacheBlockMock.mockResolvedValue({ error: new Error('network failed') }); + + const result = await readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(result.error).toBeInstanceOf(FuseIOError); + expect(result.error?.message).toContain('network failed'); + }); + + it('returns allocation errors as Fuse IO errors', async () => { + fileExistsOnDiskMock.mockResolvedValue(false); + allocateFileMock.mockRejectedValue(new Error('disk full')); + + const result = await readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(result.error).toBeInstanceOf(FuseIOError); + }); + + it('downloads one block once for overlapping concurrent reads', async () => { + let resolveDownload: () => void = () => undefined; + downloadAndCacheBlockMock.mockImplementation( + ({ state, blockStart, blockLength }) => + new Promise<{ data: undefined }>((resolve) => { + resolveDownload = () => { + markBlocksInRangeDownloaded(state, { position: blockStart, length: blockLength }); + resolve({ data: undefined }); + }; + }), + ); + + const first = readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + const second = readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 5, length: 10 }, + }); + + await testSleep(0); + expect(downloadAndCacheBlockMock).toHaveBeenCalledOnce(); + + resolveDownload(); + + await expect(first).resolves.toHaveProperty('data', Buffer.from('data')); + await expect(second).resolves.toHaveProperty('data', Buffer.from('data')); + expect(getExistingHydrationState(virtualFile.contentsId)?.blocksBeingDownloaded.size).toBe(0); + expect(downloadAndCacheBlockMock).toHaveBeenCalledOnce(); + }); + + it('keeps overlapping reads waiting on the existing block download promise', async () => { + let resolveDownload: () => void = () => undefined; + let secondSettled = false; + downloadAndCacheBlockMock.mockImplementation( + ({ state, blockStart, blockLength }) => + new Promise<{ data: undefined }>((resolve) => { + resolveDownload = () => { + markBlocksInRangeDownloaded(state, { position: blockStart, length: blockLength }); + resolve({ data: undefined }); + }; + }), + ); + + const first = readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + const second = readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 5, length: 10 }, + }).then((result) => { + secondSettled = true; + return result; + }); + + await testSleep(0); + expect(secondSettled).toBe(false); + + resolveDownload(); + await Promise.all([first, second]); + + expect(secondSettled).toBe(true); + }); + + it('settles overlapping waiters with the failed block download result', async () => { + let resolveDownload: (result: { error: Error }) => void = () => undefined; + downloadAndCacheBlockMock.mockImplementation( + () => + new Promise((resolve) => { + resolveDownload = resolve; + }), + ); + + const first = readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + const second = readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 5, length: 10 }, + }); + + await testSleep(0); + resolveDownload({ error: new Error('range request failed') }); + + const firstResult = await first; + const secondResult = await second; + const state = getExistingHydrationState(virtualFile.contentsId); + + expect(firstResult.error).toBeInstanceOf(FuseIOError); + expect(secondResult.error).toBeInstanceOf(FuseIOError); + expect(firstResult.error?.message).toContain('range request failed'); + expect(secondResult.error?.message).toContain('range request failed'); + expect(downloadAndCacheBlockMock).toHaveBeenCalledOnce(); + expect(state?.blocksBeingDownloaded.size).toBe(0); + expect(state && isRangeHydrated(state, { position: 0, length: 10 })).toBe(false); + }); + + it('does not mark failed block downloads as hydrated and allows a later retry', async () => { + downloadAndCacheBlockMock + .mockResolvedValueOnce({ error: new Error('range request failed') }) + .mockImplementationOnce(async ({ state, blockStart, blockLength }) => { + markBlocksInRangeDownloaded(state, { position: blockStart, length: blockLength }); + return { data: undefined }; + }); + + const first = await readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + const state = getExistingHydrationState(virtualFile.contentsId); + + expect(first.error).toBeInstanceOf(FuseIOError); + expect(state?.hydratedBytes).toBe(0); + expect(state?.blocksBeingDownloaded.size).toBe(0); + + const second = await readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(second.data).toStrictEqual(Buffer.from('data')); + expect(downloadAndCacheBlockMock).toHaveBeenCalledTimes(2); + expect(state?.hydratedBytes).toBe(virtualFile.size); + }); + + it('does not mark aborted block downloads as hydrated and clears the in-flight entry', async () => { + downloadAndCacheBlockMock.mockImplementation(async ({ state }) => { + state.abortController.abort(); + return { data: undefined }; + }); + + const result = await readOrHydrate({ + ...createDeps(), + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + const state = getExistingHydrationState(virtualFile.contentsId); + + expect(result.data).toStrictEqual(Buffer.alloc(0)); + expect(state?.hydratedBytes).toBe(0); + expect(state?.blocksBeingDownloaded.size).toBe(0); + }); + + it('does not finalize partially hydrated files', async () => { + const partialFile = { ...virtualFile, size: BLOCK_SIZE + 100 } as unknown as File; + const deps = createDeps(); + const state = getOrCreateHydrationState(partialFile.contentsId, partialFile.size); + markBlocksInRangeDownloaded(state, { position: 0, length: BLOCK_SIZE }); + + const result = await readOrHydrate({ + ...deps, + virtualFile: partialFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(result.data).toStrictEqual(Buffer.from('data')); + expect(deps.saveToRepository).not.toHaveBeenCalled(); + expect(state.finalized).toBe(false); + }); + + it('treats empty files as already hydrated and finalizes without downloading blocks', async () => { + const emptyFile = { ...virtualFile, contentsId: 'empty-contents-id', size: 0 } as unknown as File; + const deps = createDeps(); + readChunkFromDiskMock.mockResolvedValue(Buffer.alloc(0)); + + const result = await readOrHydrate({ + ...deps, + virtualFile: emptyFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 0 }, + }); + const state = getExistingHydrationState(emptyFile.contentsId); + + expect(result.data).toStrictEqual(Buffer.alloc(0)); + expect(downloadAndCacheBlockMock).not.toHaveBeenCalled(); + expect(deps.saveToRepository).toHaveBeenCalledOnce(); + expect(state?.hydratedBytes).toBe(0); + expect(state?.finalized).toBe(true); + }); + + it('finalizes fully hydrated files', async () => { + const deps = createDeps(); + const state = getOrCreateHydrationState(virtualFile.contentsId, virtualFile.size); + markBlocksInRangeDownloaded(state, { position: 0, length: virtualFile.size }); + + const result = await readOrHydrate({ + ...deps, + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(result.data).toStrictEqual(Buffer.from('data')); + expect(deps.saveToRepository).toHaveBeenCalledOnce(); + expect(deps.saveToRepository).toHaveBeenCalledWith( + virtualFile.contentsId, + virtualFile.size, + virtualFile.uuid, + virtualFile.name, + virtualFile.type, + ); + expect(state.finalized).toBe(true); + }); + + it('registers once for concurrent full-hydration reads and shares in-flight finalization', async () => { + let resolveRegistration: () => void = () => undefined; + const saveToRepository = vi.fn( + () => + new Promise((resolve) => { + resolveRegistration = resolve; + }), + ); + const deps = createDeps({ saveToRepository }); + const state = getOrCreateHydrationState(virtualFile.contentsId, virtualFile.size); + markBlocksInRangeDownloaded(state, { position: 0, length: virtualFile.size }); + + const first = readOrHydrate({ + ...deps, + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + const second = readOrHydrate({ + ...deps, + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 5, length: 10 }, + }); + + await testSleep(0); + expect(saveToRepository).toHaveBeenCalledOnce(); + expect(state.finalization).toBeDefined(); + expect(state.finalized).toBe(false); + + resolveRegistration(); + + await expect(first).resolves.toHaveProperty('data', Buffer.from('data')); + await expect(second).resolves.toHaveProperty('data', Buffer.from('data')); + expect(state.finalization).toBeUndefined(); + expect(state.finalized).toBe(true); + }); + + it('allows failed finalization to be retried by a later normal read', async () => { + const saveToRepository = vi.fn().mockRejectedValueOnce(new Error('register failed')).mockResolvedValueOnce(undefined); + const deps = createDeps({ saveToRepository }); + const state = getOrCreateHydrationState(virtualFile.contentsId, virtualFile.size); + markBlocksInRangeDownloaded(state, { position: 0, length: virtualFile.size }); + + const first = await readOrHydrate({ + ...deps, + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(first.error).toBeInstanceOf(FuseIOError); + expect(first.error?.message).toContain('register failed'); + expect(state.finalization).toBeUndefined(); + expect(state.finalized).toBe(false); + + const second = await readOrHydrate({ + ...deps, + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(second.data).toStrictEqual(Buffer.from('data')); + expect(saveToRepository).toHaveBeenCalledTimes(2); + expect(state.finalized).toBe(true); + }); + + it('fires downloadFinished once after successful finalization', async () => { + const downloadFinished = vi.fn(); + const saveToRepository = vi.fn().mockImplementation(async () => { + downloadFinished(); + }); + const deps = createDeps({ saveToRepository }); + const state = getOrCreateHydrationState(virtualFile.contentsId, virtualFile.size); + markBlocksInRangeDownloaded(state, { position: 0, length: virtualFile.size }); + + await readOrHydrate({ + ...deps, + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + await readOrHydrate({ + ...deps, + virtualFile, + filePath: '/tmp/cache-file', + range: { position: 0, length: 10 }, + }); + + expect(saveToRepository).toHaveBeenCalledOnce(); + expect(downloadFinished).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/backend/features/fuse/on-read/read-or-hydrate.ts b/src/backend/features/fuse/on-read/read-or-hydrate.ts new file mode 100644 index 0000000000..8a82e56b21 --- /dev/null +++ b/src/backend/features/fuse/on-read/read-or-hydrate.ts @@ -0,0 +1,198 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { type File } from '../../../../context/virtual-drive/files/domain/File'; +import { FuseError, FuseIOError } from '../../../../apps/drive/fuse/callbacks/FuseErrors'; +import { type Result } from '../../../../context/shared/domain/Result'; +import { formatBytes } from '../../../../shared/format-bytes'; +import { readChunkFromDisk } from './read-chunk-from-disk'; +import { EMPTY } from './constants'; +import { BLOCK_SIZE } from './download-cache/constants'; +import { downloadAndCacheBlock } from './download-cache/download-and-save-block'; +import { expandToBlockBoundaries } from './download-cache/expand-to-block-boundaries'; +import { fileExistsOnDisk } from './download-cache/file-exists-on-disk'; +import { + ensureAllocatedOnce, + finalizeIfNeeded, + type FileHydrationState, + getBlocksBeingDownloaded, + getMissingBlocks, + getOrCreateHydrationState, + isFileHydrated, + isRangeHydrated, + clearBlockDownloadInFlight, + setBlockDownloadInFlight, +} from './download-cache/hydration-state'; +import { type HandleReadDeps, type ReadRange } from './types'; +export type ReadOrHydrateDeps = HandleReadDeps; + +type Props = HandleReadDeps & { + virtualFile: File; + filePath: string; + range: ReadRange; +}; + +export async function readOrHydrate({ + onDownloadProgress, + saveToRepository, + bucketId, + mnemonic, + network, + virtualFile, + filePath, + range, +}: Props): Promise> { + logger.debug({ + msg: '[ReadCallback] read request:', + file: virtualFile.nameWithExtension, + position: formatBytes(range.position), + length: formatBytes(range.length), + }); + + const state = await ensureFileAllocated(filePath, virtualFile); + if (state.error) return { error: state.error }; + if (wasAborted(state.data)) return { data: EMPTY }; + + try { + if (!isRangeHydrated(state.data, range)) { + const downloadResult = await ensureRangeDownloaded({ + onDownloadProgress, + bucketId, + mnemonic, + network, + virtualFile, + filePath, + state: state.data, + range, + }); + if (wasAborted(state.data)) return { data: EMPTY }; + if (downloadResult.error) return { error: fuseIOErrorFrom(downloadResult.error) }; + } else { + logger.debug({ msg: '[ReadCallback] serving from disk cache', file: virtualFile.nameWithExtension }); + } + + await finalizeFullyHydratedFileIfNeeded(saveToRepository, virtualFile, state.data); + if (wasAborted(state.data)) return { data: EMPTY }; + + return { data: await readChunkFromDisk(filePath, range.length, range.position) }; + } catch (error) { + if (wasAborted(state.data)) return { data: EMPTY }; + return { error: fuseIOErrorFrom(error) }; + } +} + +async function ensureFileAllocated( + filePath: string, + virtualFile: File, +): Promise> { + const state = getOrCreateHydrationState(virtualFile.contentsId, virtualFile.size); + const allocated = await fileExistsOnDisk(filePath); + if (wasAborted(state)) return { data: state }; + + if (!allocated) { + const { error } = await ensureAllocatedOnce(state, filePath, virtualFile.size); + if (error) { + return { error: new FuseIOError('Unable to allocate cache file.') }; + } + } + return { data: state }; +} + +async function ensureRangeDownloaded({ + bucketId, + mnemonic, + network, + onDownloadProgress, + virtualFile, + filePath, + state, + range, +}: { + onDownloadProgress: HandleReadDeps['onDownloadProgress']; + bucketId: HandleReadDeps['bucketId']; + mnemonic: HandleReadDeps['mnemonic']; + network: HandleReadDeps['network']; + virtualFile: File; + filePath: string; + range: ReadRange; + state: FileHydrationState; +}): Promise> { + const { blockStart, blockLength } = expandToBlockBoundaries({ range, fileSize: virtualFile.size }); + + const blocksBeingDownloaded = getBlocksBeingDownloaded(state, { position: blockStart, length: blockLength }); + if (blocksBeingDownloaded.size > 0) { + logger.debug({ + msg: '[ReadCallback] waiting for requested blocks being downloaded', + file: virtualFile.nameWithExtension, + }); + const result = await waitForBlockDownloads([...blocksBeingDownloaded.values()]); + if (result.error) return { error: result.error }; + } + + if (wasAborted(state)) return { data: undefined }; + + const missingBlocks = getMissingBlocks(state, { position: blockStart, length: blockLength }); + if (missingBlocks.length > 0) { + logger.debug({ + msg: '[ReadCallback] downloading missing blocks', + file: virtualFile.nameWithExtension, + blocks: missingBlocks, + }); + const downloads = missingBlocks.map((block) => { + const start = block * BLOCK_SIZE; + const end = Math.min(start + BLOCK_SIZE, virtualFile.size); + const download = downloadAndCacheBlock({ + bucketId, + mnemonic, + network, + onDownloadProgress, + virtualFile, + filePath, + state, + blockStart: start, + blockLength: end - start, + }); + setBlockDownloadInFlight(state, block, download); + download.finally(() => clearBlockDownloadInFlight(state, block, download)); + return download; + }); + const result = await waitForBlockDownloads(downloads); + if (result.error) return { error: result.error }; + } + + return { data: undefined }; +} + +async function waitForBlockDownloads(downloads: Array>>): Promise> { + const results = await Promise.all(downloads); + const failed = results.find((result) => result.error); + if (failed?.error) return { error: failed.error }; + return { data: undefined }; +} + +async function finalizeFullyHydratedFileIfNeeded( + saveToRepository: HandleReadDeps['saveToRepository'], + virtualFile: File, + state: FileHydrationState, +): Promise { + if (!isFileHydrated(state)) return; + + await finalizeIfNeeded(state, async () => { + await saveToRepository( + virtualFile.contentsId, + virtualFile.size, + virtualFile.uuid, + virtualFile.name, + virtualFile.type, + ); + state.stopwatch = undefined; + }); +} + +function fuseIOErrorFrom(error: unknown): FuseError { + if (error instanceof FuseError) return error; + const details = error instanceof Error ? error.message : 'Unknown error occurred'; + return new FuseIOError(details); +} + +function wasAborted(state: FileHydrationState): boolean { + return state.abortController.signal.aborted; +} diff --git a/src/backend/features/fuse/on-read/types.ts b/src/backend/features/fuse/on-read/types.ts new file mode 100644 index 0000000000..85b6b6b8d8 --- /dev/null +++ b/src/backend/features/fuse/on-read/types.ts @@ -0,0 +1,20 @@ +import { type Network } from '@internxt/sdk'; + +export type ReadRange = { + position: number; + length: number; +}; + +export type HandleReadDeps = { + onDownloadProgress: ( + name: string, + extension: string, + bytesDownloaded: number, + fileSize: number, + elapsedTime: number, + ) => void; + saveToRepository: (contentsId: string, size: number, uuid: string, name: string, extension: string) => Promise; + bucketId: string; + mnemonic: string; + network: Network.Network; +}; diff --git a/src/backend/features/virtual-drive/services/operations/read.service.test.ts b/src/backend/features/virtual-drive/services/operations/read.service.test.ts index 4f01105caa..5a6b3d048a 100644 --- a/src/backend/features/virtual-drive/services/operations/read.service.test.ts +++ b/src/backend/features/virtual-drive/services/operations/read.service.test.ts @@ -7,19 +7,37 @@ import { FirstsFileSearcher } from '../../../../../context/virtual-drive/files/a import { TemporalFileByPathFinder } from '../../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder'; import { StorageFilesRepository } from '../../../../../context/storage/StorageFiles/domain/StorageFilesRepository'; import { FuseCodes } from '../../../../../apps/drive/fuse/callbacks/FuseCodes'; +import { DownloadProgressTracker } from '../../../../../context/shared/domain/DownloadProgressTracker'; +import * as getCredentialsModule from '../../../../../apps/main/auth/get-credentials'; +import { DependencyInjectionUserProvider } from '../../../../../apps/shared/dependency-injection/DependencyInjectionUserProvider'; +import * as buildNetworkClientModule from '../../../../../infra/environment/download-file/build-network-client'; const handleReadCallbackMock = partialSpyOn(handleReadCallbackModule, 'handleReadCallback'); +const getCredentialsMock = partialSpyOn(getCredentialsModule, 'getCredentials'); +const userProviderGetMock = partialSpyOn(DependencyInjectionUserProvider, 'get'); +const buildNetworkClientMock = partialSpyOn(buildNetworkClientModule, 'buildNetworkClient'); describe('read', () => { let container: ReturnType>; const fileSearcher = mockDeep(); const temporalFinder = mockDeep(); const repo = mockDeep(); + const tracker = mockDeep(); + const network = {}; + beforeEach(() => { container = mockDeep(); container.get.calledWith(FirstsFileSearcher).mockReturnValue(fileSearcher); container.get.calledWith(TemporalFileByPathFinder).mockReturnValue(temporalFinder); container.get.calledWith(StorageFilesRepository).mockReturnValue(repo); + container.get.calledWith(DownloadProgressTracker).mockReturnValue(tracker); + getCredentialsMock.mockReturnValue({ mnemonic: 'mnemonic' } as never); + userProviderGetMock.mockReturnValue({ + bucket: 'bucket-id', + bridgeUser: 'bridge-user', + userId: 'user-id', + } as never); + buildNetworkClientMock.mockReturnValue(network as never); }); describe('when handleReadCallback succeeds', () => { @@ -38,7 +56,19 @@ describe('read', () => { await read('/file.mp4', 32768, 4096, 'vlc', container); - expect(handleReadCallbackMock).toHaveBeenCalledWith(expect.any(Object), '/file.mp4', 32768, 4096, 'vlc'); + expect(handleReadCallbackMock).toHaveBeenCalledWith( + expect.objectContaining({ + bucketId: 'bucket-id', + mnemonic: 'mnemonic', + network, + path: '/file.mp4', + range: { + length: 32768, + position: 4096, + }, + processName: 'vlc', + }), + ); }); }); diff --git a/src/backend/features/virtual-drive/services/operations/read.service.ts b/src/backend/features/virtual-drive/services/operations/read.service.ts index 018f6afdf7..e875de543a 100644 --- a/src/backend/features/virtual-drive/services/operations/read.service.ts +++ b/src/backend/features/virtual-drive/services/operations/read.service.ts @@ -49,8 +49,10 @@ export async function read( mnemonic, network, path, - length, - position, + range: { + length, + position, + }, processName, }); } catch (err) { diff --git a/src/backend/features/virtual-drive/services/virtual-drive.service.test.ts b/src/backend/features/virtual-drive/services/virtual-drive.service.test.ts new file mode 100644 index 0000000000..4cd7fd3f0c --- /dev/null +++ b/src/backend/features/virtual-drive/services/virtual-drive.service.test.ts @@ -0,0 +1,48 @@ +import { stopDaemon } from './daemon.service'; +import { stopFuseDaemonServer } from './server.service'; +import { abortAllHydrations, clearHydrationState } from '../../fuse/on-read/download-cache/hydration-state'; +import { stopVirtualDrive } from './virtual-drive.service'; + +vi.mock('@internxt/drive-desktop-core/build/backend', () => ({ + logger: { + debug: vi.fn(), + }, +})); + +vi.mock('./daemon.service', () => ({ + startDaemon: vi.fn(), + stopDaemon: vi.fn(), +})); + +vi.mock('./server.service', () => ({ + startFuseDaemonServer: vi.fn(), + stopFuseDaemonServer: vi.fn(), +})); + +vi.mock('../../fuse/on-read/download-cache/hydration-state', () => ({ + abortAllHydrations: vi.fn(), + clearHydrationState: vi.fn(), +})); + +const stopDaemonMock = vi.mocked(stopDaemon); +const stopFuseDaemonServerMock = vi.mocked(stopFuseDaemonServer); +const abortAllHydrationsMock = vi.mocked(abortAllHydrations); +const clearHydrationStateMock = vi.mocked(clearHydrationState); + +describe('stopVirtualDrive', () => { + beforeEach(() => { + vi.clearAllMocks(); + stopDaemonMock.mockResolvedValue(undefined); + stopFuseDaemonServerMock.mockResolvedValue(undefined); + }); + + it('aborts active hydrations before clearing hydration state', async () => { + await stopVirtualDrive(); + + expect(abortAllHydrationsMock).toHaveBeenCalledOnce(); + expect(clearHydrationStateMock).toHaveBeenCalledOnce(); + expect(abortAllHydrationsMock.mock.invocationCallOrder[0]).toBeLessThan( + clearHydrationStateMock.mock.invocationCallOrder[0], + ); + }); +}); diff --git a/src/backend/features/virtual-drive/services/virtual-drive.service.ts b/src/backend/features/virtual-drive/services/virtual-drive.service.ts index 122225ebfc..754e6e0d51 100644 --- a/src/backend/features/virtual-drive/services/virtual-drive.service.ts +++ b/src/backend/features/virtual-drive/services/virtual-drive.service.ts @@ -7,7 +7,7 @@ import { startFuseDaemonServer, stopFuseDaemonServer } from './server.service'; import { updateVirtualDriveContainer } from './update-virtual-drive-container.service'; import { DependencyInjectionUserProvider } from '../../../../apps/shared/dependency-injection/DependencyInjectionUserProvider'; import { StorageClearer } from '../../../../context/storage/StorageFiles/application/delete/StorageClearer'; -import { clearHydrationState } from '../../fuse/on-read/download-cache/hydration-state'; +import { abortAllHydrations, clearHydrationState } from '../../fuse/on-read/download-cache/hydration-state'; let container: Container | undefined; @@ -20,11 +20,8 @@ export async function startVirtualDrive() { container = await DriveDependencyContainerFactory.build(); await updateVirtualDriveContainer({ container, user: DependencyInjectionUserProvider.get() }); /** - * v2.5.4 - * Alexis Mora - * If a user abruptly quits the app, all the hydrated files will be orphaned. - * Hence why we clear the cache before starting up the virtual drive. - * To ensure that every time we get a fresh start. + * Clear stale block-cache state and orphaned hydrated files before mounting. + * Future virtual-drive reads recreate cache files and hydrate only requested blocks. */ clearHydrationState(); await container.get(StorageClearer).run(); @@ -34,6 +31,7 @@ export async function startVirtualDrive() { export async function stopVirtualDrive() { logger.debug({ msg: '[VIRTUAL DRIVE] stopping daemon...' }); + abortAllHydrations(); await stopDaemon(); logger.debug({ msg: '[VIRTUAL DRIVE] clearing storage cache...' }); clearHydrationState(); diff --git a/src/infra/environment/download-file/build-crypto-lib.test.ts b/src/infra/environment/download-file/build-crypto-lib.test.ts new file mode 100644 index 0000000000..094754946f --- /dev/null +++ b/src/infra/environment/download-file/build-crypto-lib.test.ts @@ -0,0 +1,65 @@ +import { Network } from '@internxt/sdk'; +import { Environment } from '@internxt/inxt-js'; +import { validateMnemonic } from 'bip39'; +import { buildCryptoLib } from './build-crypto-lib'; + +vi.mock('@internxt/sdk', () => ({ + Network: { + ALGORITHMS: { + AES256CTR: 'aes-256-ctr', + }, + }, +})); + +vi.mock('@internxt/inxt-js', () => ({ + Environment: { + utils: { + generateFileKey: vi.fn(() => Buffer.from('file-key')), + }, + }, +})); + +vi.mock('bip39', () => ({ + validateMnemonic: vi.fn(), +})); + +const validateMnemonicMock = vi.mocked(validateMnemonic); +const generateFileKeyMock = vi.mocked(Environment.utils.generateFileKey); + +describe('buildCryptoLib', () => { + beforeEach(() => { + vi.clearAllMocks(); + }); + + it('uses AES-256-CTR as the crypto algorithm', () => { + const cryptoLib = buildCryptoLib(); + + expect(cryptoLib.algorithm).toBe(Network.ALGORITHMS.AES256CTR); + }); + + it('delegates mnemonic validation to bip39', () => { + validateMnemonicMock.mockReturnValue(true); + const cryptoLib = buildCryptoLib(); + + const result = cryptoLib.validateMnemonic('seed phrase'); + + expect(result).toBe(true); + expect(validateMnemonicMock).toHaveBeenCalledWith('seed phrase'); + }); + + it('delegates file-key generation to Environment utils', () => { + const index = Buffer.from('index'); + const cryptoLib = buildCryptoLib(); + + const result = cryptoLib.generateFileKey('mnemonic', 'bucket-id', index); + + expect(result).toStrictEqual(Buffer.from('file-key')); + expect(generateFileKeyMock).toHaveBeenCalledWith('mnemonic', 'bucket-id', index); + }); + + it('exposes randomBytes from node crypto', () => { + const cryptoLib = buildCryptoLib(); + + expect(cryptoLib.randomBytes(8)).toHaveLength(8); + }); +}); diff --git a/src/infra/environment/download-file/build-network-client.test.ts b/src/infra/environment/download-file/build-network-client.test.ts new file mode 100644 index 0000000000..3cd0c873f8 --- /dev/null +++ b/src/infra/environment/download-file/build-network-client.test.ts @@ -0,0 +1,43 @@ +import { Network } from '@internxt/sdk'; +import { createHash } from 'node:crypto'; +import { INTERNXT_CLIENT, INTERNXT_VERSION } from '../../../core/utils/utils'; +import { buildNetworkClient } from './build-network-client'; + +vi.mock('@internxt/sdk', () => ({ + Network: { + Network: { + client: vi.fn(() => ({ network: true })), + }, + }, +})); + +const networkClientMock = vi.mocked(Network.Network.client); + +describe('buildNetworkClient', () => { + beforeEach(() => { + vi.clearAllMocks(); + process.env.BRIDGE_URL = 'https://bridge.test'; + process.env.INTERNXT_DESKTOP_HEADER_KEY = 'desktop-header'; + }); + + it('builds an SDK network client with app metadata and hashed user id', () => { + const client = buildNetworkClient({ + bridgeUser: 'bridge-user', + userId: 'user-id', + }); + + expect(client).toStrictEqual({ network: true }); + expect(networkClientMock).toHaveBeenCalledWith( + 'https://bridge.test', + { + clientName: INTERNXT_CLIENT, + clientVersion: INTERNXT_VERSION, + desktopHeader: 'desktop-header', + }, + { + bridgeUser: 'bridge-user', + userId: createHash('sha256').update('user-id').digest('hex'), + }, + ); + }); +}); diff --git a/src/infra/environment/download-file/decrypt-at-offset.test.ts b/src/infra/environment/download-file/decrypt-at-offset.test.ts new file mode 100644 index 0000000000..0647fdf3bb --- /dev/null +++ b/src/infra/environment/download-file/decrypt-at-offset.test.ts @@ -0,0 +1,40 @@ +import { createCipheriv } from 'node:crypto'; +import { decryptAtOffset } from './decrypt-at-offset'; + +function encrypt(plainText: Buffer, key: Buffer, iv: Buffer): Buffer { + const cipher = createCipheriv('aes-256-ctr', new Uint8Array(key), new Uint8Array(iv)); + return cipher.update(new Uint8Array(plainText)); +} + +describe('decryptAtOffset', () => { + const key = Buffer.from('00112233445566778899aabbccddeeff00112233445566778899aabbccddeeff', 'hex'); + const iv = Buffer.from('0102030405060708090a0b0c0d0e0f10', 'hex'); + const plainText = Buffer.from('abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ'); + const encrypted = encrypt(plainText, key, iv); + + it('decrypts a range starting at a block boundary', () => { + const position = 16; + const encryptedRange = encrypted.subarray(position, position + 12); + + const decrypted = decryptAtOffset(encryptedRange, key, iv, position); + + expect(decrypted).toStrictEqual(plainText.subarray(position, position + 12)); + }); + + it('decrypts a range starting in the middle of a block', () => { + const position = 19; + const encryptedRange = encrypted.subarray(position, position + 17); + + const decrypted = decryptAtOffset(encryptedRange, key, iv, position); + + expect(decrypted).toStrictEqual(plainText.subarray(position, position + 17)); + }); + + it('decrypts a range from the beginning of the file', () => { + const encryptedRange = encrypted.subarray(0, 20); + + const decrypted = decryptAtOffset(encryptedRange, key, iv, 0); + + expect(decrypted).toStrictEqual(plainText.subarray(0, 20)); + }); +}); diff --git a/src/infra/environment/download-file/download-file.test.ts b/src/infra/environment/download-file/download-file.test.ts new file mode 100644 index 0000000000..05bf8262d1 --- /dev/null +++ b/src/infra/environment/download-file/download-file.test.ts @@ -0,0 +1,71 @@ +import { Readable } from 'node:stream'; +import axios from 'axios'; +import { downloadFile as sdkDownloadFile } from '@internxt/sdk/dist/network/download'; +import { decryptAtOffset } from './decrypt-at-offset'; +import { downloadFileRange } from './download-file'; + +vi.mock('axios', () => ({ + default: { + get: vi.fn(), + }, +})); + +vi.mock('@internxt/sdk/dist/network/download', () => ({ + downloadFile: vi.fn(), +})); + +vi.mock('./build-crypto-lib', () => ({ + buildCryptoLib: vi.fn(() => ({})), +})); + +vi.mock('./decrypt-at-offset', () => ({ + decryptAtOffset: vi.fn(), +})); + +const axiosGetMock = vi.mocked(axios.get); +const sdkDownloadFileMock = vi.mocked(sdkDownloadFile); +const decryptAtOffsetMock = vi.mocked(decryptAtOffset); + +describe('downloadFileRange', () => { + beforeEach(() => { + vi.clearAllMocks(); + axiosGetMock.mockResolvedValue({ + data: Readable.from([Buffer.from('encrypted')]), + }); + decryptAtOffsetMock.mockReturnValue(Buffer.from('decrypted')); + sdkDownloadFileMock.mockImplementation(async (...args) => { + const downloadFileCb = args[6]; + const decryptFileCb = args[7]; + + await downloadFileCb([{ url: 'https://download.test/file' }] as never, 9); + await decryptFileCb( + undefined as never, + Buffer.from('keykeykeykeykeykeykeykeykeykey12'), + Buffer.from('iviviviviviviviv'), + 9, + ); + }); + }); + + it('passes the abort signal to the HTTP range request', async () => { + const abortController = new AbortController(); + + const result = await downloadFileRange({ + fileId: 'file-id', + bucketId: 'bucket-id', + mnemonic: 'mnemonic', + network: {} as never, + range: { position: 10, length: 20 }, + signal: abortController.signal, + }); + + expect(result.data).toStrictEqual(Buffer.from('decrypted')); + expect(axiosGetMock).toHaveBeenCalledWith('https://download.test/file', { + responseType: 'stream', + signal: abortController.signal, + headers: { + range: 'bytes=10-29', + }, + }); + }); +}); diff --git a/src/infra/environment/download-file/download-file.ts b/src/infra/environment/download-file/download-file.ts index 65bed096e4..5d16ce71d5 100644 --- a/src/infra/environment/download-file/download-file.ts +++ b/src/infra/environment/download-file/download-file.ts @@ -3,24 +3,8 @@ import { downloadFile as sdkDownloadFile } from '@internxt/sdk/dist/network/down import axios from 'axios'; import { buildCryptoLib } from './build-crypto-lib'; import { DownloadFileProps } from './types'; -import { DriveDesktopError } from '../../../context/shared/domain/errors/DriveDesktopError'; import { decryptAtOffset } from './decrypt-at-offset'; - -async function fetchEncryptedRange(url: string, position: number, length: number): Promise { - const response = await axios.get(url, { - responseType: 'stream', - headers: { - range: `bytes=${position}-${position + length - 1}`, - }, - }); - - return new Promise((resolve, reject) => { - const chunks: Uint8Array[] = []; - response.data.on('data', (chunk: Uint8Array) => chunks.push(chunk)); - response.data.on('end', () => resolve(Buffer.concat(chunks))); - response.data.on('error', reject); - }); -} +import { type Result } from '../../../context/shared/domain/Result'; export async function downloadFileRange({ signal, @@ -29,25 +13,33 @@ export async function downloadFileRange({ mnemonic, network, range, -}: DownloadFileProps): Promise { +}: DownloadFileProps): Promise> { let encryptedBytes: Buffer | undefined; let decryptedBuffer: Buffer | undefined; + let operationError: Error | undefined; const downloadFileCb: DownloadFileFunction = async (downloadables) => { if (range && downloadables.length > 1) { - throw new Error('Multi-Part Download with Range-Requests is not implemented'); + operationError = new Error('Multi-Part Download with Range-Requests is not implemented'); + return; } for (const downloadable of downloadables) { - if (signal.signal.aborted) { - throw new DriveDesktopError('ABORTED'); + if (signal.aborted) { + return; } // eslint-disable-next-line no-await-in-loop - encryptedBytes = await fetchEncryptedRange(downloadable.url, range.position, range.length); + encryptedBytes = await fetchEncryptedRange(downloadable.url, range.position, range.length, signal); } }; const decryptFileCb: DecryptFileFunction = async (_, key, iv) => { - if (!encryptedBytes) throw new Error('No encrypted bytes to decrypt'); + if (signal.aborted) { + return; + } + if (!encryptedBytes) { + operationError = new Error('No encrypted bytes to decrypt'); + return; + } decryptedBuffer = decryptAtOffset( encryptedBytes, Buffer.from(key.toString('hex'), 'hex'), @@ -56,17 +48,50 @@ export async function downloadFileRange({ ); }; - await sdkDownloadFile( - fileId, - bucketId, - mnemonic, - network, - buildCryptoLib(), - Buffer.from, - downloadFileCb, - decryptFileCb, + try { + await sdkDownloadFile( + fileId, + bucketId, + mnemonic, + network, + buildCryptoLib(), + Buffer.from, + downloadFileCb, + decryptFileCb, ); + } catch (error) { + if (signal.aborted) return abortedDownloadResult(); + return { error: error instanceof Error ? error : new Error('Unknown error occurred') }; + } - if (!decryptedBuffer) throw new Error('Decryption did not produce a buffer'); - return decryptedBuffer; + if (signal.aborted) return abortedDownloadResult(); + if (operationError) return { error: operationError }; + if (!decryptedBuffer) return { error: new Error('Decryption did not produce a buffer') }; + return { data: decryptedBuffer }; +} + +function abortedDownloadResult(): Result { + return { data: Buffer.alloc(0) }; +} + +async function fetchEncryptedRange( + url: string, + position: number, + length: number, + signal: AbortSignal, +): Promise { + const response = await axios.get(url, { + responseType: 'stream', + signal, + headers: { + range: `bytes=${position}-${position + length - 1}`, + }, + }); + + return new Promise((resolve, reject) => { + const chunks: Uint8Array[] = []; + response.data.on('data', (chunk: Uint8Array) => chunks.push(chunk)); + response.data.on('end', () => resolve(Buffer.concat(chunks))); + response.data.on('error', reject); + }); } diff --git a/src/infra/environment/download-file/types.ts b/src/infra/environment/download-file/types.ts index 5c6c681983..8568a189b1 100644 --- a/src/infra/environment/download-file/types.ts +++ b/src/infra/environment/download-file/types.ts @@ -1,7 +1,7 @@ import { Network } from '@internxt/sdk'; export type DownloadFileProps = { - signal: AbortController; + signal: AbortSignal; fileId: string; bucketId: string; mnemonic: string; From c473bc5108034f5371fcb812ae3e432ad7ed1fbe Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Tue, 5 May 2026 19:36:43 +0200 Subject: [PATCH 08/10] fix:format --- .eslintrc.js | 2 +- .../on-read/download-cache/download-and-save-block.ts | 8 +------- .../fuse/on-read/download-cache/read-if-hydrated.ts | 6 +----- src/backend/features/fuse/on-read/read-or-hydrate.test.ts | 5 ++++- src/infra/environment/download-file/download-file.ts | 2 +- 5 files changed, 8 insertions(+), 15 deletions(-) diff --git a/.eslintrc.js b/.eslintrc.js index 4970826224..92f3a8cbe4 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -1,6 +1,6 @@ module.exports = { extends: ['@internxt/eslint-config-internxt'], - ignorePatterns: ['src/infra/schemas.d.ts'], + ignorePatterns: ['src/infra/schemas.d.ts', 'assets/assets.d.ts'], overrides: [ { files: ['*.ts', '*.tsx'], diff --git a/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts b/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts index ecbc769b82..ccfccc5521 100644 --- a/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts +++ b/src/backend/features/fuse/on-read/download-cache/download-and-save-block.ts @@ -49,13 +49,7 @@ export async function downloadAndCacheBlock({ markBlocksInRangeDownloaded(state, { position: blockStart, length: blockLength }); const elapsedTime = state.stopwatch?.elapsedTime() ?? 0; - onDownloadProgress( - virtualFile.name, - virtualFile.type, - getHydratedBytes(state), - virtualFile.size, - elapsedTime, - ); + onDownloadProgress(virtualFile.name, virtualFile.type, getHydratedBytes(state), virtualFile.size, elapsedTime); return { data: undefined }; } catch (error) { if (isAborted(state)) return { data: undefined }; diff --git a/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.ts b/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.ts index 13c41497c2..a204506a80 100644 --- a/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.ts +++ b/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.ts @@ -6,11 +6,7 @@ type Range = { length: number; }; -export async function readIfHydrated( - filePath: string, - contentsId: string, - range: Range, -): Promise { +export async function readIfHydrated(filePath: string, contentsId: string, range: Range): Promise { const state = getExistingHydrationState(contentsId); if (!state) return undefined; if (!isRangeHydrated(state, range)) return undefined; diff --git a/src/backend/features/fuse/on-read/read-or-hydrate.test.ts b/src/backend/features/fuse/on-read/read-or-hydrate.test.ts index 21092ee453..4fbd395dc6 100644 --- a/src/backend/features/fuse/on-read/read-or-hydrate.test.ts +++ b/src/backend/features/fuse/on-read/read-or-hydrate.test.ts @@ -436,7 +436,10 @@ describe('readOrHydrate', () => { }); it('allows failed finalization to be retried by a later normal read', async () => { - const saveToRepository = vi.fn().mockRejectedValueOnce(new Error('register failed')).mockResolvedValueOnce(undefined); + const saveToRepository = vi + .fn() + .mockRejectedValueOnce(new Error('register failed')) + .mockResolvedValueOnce(undefined); const deps = createDeps({ saveToRepository }); const state = getOrCreateHydrationState(virtualFile.contentsId, virtualFile.size); markBlocksInRangeDownloaded(state, { position: 0, length: virtualFile.size }); diff --git a/src/infra/environment/download-file/download-file.ts b/src/infra/environment/download-file/download-file.ts index 5d16ce71d5..9f250d1589 100644 --- a/src/infra/environment/download-file/download-file.ts +++ b/src/infra/environment/download-file/download-file.ts @@ -58,7 +58,7 @@ export async function downloadFileRange({ Buffer.from, downloadFileCb, decryptFileCb, - ); + ); } catch (error) { if (signal.aborted) return abortedDownloadResult(); return { error: error instanceof Error ? error : new Error('Unknown error occurred') }; From 1843063acd0f05cff0b382f8b3a2efe976a016b6 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Wed, 6 May 2026 12:41:07 +0200 Subject: [PATCH 09/10] fix: delete orphaned files --- ...OrmAndNodeFsStorageFilesRepository.test.ts | 120 ++++++++---------- .../TypeOrmAndNodeFsStorageFilesRepository.ts | 13 +- 2 files changed, 63 insertions(+), 70 deletions(-) diff --git a/src/context/storage/StorageFiles/infrastructure/persistance/repository/typeorm/TypeOrmAndNodeFsStorageFilesRepository.test.ts b/src/context/storage/StorageFiles/infrastructure/persistance/repository/typeorm/TypeOrmAndNodeFsStorageFilesRepository.test.ts index ef1759b405..0a2db617ae 100644 --- a/src/context/storage/StorageFiles/infrastructure/persistance/repository/typeorm/TypeOrmAndNodeFsStorageFilesRepository.test.ts +++ b/src/context/storage/StorageFiles/infrastructure/persistance/repository/typeorm/TypeOrmAndNodeFsStorageFilesRepository.test.ts @@ -1,71 +1,53 @@ -// import 'reflect-metadata'; -// import path from 'node:path'; -// import { DataSource } from 'typeorm'; -// import { TypeOrmAndNodeFsStorageFilesRepository } from './TypeOrmAndNodeFsStorageFilesRepository'; -// import { obtainSqliteDataSource } from './__test-helpers__/sqlDataSource'; -// import { StorageFileMother } from '../../../../../__test-helpers__/StorageFileMother'; -// import { createReadable } from './__test-helpers__/createReadable'; -// import { createFile } from './__test-helpers__/createFile'; +import 'reflect-metadata'; +import { mkdtemp, mkdir, readdir, rm, writeFile } from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { DataSource } from 'typeorm'; +import { TypeOrmAndNodeFsStorageFilesRepository } from './TypeOrmAndNodeFsStorageFilesRepository'; -/** - * SKIPPED: This test requires better-sqlite3 native module which must be compiled - * for the specific Node.js version being used. - * - * The production app runs on Node.js v16 (NODE_MODULE_VERSION 106), but if you're - * running tests with a different Node version, better-sqlite3 will fail to load. - * - * ADDITIONAL ISSUE: typeorm and better-sqlite3 are installed in release/app/package.json - * (separate from the main package.json), which creates module resolution issues in the - * test environment. The proper solution is to consolidate into a single package.json. - * - * To fix: - * 1. Use Node.js v16 to match production: `nvm use 16` - * 2. Or rebuild the native module: `npm rebuild better-sqlite3` - * 3. Or run tests in a container with the correct Node version - * - * Once you're on Node v16, remove the .skip to enable this test. - */ -describe.skip('TypeOrmAndNodeFsStorageFilesRepository', () => { - // const directory = 'sqlite'; - // // let dataSource: DataSource; - // let repository: TypeOrmAndNodeFsStorageFilesRepository; - // beforeAll(async () => { - // const on = path.join(__dirname, directory); - // dataSource = await obtainSqliteDataSource(on); - // repository = new TypeOrmAndNodeFsStorageFilesRepository(on, dataSource); - // }); - // afterAll(async () => { - // await dataSource?.dropDatabase(); - // }); - // afterEach(async () => { - // await repository.deleteAll(); - // }); - // it('stores and retrieve a file from database and file system', async () => { - // const file = StorageFileMother.random(); - // const content = 'Hello Wold!!'; - // await repository.store(file, createReadable(content)); - // const retrievedBuffer = await repository.read(file.id); - // expect(retrievedBuffer.toString()).toBe(content); - // }); - // it('deletes a stored file', async () => { - // const file = await createFile(repository); - // await repository.delete(file.id); - // const result = await repository.exists(file.id); - // expect(result).toBe(false); - // }); - // it('finds a file after being stored', async () => { - // const stored = await createFile(repository); - // const exists = await repository.exists(stored.id); - // expect(exists).toBe(true); - // }); - // it('retrieves a stored Storage File', async () => { - // const stored = await createFile(repository); - // const retrieved = await repository.retrieve(stored.id); - // expect(stored).toEqual(retrieved); - // }); - // it('returns all files', async () => { - // const files = await Promise.all([createFile(repository), createFile(repository), createFile(repository)]); - // const allFilesRetrieved = await repository.all(); - // expect(files).toEqual(expect.arrayContaining(allFilesRetrieved)); - // }); +describe('TypeOrmAndNodeFsStorageFilesRepository', () => { + let baseFolder: string; + let db: { + find: ReturnType; + delete: ReturnType; + }; + let repository: TypeOrmAndNodeFsStorageFilesRepository; + + beforeEach(async () => { + baseFolder = await mkdtemp(path.join(os.tmpdir(), 'storage-files-repository-')); + db = { + find: vi.fn().mockResolvedValue([]), + delete: vi.fn().mockResolvedValue(undefined), + }; + + const dataSource = { + getRepository: vi.fn().mockReturnValue(db), + } as unknown as DataSource; + + repository = new TypeOrmAndNodeFsStorageFilesRepository(baseFolder, dataSource); + }); + + afterEach(async () => { + await rm(baseFolder, { recursive: true, force: true }); + }); + + it('deletes orphaned files from the storage folder when deleting all', async () => { + await writeFile(path.join(baseFolder, 'orphaned-contents-id'), 'partial hydration'); + + await repository.deleteAll(); + + await expect(readdir(baseFolder)).resolves.toEqual([]); + }); + + it('deletes registered files and any remaining orphaned files from the storage folder', async () => { + db.find.mockResolvedValue([{ id: 'registeredcontentsid0000' }]); + await writeFile(path.join(baseFolder, 'registeredcontentsid0000'), 'hydrated file'); + await writeFile(path.join(baseFolder, 'orphaned-contents-id'), 'partial hydration'); + await mkdir(path.join(baseFolder, 'nested-directory')); + + await repository.deleteAll(); + + expect(db.delete).toHaveBeenCalledWith({ id: 'registeredcontentsid0000' }); + await expect(readdir(baseFolder)).resolves.toEqual(['nested-directory']); + }); }); diff --git a/src/context/storage/StorageFiles/infrastructure/persistance/repository/typeorm/TypeOrmAndNodeFsStorageFilesRepository.ts b/src/context/storage/StorageFiles/infrastructure/persistance/repository/typeorm/TypeOrmAndNodeFsStorageFilesRepository.ts index 377fac1d87..e1634bdcf4 100644 --- a/src/context/storage/StorageFiles/infrastructure/persistance/repository/typeorm/TypeOrmAndNodeFsStorageFilesRepository.ts +++ b/src/context/storage/StorageFiles/infrastructure/persistance/repository/typeorm/TypeOrmAndNodeFsStorageFilesRepository.ts @@ -1,6 +1,6 @@ import { Service } from 'diod'; import { Readable } from 'form-data'; -import { readFile, unlink } from 'fs/promises'; +import { readFile, readdir, unlink } from 'fs/promises'; import path from 'path'; import { DataSource, Repository } from 'typeorm'; import { tryCatch } from '../../../../../../../shared/try-catch'; @@ -86,6 +86,7 @@ export class TypeOrmAndNodeFsStorageFilesRepository implements StorageFilesRepos .map((id: StorageFileId) => this.delete(id)); await Promise.all(deleted); + await this.deleteOrphanFilesFromBaseFolder(); } async all(): Promise { @@ -93,4 +94,14 @@ export class TypeOrmAndNodeFsStorageFilesRepository implements StorageFilesRepos return all.map(StorageFile.from); } + + private async deleteOrphanFilesFromBaseFolder(): Promise { + const entries = await readdir(this.baseFolder, { withFileTypes: true }); + const deleted = entries + .filter((entry) => entry.isFile() || entry.isSymbolicLink()) + .map((entry) => path.join(this.baseFolder, entry.name)) + .map((pathToUnlink) => tryCatch(() => unlink(pathToUnlink))); + + await Promise.all(deleted); + } } From 44017f88a8b8f9861bf48c54957f6948efe93870 Mon Sep 17 00:00:00 2001 From: AlexisMora Date: Fri, 8 May 2026 12:39:59 +0200 Subject: [PATCH 10/10] fix: tests + removed unused code --- .../download-cache/allocate-file.test.ts | 4 ---- .../download-and-save-block.test.ts | 1 - .../download-cache/file-exists-on-disk.test.ts | 4 ---- .../download-cache/hydration-stopwatch.ts | 17 ----------------- .../download-cache/read-if-hydrated.test.ts | 1 - .../features/fuse/on-read/read-or-hydrate.ts | 7 ++++--- .../services/virtual-drive.service.test.ts | 7 ------- .../download-file/build-crypto-lib.test.ts | 4 ---- .../download-file/build-network-client.test.ts | 1 - .../download-file/build-network-client.ts | 2 +- .../download-file/download-file.test.ts | 1 - 11 files changed, 5 insertions(+), 44 deletions(-) delete mode 100644 src/backend/features/fuse/on-read/download-cache/hydration-stopwatch.ts diff --git a/src/backend/features/fuse/on-read/download-cache/allocate-file.test.ts b/src/backend/features/fuse/on-read/download-cache/allocate-file.test.ts index c4887aa787..88bed40bf3 100644 --- a/src/backend/features/fuse/on-read/download-cache/allocate-file.test.ts +++ b/src/backend/features/fuse/on-read/download-cache/allocate-file.test.ts @@ -17,10 +17,6 @@ function createHandle() { } describe('allocateFile', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - it('opens the file for writing and truncates it to the requested size', async () => { const handle = createHandle(); fsMock.open.mockResolvedValue(handle as unknown as Awaited>); diff --git a/src/backend/features/fuse/on-read/download-cache/download-and-save-block.test.ts b/src/backend/features/fuse/on-read/download-cache/download-and-save-block.test.ts index dfe0c025e3..f082fda5c4 100644 --- a/src/backend/features/fuse/on-read/download-cache/download-and-save-block.test.ts +++ b/src/backend/features/fuse/on-read/download-cache/download-and-save-block.test.ts @@ -60,7 +60,6 @@ function createProps(overrides: Partial describe('downloadAndCacheBlock', () => { beforeEach(() => { clearHydrationState(); - vi.clearAllMocks(); downloadFileRangeMock.mockResolvedValue({ data: Buffer.from('downloaded') }); writeChunkToDiskMock.mockResolvedValue(undefined); }); diff --git a/src/backend/features/fuse/on-read/download-cache/file-exists-on-disk.test.ts b/src/backend/features/fuse/on-read/download-cache/file-exists-on-disk.test.ts index 164b58b73f..2b5318f6f1 100644 --- a/src/backend/features/fuse/on-read/download-cache/file-exists-on-disk.test.ts +++ b/src/backend/features/fuse/on-read/download-cache/file-exists-on-disk.test.ts @@ -10,10 +10,6 @@ vi.mock('node:fs/promises', () => ({ const fsMock = vi.mocked(fs); describe('fileExistsOnDisk', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - it('returns true when fs.stat succeeds', async () => { fsMock.stat.mockResolvedValue({} as Awaited>); diff --git a/src/backend/features/fuse/on-read/download-cache/hydration-stopwatch.ts b/src/backend/features/fuse/on-read/download-cache/hydration-stopwatch.ts deleted file mode 100644 index 08516a4952..0000000000 --- a/src/backend/features/fuse/on-read/download-cache/hydration-stopwatch.ts +++ /dev/null @@ -1,17 +0,0 @@ -import { Stopwatch } from '../../../../../apps/shared/types/Stopwatch'; - -const stopwatches = new Map(); - -export function startStopwatch(contentsId: string): void { - const stopWatch = new Stopwatch(); - stopWatch.start(); - stopwatches.set(contentsId, stopWatch); -} - -export function getStopwatch(contentsId: string): Stopwatch | undefined { - return stopwatches.get(contentsId); -} - -export function deleteStopwatch(contentsId: string): void { - stopwatches.delete(contentsId); -} diff --git a/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.test.ts b/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.test.ts index 6e9b203f7f..033b9d6639 100644 --- a/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.test.ts +++ b/src/backend/features/fuse/on-read/download-cache/read-if-hydrated.test.ts @@ -16,7 +16,6 @@ const readChunkFromDiskMock = vi.mocked(readChunkFromDisk); describe('readIfHydrated', () => { beforeEach(() => { clearHydrationState(); - vi.clearAllMocks(); }); it('returns undefined when no hydration state exists', async () => { diff --git a/src/backend/features/fuse/on-read/read-or-hydrate.ts b/src/backend/features/fuse/on-read/read-or-hydrate.ts index 8a82e56b21..0b02e20180 100644 --- a/src/backend/features/fuse/on-read/read-or-hydrate.ts +++ b/src/backend/features/fuse/on-read/read-or-hydrate.ts @@ -52,7 +52,10 @@ export async function readOrHydrate({ if (wasAborted(state.data)) return { data: EMPTY }; try { - if (!isRangeHydrated(state.data, range)) { + if (isRangeHydrated(state.data, range)) { + logger.debug({ msg: '[ReadCallback] serving from disk cache', file: virtualFile.nameWithExtension }); + } else { + logger.debug({ msg: '[ReadCallback] downloading range', file: virtualFile.nameWithExtension }); const downloadResult = await ensureRangeDownloaded({ onDownloadProgress, bucketId, @@ -65,8 +68,6 @@ export async function readOrHydrate({ }); if (wasAborted(state.data)) return { data: EMPTY }; if (downloadResult.error) return { error: fuseIOErrorFrom(downloadResult.error) }; - } else { - logger.debug({ msg: '[ReadCallback] serving from disk cache', file: virtualFile.nameWithExtension }); } await finalizeFullyHydratedFileIfNeeded(saveToRepository, virtualFile, state.data); diff --git a/src/backend/features/virtual-drive/services/virtual-drive.service.test.ts b/src/backend/features/virtual-drive/services/virtual-drive.service.test.ts index 4cd7fd3f0c..d1e17e1130 100644 --- a/src/backend/features/virtual-drive/services/virtual-drive.service.test.ts +++ b/src/backend/features/virtual-drive/services/virtual-drive.service.test.ts @@ -2,13 +2,6 @@ import { stopDaemon } from './daemon.service'; import { stopFuseDaemonServer } from './server.service'; import { abortAllHydrations, clearHydrationState } from '../../fuse/on-read/download-cache/hydration-state'; import { stopVirtualDrive } from './virtual-drive.service'; - -vi.mock('@internxt/drive-desktop-core/build/backend', () => ({ - logger: { - debug: vi.fn(), - }, -})); - vi.mock('./daemon.service', () => ({ startDaemon: vi.fn(), stopDaemon: vi.fn(), diff --git a/src/infra/environment/download-file/build-crypto-lib.test.ts b/src/infra/environment/download-file/build-crypto-lib.test.ts index 094754946f..05b1003f1f 100644 --- a/src/infra/environment/download-file/build-crypto-lib.test.ts +++ b/src/infra/environment/download-file/build-crypto-lib.test.ts @@ -27,10 +27,6 @@ const validateMnemonicMock = vi.mocked(validateMnemonic); const generateFileKeyMock = vi.mocked(Environment.utils.generateFileKey); describe('buildCryptoLib', () => { - beforeEach(() => { - vi.clearAllMocks(); - }); - it('uses AES-256-CTR as the crypto algorithm', () => { const cryptoLib = buildCryptoLib(); diff --git a/src/infra/environment/download-file/build-network-client.test.ts b/src/infra/environment/download-file/build-network-client.test.ts index 3cd0c873f8..dbc48e9df6 100644 --- a/src/infra/environment/download-file/build-network-client.test.ts +++ b/src/infra/environment/download-file/build-network-client.test.ts @@ -15,7 +15,6 @@ const networkClientMock = vi.mocked(Network.Network.client); describe('buildNetworkClient', () => { beforeEach(() => { - vi.clearAllMocks(); process.env.BRIDGE_URL = 'https://bridge.test'; process.env.INTERNXT_DESKTOP_HEADER_KEY = 'desktop-header'; }); diff --git a/src/infra/environment/download-file/build-network-client.ts b/src/infra/environment/download-file/build-network-client.ts index 43414b2308..4a45688e36 100644 --- a/src/infra/environment/download-file/build-network-client.ts +++ b/src/infra/environment/download-file/build-network-client.ts @@ -9,7 +9,7 @@ export type NetworkClientCredentials = { export function buildNetworkClient(credentials: NetworkClientCredentials): Network.Network { return Network.Network.client( - process.env.BRIDGE_URL as string, + process.env.BRIDGE_URL, { clientName: INTERNXT_CLIENT, clientVersion: INTERNXT_VERSION, diff --git a/src/infra/environment/download-file/download-file.test.ts b/src/infra/environment/download-file/download-file.test.ts index 05bf8262d1..e805d8c900 100644 --- a/src/infra/environment/download-file/download-file.test.ts +++ b/src/infra/environment/download-file/download-file.test.ts @@ -28,7 +28,6 @@ const decryptAtOffsetMock = vi.mocked(decryptAtOffset); describe('downloadFileRange', () => { beforeEach(() => { - vi.clearAllMocks(); axiosGetMock.mockResolvedValue({ data: Readable.from([Buffer.from('encrypted')]), });