[PB-6217]: Implement range requests reads for hydrating virtual files

.eslintrc.js

@@ -1,6 +1,6 @@
 module.exports = {
   extends: ['@internxt/eslint-config-internxt'],
-  ignorePatterns: ['src/infra/schemas.d.ts'],
+  ignorePatterns: ['src/infra/schemas.d.ts', 'assets/assets.d.ts'],
   overrides: [
     {
       files: ['*.ts', '*.tsx'],
@@ -14,7 +14,7 @@ module.exports = {
   ],
   rules: {
     'no-await-in-loop': 'warn',
-    'no-use-before-define': 'warn',
+    '@typescript-eslint/no-use-before-define': ['warn', { functions: false, classes: true, variables: true }],
     'array-callback-return': 'warn',
     'max-len': [
       'warn', // TODO: Change back to 'error' after fixing existing violations

.gitignore

@@ -56,3 +56,4 @@ enable-sso.sh
 .github/copilot-instructions.md
 .github/instructions/contributing.instructions.md
 .github/instructions/testing.instructions.md
+.codex

src/apps/drive/fuse/FuseApp.ts

@@ -1,7 +1,7 @@
 import { Container } from 'diod';
 import { logger } from '@internxt/drive-desktop-core/build/backend';
 import { StorageClearer } from '../../../context/storage/StorageFiles/application/delete/StorageClearer';
-import { destroyAllHydrations } from '../../../backend/features/fuse/on-read/hydration-registry';
+import { clearHydrationState } from '../../../backend/features/fuse/on-read/download-cache/hydration-state';
 import { VirtualDrive } from '../virtual-drive/VirtualDrive';
 import { FuseDriveStatus } from './FuseDriveStatus';
 import { CreateCallback } from './callbacks/CreateCallback';
@@ -116,7 +116,7 @@ export class FuseApp extends EventEmitter {
   }
 
   async clearCache(): Promise<void> {
-    await destroyAllHydrations();
+    clearHydrationState();
     await this.container.get(StorageClearer).run();
   }
 

src/apps/drive/fuse/callbacks/ReadCallback.ts

@@ -1,55 +1,59 @@
 // import { Container } from 'diod';
 // import { logger } from '@internxt/drive-desktop-core/build/backend';
+// import { TemporalFileByPathFinder } from '../../../../context/storage/TemporalFiles/application/find/TemporalFileByPathFinder';
+// import { TemporalFileChunkReader } from '../../../../context/storage/TemporalFiles/application/read/TemporalFileChunkReader';
 // import { FirstsFileSearcher } from '../../../../context/virtual-drive/files/application/search/FirstsFileSearcher';
 // import { StorageFilesRepository } from '../../../../context/storage/StorageFiles/domain/StorageFilesRepository';
-// import { StorageFileId } from '../../../../context/storage/StorageFiles/domain/StorageFileId';
 // import { StorageFile } from '../../../../context/storage/StorageFiles/domain/StorageFile';
 // import { DownloadProgressTracker } from '../../../../context/shared/domain/DownloadProgressTracker';
-// import { type File } from '../../../../context/virtual-drive/files/domain/File';
 // import {
 //   handleReadCallback,
 //   type HandleReadCallbackDeps,
 // } from '../../../../backend/features/fuse/on-read/handle-read-callback';
-
-// import Fuse from '@gcas/fuse';
+// import { buildNetworkClient } from '../../../../infra/environment/download-file/build-network-client';
+// import { getCredentials } from '../../../main/auth/get-credentials';
+// import { DependencyInjectionUserProvider } from '../../../shared/dependency-injection/DependencyInjectionUserProvider';
 
 // export class ReadCallback {
 //   constructor(private readonly container: Container) {}
 
-//   async execute(path: string, _fd: number, buf: Buffer, len: number, pos: number, cb: (bytesRead?: number) => void) {
-//     try {
-//       const repo = this.container.get(StorageFilesRepository);
-//       const downloader = this.container.get(StorageFileDownloader);
-//       const tracker = this.container.get(DownloadProgressTracker);
-
-//       const deps: HandleReadCallbackDeps = {
-//         findVirtualFile: (p: string) => this.container.get(FirstsFileSearcher).run({ path: p }),
-//         findTemporalFile: (p: string) => this.container.get(TemporalFileByPathFinder).run(p),
-//         existsOnDisk: (contentsId: string) => repo.exists(new StorageFileId(contentsId)),
+// async execute(
+//   path: string,
+//   _fd: unknown,
+//   buf: Buffer,
+//   len: number,
+//   pos: number,
+//   cb: (code: number, params?: unknown) => void,
+// ) {
+//   try {
+//     const { mnemonic } = getCredentials();
+//     const user = DependencyInjectionUserProvider.get();
+//     const network = buildNetworkClient({ bridgeUser: user.bridgeUser, userId: user.userId });
+//     const repo = this.container.get(StorageFilesRepository);
+//     const tracker = this.container.get(DownloadProgressTracker);
 
-//         startDownload: async (virtualFile: File) => {
-//           const storage = StorageFile.from({
-//             id: virtualFile.contentsId,
-//             virtualId: virtualFile.uuid,
-//             size: virtualFile.size,
-//           });
-//           tracker.downloadStarted(virtualFile.name, virtualFile.type);
-//           const { stream, handler } = await downloader.run(storage, virtualFile);
-//           return { stream, elapsedTime: () => handler.elapsedTime() };
-//         },
-//         onDownloadProgress: (name, extension, progress) => {
-//           tracker.downloadUpdate(name, extension, progress);
-//         },
-//         saveToRepository: async (contentsId, size, uuid, name, extension) => {
-//           const storage = StorageFile.from({
-//             id: contentsId,
-//             virtualId: uuid,
-//             size,
-//           });
-//           await repo.register(storage);
-//           tracker.downloadFinished(name, extension);
-//         },
-//       };
+//     const deps: HandleReadCallbackDeps = {
+//       findVirtualFile: (p: string) => this.container.get(FirstsFileSearcher).run({ path: p }),
+//       findTemporalFile: (p: string) => this.container.get(TemporalFileByPathFinder).run(p),
+//       readTemporalFileChunk: async (p: string, length: number, position: number) => {
+//         const result = await this.container.get(TemporalFileChunkReader).run(p, length, position);
+//         return result.isPresent() ? result.get() : undefined;
+//       },
+//       onDownloadProgress: (name, extension, bytesDownloaded, fileSize, elapsedTime) => {
+//         tracker.downloadUpdate(name, extension, {
+//           percentage: Math.min(bytesDownloaded / fileSize, 1),
+//           elapsedTime,
+//         });
+//       },
+//       saveToRepository: async (contentsId, size, uuid, name, extension) => {
+//         const storage = StorageFile.from({ id: contentsId, virtualId: uuid, size });
+//         await repo.register(storage);
+//         tracker.downloadFinished(name, extension);
+//       },
+//       bucketId: user.bucket,
+//       mnemonic,
+//       network,
+//     };
 
 //       const result = await handleReadCallback(deps, path, len, pos);
 

src/apps/main/network/downloadv2.ts

@@ -104,6 +104,7 @@ const downloadOwnFile: DownloadOwnFileFunction = (params) => {
 
 const downloadFileV2: DownloadFileFunction = (params) => {
   if (params.token && params.encryptionKey) {
+    // This is de facto dead code as its never called with params.token
     return downloadSharedFile(params as DownloadSharedFileParams);
   } else if (params.creds && params.mnemonic) {
     return downloadOwnFile(params as DownloadOwnFileParams);

src/backend/features/fuse/on-read/constants.ts

@@ -0,0 +1 @@
+export const EMPTY = Buffer.alloc(0);

src/backend/features/fuse/on-read/download-cache/allocate-file.test.ts

@@ -0,0 +1,54 @@
+import fs from 'node:fs/promises';
+import { allocateFile } from './allocate-file';
+
+vi.mock('node:fs/promises', () => ({
+  default: {
+    open: vi.fn(),
+  },
+}));
+
+const fsMock = vi.mocked(fs);
+
+function createHandle() {
+  return {
+    truncate: vi.fn().mockResolvedValue(undefined),
+    close: vi.fn().mockResolvedValue(undefined),
+  };
+}
+
+describe('allocateFile', () => {
+  it('opens the file for writing and truncates it to the requested size', async () => {
+    const handle = createHandle();
+    fsMock.open.mockResolvedValue(handle as unknown as Awaited<ReturnType<typeof fs.open>>);
+
+    await allocateFile('/tmp/cache-file', 1024);
+
+    expect(fsMock.open).toHaveBeenCalledWith('/tmp/cache-file', 'w');
+    expect(handle.truncate).toHaveBeenCalledWith(1024);
+  });
+
+  it('closes the file handle after successful allocation', async () => {
+    const handle = createHandle();
+    fsMock.open.mockResolvedValue(handle as unknown as Awaited<ReturnType<typeof fs.open>>);
+
+    await allocateFile('/tmp/cache-file', 1024);
+
+    expect(handle.close).toHaveBeenCalledOnce();
+  });
+
+  it('closes the file handle when truncate fails', async () => {
+    const handle = createHandle();
+    handle.truncate.mockRejectedValue(new Error('truncate failed'));
+    fsMock.open.mockResolvedValue(handle as unknown as Awaited<ReturnType<typeof fs.open>>);
+
+    await expect(allocateFile('/tmp/cache-file', 1024)).rejects.toThrow('truncate failed');
+
+    expect(handle.close).toHaveBeenCalledOnce();
+  });
+
+  it('propagates open failures', async () => {
+    fsMock.open.mockRejectedValue(new Error('open failed'));
+
+    await expect(allocateFile('/tmp/cache-file', 1024)).rejects.toThrow('open failed');
+  });
+});

src/backend/features/fuse/on-read/download-cache/allocate-file.ts

@@ -0,0 +1,21 @@
+import fs from 'node:fs/promises';
+
+/**
+ * Pre-allocates a file on disk to the full expected size before any ranges are downloaded.
+ *
+ * This is necessary for random-access writes: since FUSE reads can arrive in any order,
+ * we need the file to exist at its full size so we can write each range at its correct
+ * byte offset. Without pre-allocation, writing at offset 500MB would fail because the
+ * file doesn't exist yet.
+ *
+ * The file is filled with zeros initially, the {@link rangeRegistry} tracks which regions
+ * contain real downloaded bytes vs unfilled zeros.
+ */
+export async function allocateFile(filePath: string, size: number): Promise<void> {
+  const handle = await fs.open(filePath, 'w');
+  try {
+    await handle.truncate(size);
+  } finally {
+    await handle.close();
+  }
+}

src/backend/features/fuse/on-read/download-cache/constants.ts

@@ -0,0 +1,7 @@
+/**
+ * 4MB blocks — matches the chunk size used by the legacy downloader, proven to work well
+ * for this codebase. Each block is downloaded in full on first access regardless of how
+ * small the FUSE read is, so subsequent reads within the same block are served from disk.
+ */
+export const BLOCK_SIZE = 4 * 1024 * 1024;
+export const BITS_PER_BYTE = 8;