diff --git a/src/apps/main/interface.d.ts b/src/apps/main/interface.d.ts index 6b62365cd0..0389b4dae6 100644 --- a/src/apps/main/interface.d.ts +++ b/src/apps/main/interface.d.ts @@ -157,8 +157,10 @@ export interface IElectronAPI { startRemoteSync(): Promise; getUpdateStatus(): Promise<{ version: string } | null>; onUpdateAvailable(callback: (info: { version: string }) => void): () => void; - getRemoteSyncStatus(): Promise; - onRemoteSyncStatusChange(callback: (status: import('./remote-sync/helpers').RemoteSyncStatus) => void): () => void; + getRemoteSyncStatus(): Promise; + onRemoteSyncStatusChange( + callback: (status: import('../../backend/features/remote-sync/helpers').RemoteSyncStatus) => void, + ): () => void; getVirtualDriveStatus(): Promise; onVirtualDriveStatusChange( callback: (event: { status: import('../drive/fuse/FuseDriveStatus').FuseDriveStatus }) => void, diff --git a/src/apps/main/main.ts b/src/apps/main/main.ts index d443d3e035..2ddd9ca6c2 100644 --- a/src/apps/main/main.ts +++ b/src/apps/main/main.ts @@ -27,7 +27,7 @@ import './analytics/handlers'; import './platform/handlers'; import './config/handlers'; import './app-info/handlers'; -import './remote-sync/handlers'; +import '../../backend/features/remote-sync/handlers'; import './../../backend/features/cleaner/ipc/handlers'; import { app } from 'electron'; diff --git a/src/apps/main/preload.d.ts b/src/apps/main/preload.d.ts index aca484dcf5..1ceb636c31 100644 --- a/src/apps/main/preload.d.ts +++ b/src/apps/main/preload.d.ts @@ -149,8 +149,10 @@ declare interface Window { getUsage: () => Promise; - onRemoteSyncStatusChange(callback: (status: import('./remote-sync/helpers').RemoteSyncStatus) => void): () => void; - getRemoteSyncStatus(): Promise; + onRemoteSyncStatusChange( + callback: (status: import('../../backend/features/remote-sync/helpers').RemoteSyncStatus) => void, + ): () => void; + getRemoteSyncStatus(): Promise; getVirtualDriveStatus(): Promise; onVirtualDriveStatusChange( callback: (event: { status: import('../drive/fuse/FuseDriveStatus').FuseDriveStatus }) => void, diff --git a/src/apps/main/remote-sync/RemoteSyncErrorHandler/RemoteSyncErrorHandler.test.ts b/src/apps/main/remote-sync/RemoteSyncErrorHandler/RemoteSyncErrorHandler.test.ts deleted file mode 100644 index bd03d9551a..0000000000 --- a/src/apps/main/remote-sync/RemoteSyncErrorHandler/RemoteSyncErrorHandler.test.ts +++ /dev/null @@ -1,94 +0,0 @@ -vi.mock('@internxt/drive-desktop-core/build/backend'); - -vi.mock('../../issues/virtual-drive', () => ({ - addVirtualDriveIssue: vi.fn(), -})); - -import { RemoteSyncErrorHandler, syncItemType } from './RemoteSyncErrorHandler'; -import { RemoteSyncError, RemoteSyncNetworkError, RemoteSyncServerError } from '../errors'; -import { addVirtualDriveIssue } from '../../issues/virtual-drive'; -import { VirtualDriveIssue } from '../../../../shared/issues/VirtualDriveIssue'; - -describe('RemoteSyncErrorHandler', () => { - let sut: RemoteSyncErrorHandler; - - beforeEach(() => { - sut = new RemoteSyncErrorHandler(); - vi.clearAllMocks(); - }); - - describe('handleSyncError ', () => { - it('should handle properly a type RemoteSyncNetworkError', () => { - const networkError = new RemoteSyncNetworkError('Test error'); - const syncType: syncItemType = 'files'; - const itemName = 'Test File'; - const checkpoint = new Date('2025-02-24'); - const handleNetworkErrorSpy = vi.spyOn(sut, 'handleNetworkError'); - - sut.handleSyncError(networkError, syncType, itemName, checkpoint); - - expect(handleNetworkErrorSpy).toHaveBeenCalledWith(networkError, syncType, itemName); - expect(addVirtualDriveIssue).toHaveBeenCalled(); - }); - - it('should handle properly a type RemoteSyncServerError', () => { - const serverError = new RemoteSyncServerError(500, { - message: 'Server error occurred', - }); - const syncType: syncItemType = 'folders'; - const itemName = 'Test Folder'; - const checkpoint = new Date('2025-02-24'); - const handleServerErrorSpy = vi.spyOn(sut, 'handleServerError'); - - sut.handleSyncError(serverError, syncType, itemName, checkpoint); - - expect(handleServerErrorSpy).toHaveBeenCalledWith(serverError, syncType, itemName); - expect(addVirtualDriveIssue).toHaveBeenCalled(); - }); - - it('should handle properly the default type RemoteSyncError', () => { - const genericError = new RemoteSyncError('Test generic error'); - const syncType: syncItemType = 'files'; - const itemName = 'Test File'; - const checkpoint = new Date('2025-02-24'); - const handleRemoteSyncErrorSpy = vi.spyOn(sut, 'handleRemoteSyncError'); - - sut.handleSyncError(genericError, syncType, itemName, checkpoint); - - expect(handleRemoteSyncErrorSpy).toHaveBeenCalledWith(genericError, syncType, itemName); - expect(addVirtualDriveIssue).toHaveBeenCalled(); - }); - }); - - describe('handleSyncErrorWithIssue', () => { - it('should properly add a virtual drive issue', () => { - const genericError = new RemoteSyncError('Test generic error'); - const syncType: syncItemType = 'files'; - const errorDetail = { - errorLabel: 'Test error label', - issue: { - error: 'UPLOAD_ERROR', - cause: 'NO_INTERNET', - name: 'Test File', - } as VirtualDriveIssue, - }; - - const issues: Record = { - files: { - error: 'UPLOAD_ERROR', - cause: 'NO_INTERNET', - name: 'Test File', - }, - folders: { - error: 'UPLOAD_ERROR', - cause: 'NO_INTERNET', - name: 'Test Folder', - }, - }; - - sut.handleSyncErrorWithIssue(genericError, syncType, errorDetail); - - expect(addVirtualDriveIssue).toHaveBeenCalledWith(issues[syncType]); - }); - }); -}); diff --git a/src/apps/main/remote-sync/RemoteSyncErrorHandler/RemoteSyncErrorHandler.ts b/src/apps/main/remote-sync/RemoteSyncErrorHandler/RemoteSyncErrorHandler.ts deleted file mode 100644 index e32c170bf6..0000000000 --- a/src/apps/main/remote-sync/RemoteSyncErrorHandler/RemoteSyncErrorHandler.ts +++ /dev/null @@ -1,113 +0,0 @@ -import { - RemoteSyncError, - RemoteSyncInvalidResponseError, - RemoteSyncNetworkError, - RemoteSyncServerError, -} from '../errors'; -import { logger } from '@internxt/drive-desktop-core/build/backend'; -import { addVirtualDriveIssue } from '../../issues/virtual-drive'; -import { VirtualDriveIssue } from '../../../../shared/issues/VirtualDriveIssue'; - -export type syncItemType = 'files' | 'folders'; - -export interface VirtualDriveIssueByType { - files: VirtualDriveIssue; - folders: VirtualDriveIssue; -} - -export class RemoteSyncErrorHandler { - public handleSyncError( - error: RemoteSyncError, - syncItemType: syncItemType, - itemName: string, - itemCheckpoint?: Date, - ): void { - switch (true) { - case error instanceof RemoteSyncNetworkError: - this.handleNetworkError(error, syncItemType, itemName); - break; - case error instanceof RemoteSyncServerError: - this.handleServerError(error, syncItemType, itemName); - break; - case error instanceof RemoteSyncInvalidResponseError: - // no-op - break; - default: - this.handleRemoteSyncError(error, syncItemType, itemName); - } - } - - handleSyncErrorWithIssue( - error: RemoteSyncError, - syncItemType: syncItemType, - errorDetail: { - errorLabel: string; - issue: VirtualDriveIssue; - }, - ): void { - logger.error({ - tag: 'SYNC-ENGINE', - msg: `[SYNC MANAGER] Remote ${syncItemType} sync failed with ${errorDetail.errorLabel} error: `, - error, - }); - addVirtualDriveIssue(errorDetail.issue); - } - - handleNetworkError(error: RemoteSyncNetworkError, syncItemType: syncItemType, itemName: string): void { - const issues: VirtualDriveIssueByType = { - files: { - error: 'DOWNLOAD_ERROR', - cause: 'NO_INTERNET', - name: itemName, - }, - folders: { - error: 'FOLDER_CREATE_ERROR', - cause: 'NO_INTERNET', - name: itemName, - }, - }; - this.handleSyncErrorWithIssue(error, syncItemType, { - errorLabel: 'network', - issue: issues[syncItemType], - }); - } - - handleServerError(error: RemoteSyncServerError, syncItemType: syncItemType, itemName: string): void { - const issues: VirtualDriveIssueByType = { - files: { - error: 'DOWNLOAD_ERROR', - cause: 'NO_REMOTE_CONNECTION', - name: itemName, - }, - folders: { - error: 'FOLDER_CREATE_ERROR', - cause: 'NO_REMOTE_CONNECTION', - name: itemName, - }, - }; - this.handleSyncErrorWithIssue(error, syncItemType, { - errorLabel: 'server', - issue: issues[syncItemType], - }); - } - - handleRemoteSyncError(error: RemoteSyncError, syncItemType: syncItemType, itemName: string): void { - const issues: VirtualDriveIssueByType = { - files: { - error: 'DOWNLOAD_ERROR', - cause: 'NO_REMOTE_CONNECTION', - name: itemName, - }, - folders: { - error: 'FOLDER_CREATE_ERROR', - cause: 'NO_REMOTE_CONNECTION', - name: itemName, - }, - }; - - this.handleSyncErrorWithIssue(error, syncItemType, { - errorLabel: 'remote', - issue: issues[syncItemType], - }); - } -} diff --git a/src/apps/main/remote-sync/RemoteSyncManager.test.ts b/src/apps/main/remote-sync/RemoteSyncManager.test.ts deleted file mode 100644 index 88e081daaf..0000000000 --- a/src/apps/main/remote-sync/RemoteSyncManager.test.ts +++ /dev/null @@ -1,313 +0,0 @@ -vi.mock('@internxt/drive-desktop-core/build/backend'); -vi.mock('../../../infra/drive-server/client/drive-server.client.instance', () => ({ - driveServerClient: { - GET: vi.fn(), - }, -})); -vi.mock('./RemoteSyncErrorHandler/RemoteSyncErrorHandler', () => ({ - RemoteSyncErrorHandler: vi.fn().mockImplementation(() => ({ - handleSyncError: vi.fn(), - })), -})); -vi.mock('../../../infra/sqlite/services/file/create-or-update-file-by-batch', () => ({ - createOrUpdateFileByBatch: vi.fn().mockResolvedValue({ data: [] }), -})); -vi.mock('../../../infra/sqlite/services/folder/create-or-update-folder-by-batch', () => ({ - createOrUpdateFolderByBatch: vi.fn().mockResolvedValue({ data: [] }), -})); -import { RemoteSyncErrorHandler } from './RemoteSyncErrorHandler/RemoteSyncErrorHandler'; -import { RemoteSyncManager } from './RemoteSyncManager'; -import { RemoteSyncedFile, RemoteSyncedFolder } from './helpers'; -import * as uuid from 'uuid'; -import { DriveServerError } from '../../../infra/drive-server/drive-server.error'; -import { driveServerClient } from '../../../infra/drive-server/client/drive-server.client.instance'; -import { DatabaseCollectionAdapter } from '../database/adapters/base'; -import { DriveFile } from '../database/entities/DriveFile'; -import { DriveFolder } from '../database/entities/DriveFolder'; -import { createOrUpdateFileByBatch } from '../../../infra/sqlite/services/file/create-or-update-file-by-batch'; -import { createOrUpdateFolderByBatch } from '../../../infra/sqlite/services/folder/create-or-update-folder-by-batch'; - -const mockedGet = vi.mocked(driveServerClient.GET); -const mockedCreateOrUpdateFileByBatch = vi.mocked(createOrUpdateFileByBatch); -const mockedCreateOrUpdateFolderByBatch = vi.mocked(createOrUpdateFolderByBatch); - -const inMemorySyncedFilesCollection: DatabaseCollectionAdapter = { - get: vi.fn(), - connect: vi.fn(), - update: vi.fn(), - create: vi.fn(), - remove: vi.fn(), - getLastUpdated: vi.fn(), -}; - -const inMemorySyncedFoldersCollection: DatabaseCollectionAdapter = { - get: vi.fn(), - connect: vi.fn(), - update: vi.fn(), - create: vi.fn(), - remove: vi.fn(), - getLastUpdated: vi.fn(), -}; - -const createRemoteSyncedFileFixture = (payload: Partial): RemoteSyncedFile => { - const result: RemoteSyncedFile = { - status: 'EXISTS', - name: `name_${uuid.v4()}`, - plainName: `plainname_${Date.now()}`, - id: Date.now(), - uuid: uuid.v4(), - fileId: Date.now().toString(), - type: 'jpg', - size: 999, - bucket: `bucket_${Date.now()}`, - folderId: 555, - folderUuid: uuid.v4(), - userId: 567, - modificationTime: new Date().toISOString(), - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - ...payload, - }; - - return result; -}; - -const createRemoteSyncedFolderFixture = (payload: Partial): RemoteSyncedFolder => { - const result: RemoteSyncedFolder = { - name: `name_${uuid.v4()}`, - plainName: `folder_${Date.now()}`, - id: Date.now(), - type: 'folder', - bucket: `bucket_${Date.now()}`, - userId: 567, - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - parentId: null, - uuid: uuid.v4(), - status: 'EXISTS', - ...payload, - }; - - return result; -}; - -describe('RemoteSyncManager', () => { - let errorHandler: RemoteSyncErrorHandler; - let sut: RemoteSyncManager; - inMemorySyncedFilesCollection.getLastUpdated = () => Promise.resolve({ success: false, result: null }); - inMemorySyncedFoldersCollection.getLastUpdated = () => Promise.resolve({ success: false, result: null }); - - beforeEach(() => { - errorHandler = new RemoteSyncErrorHandler(); - - sut = new RemoteSyncManager( - { - folders: inMemorySyncedFoldersCollection, - files: inMemorySyncedFilesCollection, - }, - { - fetchFilesLimitPerRequest: 2, - fetchFoldersLimitPerRequest: 2, - syncFiles: true, - syncFolders: true, - }, - errorHandler, - ); - mockedGet.mockClear(); - mockedCreateOrUpdateFileByBatch.mockClear(); - mockedCreateOrUpdateFolderByBatch.mockClear(); - }); - - describe('When there are files in remote, should sync them with local', () => { - it('Should sync all the files', async () => { - const sut = new RemoteSyncManager( - { - folders: inMemorySyncedFoldersCollection, - files: inMemorySyncedFilesCollection, - }, - { - fetchFilesLimitPerRequest: 2, - fetchFoldersLimitPerRequest: 2, - syncFiles: true, - syncFolders: false, - }, - errorHandler, - ); - - mockedGet - .mockResolvedValueOnce({ - data: [ - createRemoteSyncedFileFixture({ - plainName: 'file_1', - }), - createRemoteSyncedFileFixture({ - plainName: 'file_2', - }), - ], - }) - .mockResolvedValueOnce({ - data: [ - createRemoteSyncedFileFixture({ - plainName: 'file_3', - }), - ], - }); - - await sut.startRemoteSync(); - - expect(mockedGet).toBeCalledTimes(2); - expect(sut.getSyncStatus()).toBe('SYNCED'); - }); - it('Should sync all the folders', async () => { - const sut = new RemoteSyncManager( - { - folders: inMemorySyncedFoldersCollection, - files: inMemorySyncedFilesCollection, - }, - { - fetchFilesLimitPerRequest: 2, - fetchFoldersLimitPerRequest: 2, - syncFiles: false, - syncFolders: true, - }, - errorHandler, - ); - - mockedGet - .mockResolvedValueOnce({ - data: [ - createRemoteSyncedFolderFixture({ - plainName: 'folder_1', - }), - createRemoteSyncedFolderFixture({ - plainName: 'folder_2', - }), - ], - }) - .mockResolvedValueOnce({ - data: [ - createRemoteSyncedFolderFixture({ - plainName: 'folder_3', - }), - createRemoteSyncedFolderFixture({ - plainName: 'folder_4', - }), - ], - }) - .mockResolvedValueOnce({ - data: [ - createRemoteSyncedFolderFixture({ - plainName: 'folder_5', - }), - ], - }); - - await sut.startRemoteSync(); - - expect(mockedGet).toBeCalledTimes(3); - expect(sut.getSyncStatus()).toBe('SYNCED'); - }); - - it('Should save the files in the database', async () => { - const sut = new RemoteSyncManager( - { - folders: inMemorySyncedFoldersCollection, - files: inMemorySyncedFilesCollection, - }, - { - fetchFilesLimitPerRequest: 2, - fetchFoldersLimitPerRequest: 2, - syncFiles: true, - syncFolders: false, - }, - errorHandler, - ); - const file1 = createRemoteSyncedFileFixture({ - plainName: 'file_1', - }); - - const file2 = createRemoteSyncedFileFixture({ - plainName: 'file_2', - }); - - mockedGet.mockResolvedValueOnce({ data: [file1, file2] }); - - mockedGet.mockResolvedValueOnce({ data: [] }); - - await sut.startRemoteSync(); - - expect(mockedGet).toBeCalledTimes(2); - expect(sut.getSyncStatus()).toBe('SYNCED'); - expect(mockedCreateOrUpdateFileByBatch).toBeCalledWith({ files: [file1, file2] }); - }); - }); - - describe('When something fails during the sync', () => { - it('Should retry N times and then stop if sync does not succeed', async () => { - mockedGet.mockResolvedValue({ error: new DriveServerError('UNKNOWN', undefined, 'Fail on purpose') }); - - await sut.startRemoteSync(); - - expect(mockedGet).toBeCalledTimes(6); - expect(sut.getSyncStatus()).toBe('SYNC_FAILED'); - }); - - it('Should fail the sync if some files or folders cannot be retrieved', async () => { - mockedGet.mockResolvedValueOnce({ error: new DriveServerError('UNKNOWN', undefined, 'Fail on purpose') }); - - await sut.startRemoteSync(); - - expect(mockedGet).toBeCalledTimes(6); - expect(sut.getSyncStatus()).toBe('SYNC_FAILED'); - }); - - it('should handle the error while syncing files by calling the error handler properly', async () => { - const sut = new RemoteSyncManager( - { - folders: inMemorySyncedFoldersCollection, - files: inMemorySyncedFilesCollection, - }, - { - fetchFilesLimitPerRequest: 2, - fetchFoldersLimitPerRequest: 2, - syncFiles: true, - syncFolders: false, - }, - errorHandler, - ); - mockedGet.mockResolvedValueOnce({ error: new DriveServerError('UNKNOWN', undefined, 'Fail on purpose') }); - const errorHandlerInstance = sut['errorHandler']; - const errorHandlerSpy = vi.spyOn(errorHandlerInstance, 'handleSyncError'); - - await sut.startRemoteSync(); - - expect(errorHandlerSpy).toHaveBeenCalled(); - expect(errorHandlerSpy.mock.calls[0][1]).toBe('files'); - }); - - it('should handle the error while syncing folders by calling the error handler properly', async () => { - const sut = new RemoteSyncManager( - { - folders: inMemorySyncedFoldersCollection, - files: inMemorySyncedFilesCollection, - }, - { - fetchFilesLimitPerRequest: 2, - fetchFoldersLimitPerRequest: 2, - syncFiles: false, - syncFolders: true, - }, - errorHandler, - ); - - mockedGet.mockResolvedValueOnce({ error: new DriveServerError('UNKNOWN', undefined, 'Fail on purpose') }); - const errorHandlerInstance = sut['errorHandler']; - const errorHandlerSpy = vi.spyOn(errorHandlerInstance, 'handleSyncError'); - - await sut.startRemoteSync(); - - expect(errorHandlerSpy).toHaveBeenCalled(); - expect(errorHandlerSpy.mock.calls[0][1]).toBe('folders'); - }); - }); -}); diff --git a/src/apps/main/remote-sync/RemoteSyncManager.ts b/src/apps/main/remote-sync/RemoteSyncManager.ts deleted file mode 100644 index a07a57aa1d..0000000000 --- a/src/apps/main/remote-sync/RemoteSyncManager.ts +++ /dev/null @@ -1,401 +0,0 @@ -import { logger } from '@internxt/drive-desktop-core/build/backend'; -import { - RemoteSyncStatus, - RemoteSyncedFolder, - RemoteSyncedFile, - SyncConfig, - rewind, - SIX_HOURS_IN_MILLISECONDS, -} from './helpers'; -import { DatabaseCollectionAdapter } from '../database/adapters/base'; -import { DriveFolder } from '../database/entities/DriveFolder'; -import { DriveFile } from '../database/entities/DriveFile'; -import { Nullable } from '../../shared/types/Nullable'; -import { RemoteSyncError, RemoteSyncNetworkError } from './errors'; -import { RemoteSyncErrorHandler } from './RemoteSyncErrorHandler/RemoteSyncErrorHandler'; -import { createOrUpdateFolderByBatch } from '../../../infra/sqlite/services/folder/create-or-update-folder-by-batch'; -import { createOrUpdateFileByBatch } from '../../../infra/sqlite/services/file/create-or-update-file-by-batch'; -import { fetchFiles } from '../../../infra/drive-server/services/files/services/fetch-files'; -import { fetchFolders } from '../../../infra/drive-server/services/folder/services/fetch-folders'; - -export class RemoteSyncManager { - private foldersSyncStatus: RemoteSyncStatus = 'IDLE'; - private filesSyncStatus: RemoteSyncStatus = 'IDLE'; - private status: RemoteSyncStatus = 'IDLE'; - private onStatusChangeCallbacks: Array<(newStatus: RemoteSyncStatus) => void> = []; - private totalFilesSynced = 0; - private totalFoldersSynced = 0; - - constructor( - private db: { - files: DatabaseCollectionAdapter; - folders: DatabaseCollectionAdapter; - }, - private config: { - fetchFilesLimitPerRequest: number; - fetchFoldersLimitPerRequest: number; - syncFiles: boolean; - syncFolders: boolean; - }, - private errorHandler: RemoteSyncErrorHandler, - ) {} - - getTotalFilesSynced() { - return this.totalFilesSynced; - } - - onStatusChange(callback: (newStatus: RemoteSyncStatus) => void) { - if (typeof callback !== 'function') return; - this.onStatusChangeCallbacks.push(callback); - } - - getSyncStatus(): RemoteSyncStatus { - return this.status; - } - - /** - * Check if the RemoteSyncManager is in SYNCED status - * - * @returns True if local database is synced with remote files and folders - */ - localIsSynced() { - return this.status === 'SYNCED'; - } - - resetRemoteSync() { - this.changeStatus('IDLE'); - this.filesSyncStatus = 'IDLE'; - this.foldersSyncStatus = 'IDLE'; - this.totalFilesSynced = 0; - this.totalFoldersSynced = 0; - } - - /** - * Triggers a remote sync so we can populate the localDB, this sync - * is global and starts pulling all the files the user has in remote. - * - * Throws an error if there's a sync in progress for this class instance - */ - async startRemoteSync() { - const testPassed = this.smokeTest(); - - if (!testPassed) { - return; - } - this.totalFilesSynced = 0; - this.totalFoldersSynced = 0; - this.filesSyncStatus = 'IDLE'; - this.foldersSyncStatus = 'IDLE'; - - await this.db.files.connect(); - await this.db.folders.connect(); - - logger.debug({ tag: 'SYNC-ENGINE', msg: 'Starting' }); - this.changeStatus('SYNCING'); - try { - await Promise.all([ - this.config.syncFiles - ? this.syncRemoteFiles({ - retry: 1, - maxRetries: 3, - }) - : Promise.resolve(), - this.config.syncFolders - ? this.syncRemoteFolders({ - retry: 1, - maxRetries: 3, - }) - : Promise.resolve(), - ]); - } catch (error) { - this.changeStatus('SYNC_FAILED'); - logger.error({ - tag: 'SYNC-ENGINE', - msg: 'Remote sync failed with uncontrolled error: ', - error, - }); - } finally { - logger.debug({ - tag: 'SYNC-ENGINE', - msg: `Total synced files: ${this.totalFilesSynced}`, - }); - logger.debug({ - tag: 'SYNC-ENGINE', - msg: `Total synced folders: ${this.totalFoldersSynced}`, - }); - } - } - - /** - * Run smoke tests before starting the RemoteSyncManager, otherwise fail - */ - private smokeTest() { - if (this.status === 'SYNCING') { - logger.warn({ - tag: 'SYNC-ENGINE', - msg: 'RemoteSyncManager should not be in SYNCING status to start, not starting again', - }); - - return false; - } - - return true; - } - - private changeStatus(newStatus: RemoteSyncStatus) { - if (newStatus === this.status) return; - logger.debug({ - tag: 'SYNC-ENGINE', - msg: `RemoteSyncManager ${this.status} -> ${newStatus}`, - }); - this.status = newStatus; - this.onStatusChangeCallbacks.forEach((callback) => { - if (typeof callback !== 'function') return; - callback(newStatus); - }); - } - - private checkRemoteSyncStatus() { - // We only syncing files - if (this.config.syncFiles && !this.config.syncFolders && this.filesSyncStatus === 'SYNCED') { - this.changeStatus('SYNCED'); - return; - } - - // We only syncing folders - if (!this.config.syncFiles && this.config.syncFolders && this.foldersSyncStatus === 'SYNCED') { - this.changeStatus('SYNCED'); - return; - } - // Files and folders are synced, RemoteSync is Synced - if (this.foldersSyncStatus === 'SYNCED' && this.filesSyncStatus === 'SYNCED') { - this.changeStatus('SYNCED'); - return; - } - - // Files OR Folders sync failed, RemoteSync Failed - if (this.foldersSyncStatus === 'SYNC_FAILED' || this.filesSyncStatus === 'SYNC_FAILED') { - this.changeStatus('SYNC_FAILED'); - return; - } - } - - private async getFileCheckpoint(): Promise> { - const { success, result } = await this.db.files.getLastUpdated(); - - if (!success) return undefined; - - if (!result) return undefined; - - const updatedAt = new Date(result.updatedAt); - - return rewind(updatedAt, SIX_HOURS_IN_MILLISECONDS); - } - - /** - * Syncs all the remote files and saves them into the local db - * @param syncConfig Config to execute the sync with - * @returns - */ - private async syncRemoteFiles(syncConfig: SyncConfig, from?: Date) { - let fileCheckPoint = from ?? (await this.getFileCheckpoint()); - let hasMore = true; - let retryCount = 0; - - while (hasMore && retryCount < syncConfig.maxRetries) { - let lastFileSynced = null; - - try { - const { hasMore: moreAvailable, result } = await this.fetchFilesFromRemote(fileCheckPoint); - - await createOrUpdateFileByBatch({ files: result }); - this.totalFilesSynced += result.length; - lastFileSynced = result.length > 0 ? result[result.length - 1] : null; - - hasMore = moreAvailable; - - if (hasMore && lastFileSynced) { - fileCheckPoint = new Date(lastFileSynced.updatedAt); - } - - retryCount = 0; - } catch (error) { - retryCount++; - - if (error instanceof RemoteSyncError) { - this.errorHandler.handleSyncError(error, 'files', lastFileSynced?.name ?? 'unknown', fileCheckPoint); - } else { - logger.error({ - tag: 'SYNC-ENGINE', - msg: 'Remote files sync failed with uncontrolled error: ', - error, - }); - } - - if (retryCount >= syncConfig.maxRetries) { - this.filesSyncStatus = 'SYNC_FAILED'; - this.checkRemoteSyncStatus(); - return; - } - - await new Promise((resolve) => setTimeout(resolve, 1000 * retryCount)); - } - } - - logger.debug({ - tag: 'SYNC-ENGINE', - msg: 'Remote files sync finished', - }); - this.filesSyncStatus = 'SYNCED'; - this.checkRemoteSyncStatus(); - } - - private async getLastFolderSyncAt(): Promise> { - const { success, result } = await this.db.folders.getLastUpdated(); - - if (!success) return undefined; - - if (!result) return undefined; - - const updatedAt = new Date(result.updatedAt); - return rewind(updatedAt, SIX_HOURS_IN_MILLISECONDS); - } - - /** - * Syncs all the remote folders and saves them into the local db - * @param syncConfig Config to execute the sync with - * @returns - */ - private async syncRemoteFolders(syncConfig: SyncConfig, from?: Date) { - let folderCheckPoint = from ?? (await this.getLastFolderSyncAt()); - let hasMore = true; - let retryCount = 0; - - while (hasMore && retryCount < syncConfig.maxRetries) { - let lastFolderSynced = null; - - try { - const { hasMore: moreAvailable, result } = await this.fetchFoldersFromRemote(folderCheckPoint); - - await createOrUpdateFolderByBatch({ folders: result }); - this.totalFoldersSynced += result.length; - lastFolderSynced = result.length > 0 ? result[result.length - 1] : null; - - hasMore = moreAvailable; - - if (hasMore && lastFolderSynced) { - folderCheckPoint = new Date(lastFolderSynced.updatedAt); - } - - // Reset retry count on successful fetch - retryCount = 0; - } catch (error) { - retryCount++; - - if (error instanceof RemoteSyncError) { - this.errorHandler.handleSyncError(error, 'folders', lastFolderSynced?.name ?? 'unknown', folderCheckPoint); - } else { - logger.error({ - tag: 'SYNC-ENGINE', - msg: 'Remote folders sync failed with uncontrolled error: ', - error, - }); - } - - if (retryCount >= syncConfig.maxRetries) { - this.foldersSyncStatus = 'SYNC_FAILED'; - this.checkRemoteSyncStatus(); - return; - } - - // Brief delay before retry to avoid hammering the server - await new Promise((resolve) => setTimeout(resolve, 1000 * retryCount)); - } - } - - logger.debug({ - tag: 'SYNC-ENGINE', - msg: 'Remote folders sync finished', - }); - this.foldersSyncStatus = 'SYNCED'; - this.checkRemoteSyncStatus(); - } - - /** - * Fetch the files that were updated after the given date - * - * @param updatedAtCheckpoint Retrieve files that were updated after this date - */ - private async fetchFilesFromRemote(updatedAtCheckpoint?: Date): Promise<{ - hasMore: boolean; - result: RemoteSyncedFile[]; - }> { - const { data, error } = await fetchFiles({ - limit: this.config.fetchFilesLimitPerRequest, - offset: 0, - status: 'ALL', - updatedAt: updatedAtCheckpoint?.toISOString(), - }); - - if (error) { - throw new RemoteSyncNetworkError(error.message, undefined, error.statusCode); - } - - return { - hasMore: data.hasMore, - result: data.files.map(this.patchDriveFileResponseItem), - }; - } - - /** - * Fetch the folders that were updated after the given date - * - * @param updatedAtCheckpoint Retrieve folders that were updated after this date - */ - private async fetchFoldersFromRemote(updatedAtCheckpoint?: Date): Promise<{ - hasMore: boolean; - result: RemoteSyncedFolder[]; - }> { - const { data, error } = await fetchFolders({ - limit: this.config.fetchFilesLimitPerRequest, - offset: 0, - status: 'ALL', - updatedAt: updatedAtCheckpoint?.toISOString(), - }); - - if (error) { - throw new RemoteSyncNetworkError(error.message, undefined, error.statusCode); - } - - return { - hasMore: data.hasMore, - result: data.folders.map(this.patchDriveFolderResponseItem), - }; - } - - private patchDriveFolderResponseItem = (payload: Record): RemoteSyncedFolder => { - const status = this.resolveFolderStatus(payload); - - return { - ...(payload as Omit), - status, - name: typeof payload.name === 'string' ? payload.name : undefined, - }; - }; - - private resolveFolderStatus(payload: Record): RemoteSyncedFolder['status'] { - if (typeof payload.status === 'string' && payload.status) return payload.status; - if (payload.removed) return 'REMOVED'; - if (payload.deleted) return 'DELETED'; - return 'EXISTS'; - } - - private readonly patchDriveFileResponseItem = (payload: Record): RemoteSyncedFile => { - return { - ...(payload as Omit), - fileId: typeof payload.fileId === 'string' ? payload.fileId : '', - size: typeof payload.size === 'string' ? Number.parseInt(payload.size) : (payload.size as number), - name: typeof payload.name === 'string' ? payload.name : undefined, - }; - }; -} diff --git a/src/apps/renderer/context/SyncContext.test.tsx b/src/apps/renderer/context/SyncContext.test.tsx index 89ef64887d..92a2dc2c66 100644 --- a/src/apps/renderer/context/SyncContext.test.tsx +++ b/src/apps/renderer/context/SyncContext.test.tsx @@ -1,6 +1,6 @@ import { renderHook, act } from '@testing-library/react-hooks'; import { SyncProvider, useSyncContext } from './SyncContext'; -import { RemoteSyncStatus } from '../../main/remote-sync/helpers'; +import { RemoteSyncStatus } from '../../../backend/features/remote-sync/helpers'; import { partialSpyOn } from '../../../../tests/vitest/utils.helper'; describe('SyncContext', () => { diff --git a/src/apps/renderer/context/SyncContext.tsx b/src/apps/renderer/context/SyncContext.tsx index 2d6ffa6bee..9451bce318 100644 --- a/src/apps/renderer/context/SyncContext.tsx +++ b/src/apps/renderer/context/SyncContext.tsx @@ -1,6 +1,6 @@ import { createContext, ReactNode, useCallback, useContext, useEffect, useMemo, useState } from 'react'; import { SyncStatus } from '../../../context/desktop/sync/domain/SyncStatus'; -import { RemoteSyncStatus } from '../../main/remote-sync/helpers'; +import { RemoteSyncStatus } from '../../../backend/features/remote-sync/helpers'; const statusesMap: Record = { SYNCING: 'RUNNING', diff --git a/src/apps/renderer/hooks/useOnSyncRunning.test.tsx b/src/apps/renderer/hooks/useOnSyncRunning.test.tsx index 0fae939d0b..23ae7ff675 100644 --- a/src/apps/renderer/hooks/useOnSyncRunning.test.tsx +++ b/src/apps/renderer/hooks/useOnSyncRunning.test.tsx @@ -1,7 +1,7 @@ import { renderHook, act } from '@testing-library/react-hooks'; import { useOnSyncRunning } from './useOnSyncRunning'; import { SyncProvider } from '../context/SyncContext'; -import { RemoteSyncStatus } from '../../main/remote-sync/helpers'; +import { RemoteSyncStatus } from '../../../backend/features/remote-sync/helpers'; import { partialSpyOn } from '../../../../tests/vitest/utils.helper'; describe('useOnSyncRunning', () => { diff --git a/src/apps/main/remote-sync/InitialSyncReady.ts b/src/backend/features/remote-sync/InitialSyncReady.ts similarity index 100% rename from src/apps/main/remote-sync/InitialSyncReady.ts rename to src/backend/features/remote-sync/InitialSyncReady.ts diff --git a/src/apps/main/remote-sync/errors.ts b/src/backend/features/remote-sync/errors.ts similarity index 100% rename from src/apps/main/remote-sync/errors.ts rename to src/backend/features/remote-sync/errors.ts diff --git a/src/backend/features/remote-sync/fetch-remote-files.ts b/src/backend/features/remote-sync/fetch-remote-files.ts new file mode 100644 index 0000000000..b02381ab2b --- /dev/null +++ b/src/backend/features/remote-sync/fetch-remote-files.ts @@ -0,0 +1,35 @@ +import { RemoteSyncedFile } from './helpers'; +import { patchDriveFileResponseItem } from './patch-drive-file-response-item'; +import { fetchFiles } from '../../../infra/drive-server/services/files/services/fetch-files'; +import { Result } from '../../../context/shared/domain/Result'; + +type Props = { + limit: number; + updatedAtCheckpoint?: Date; +}; + +type FetchFilesResponse = { + hasMore: boolean; + files: RemoteSyncedFile[]; +}; + +export async function fetchRemoteFiles({ + limit, + updatedAtCheckpoint, +}: Props): Promise> { + const { data, error } = await fetchFiles({ + limit, + offset: 0, + status: 'ALL', + updatedAt: updatedAtCheckpoint?.toISOString(), + }); + + if (error) return { error }; + + return { + data: { + hasMore: data.hasMore, + files: data.files.map(patchDriveFileResponseItem), + }, + }; +} diff --git a/src/backend/features/remote-sync/fetch-remote-folders.ts b/src/backend/features/remote-sync/fetch-remote-folders.ts new file mode 100644 index 0000000000..0ea5aa40d0 --- /dev/null +++ b/src/backend/features/remote-sync/fetch-remote-folders.ts @@ -0,0 +1,36 @@ +import { RemoteSyncNetworkError } from './errors'; +import { RemoteSyncedFolder } from './helpers'; +import { patchDriveFolderResponseItem } from './patch-drive-folder-response-item'; +import { fetchFolders } from '../../../infra/drive-server/services/folder/services/fetch-folders'; +import { Result } from '../../../context/shared/domain/Result'; + +type Props = { + limit: number; + updatedAtCheckpoint?: Date; +}; + +type FetchFoldersResponse = { + hasMore: boolean; + folders: RemoteSyncedFolder[]; +}; + +export async function fetchRemoteFolders({ + limit, + updatedAtCheckpoint, +}: Props): Promise> { + const { data, error } = await fetchFolders({ + limit, + offset: 0, + status: 'ALL', + updatedAt: updatedAtCheckpoint?.toISOString(), + }); + + if (error) return { error }; + + return { + data: { + hasMore: data.hasMore, + folders: data.folders.map(patchDriveFolderResponseItem), + }, + }; +} diff --git a/src/backend/features/remote-sync/get-last-updated-checkpoint.ts b/src/backend/features/remote-sync/get-last-updated-checkpoint.ts new file mode 100644 index 0000000000..68725e45a1 --- /dev/null +++ b/src/backend/features/remote-sync/get-last-updated-checkpoint.ts @@ -0,0 +1,27 @@ +import { rewind } from './helpers'; +import { DatabaseCollectionAdapter } from '../../../apps/main/database/adapters/base'; +import { Nullable } from '../../../apps/shared/types/Nullable'; + +type DatabaseItemWithUpdatedAt = { + updatedAt: string; +}; + +type Props = { + collection: DatabaseCollectionAdapter; + rewindMilliseconds: number; +}; + +export async function getLastUpdatedCheckpoint({ + collection, + rewindMilliseconds, +}: Props): Promise> { + const { success, result } = await collection.getLastUpdated(); + + if (!success || !result) { + return undefined; + } + + const updatedAt = new Date(result.updatedAt); + + return rewind(updatedAt, rewindMilliseconds); +} diff --git a/src/backend/features/remote-sync/get-remote-sync-error-detail.ts b/src/backend/features/remote-sync/get-remote-sync-error-detail.ts new file mode 100644 index 0000000000..b72045865a --- /dev/null +++ b/src/backend/features/remote-sync/get-remote-sync-error-detail.ts @@ -0,0 +1,81 @@ +import { + RemoteSyncError, + RemoteSyncInvalidResponseError, + RemoteSyncNetworkError, + RemoteSyncServerError, +} from './errors'; +import { VirtualDriveIssue } from '../../../shared/issues/VirtualDriveIssue'; + +export type RemoteSyncItemType = 'files' | 'folders'; + +export type RemoteSyncErrorDetail = { + errorLabel: 'network' | 'server' | 'remote'; + issue: VirtualDriveIssue; +}; + +type GetRemoteSyncErrorDetailPops = { + error: RemoteSyncError; + syncItemType: RemoteSyncItemType; + itemName: string; +}; + +function createVirtualDriveIssue({ + syncItemType, + itemName, + cause, +}: { + syncItemType: RemoteSyncItemType; + itemName: string; + cause: VirtualDriveIssue['cause']; +}): VirtualDriveIssue { + if (syncItemType === 'files') { + return { + error: 'DOWNLOAD_ERROR', + cause, + name: itemName, + }; + } + + return { + error: 'FOLDER_CREATE_ERROR', + cause, + name: itemName, + }; +} + +export function getRemoteSyncErrorDetail({ error, syncItemType, itemName }: GetRemoteSyncErrorDetailPops) { + if (error instanceof RemoteSyncInvalidResponseError) { + return null; + } + + if (error instanceof RemoteSyncNetworkError) { + return { + errorLabel: 'network', + issue: createVirtualDriveIssue({ + syncItemType, + itemName, + cause: 'NO_INTERNET', + }), + } satisfies RemoteSyncErrorDetail; + } + + if (error instanceof RemoteSyncServerError) { + return { + errorLabel: 'server', + issue: createVirtualDriveIssue({ + syncItemType, + itemName, + cause: 'NO_REMOTE_CONNECTION', + }), + } satisfies RemoteSyncErrorDetail; + } + + return { + errorLabel: 'remote', + issue: createVirtualDriveIssue({ + syncItemType, + itemName, + cause: 'NO_REMOTE_CONNECTION', + }), + } satisfies RemoteSyncErrorDetail; +} diff --git a/src/backend/features/remote-sync/handlers.test.ts b/src/backend/features/remote-sync/handlers.test.ts new file mode 100644 index 0000000000..29e0779af1 --- /dev/null +++ b/src/backend/features/remote-sync/handlers.test.ts @@ -0,0 +1,150 @@ +import { ipcMain } from 'electron'; +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { vi } from 'vitest'; +import { getIpcHandler } from '../../../../tests/vitest/ipc.helper'; +import { call, calls, partialSpyOn } from '../../../../tests/vitest/utils.helper'; + +const mocks = vi.hoisted(() => { + return { + remoteSyncController: { + getSyncStatus: vi.fn(() => 'IDLE'), + resetRemoteSync: vi.fn(), + startRemoteSync: vi.fn(), + }, + resyncRemoteSync: vi.fn(), + startRemoteSync: vi.fn(), + }; +}); + +vi.mock('./service', () => ({ + remoteSyncController: mocks.remoteSyncController, + resyncRemoteSync: mocks.resyncRemoteSync, + startRemoteSync: mocks.startRemoteSync, +})); + +async function loadHandlersModule() { + vi.resetModules(); + + const eventBusModule = await import('../../../apps/main/event-bus'); + const initialSyncReadyModule = await import('./InitialSyncReady'); + + const eventBusOnMock = partialSpyOn(eventBusModule.default, 'on', false); + const eventBusEmitMock = partialSpyOn(eventBusModule.default, 'emit'); + const setInitialSyncStateMock = partialSpyOn(initialSyncReadyModule, 'setInitialSyncState'); + + await import('./handlers'); + + return { + eventBusEmitMock, + eventBusOnMock, + setInitialSyncStateMock, + }; +} + +function getEventBusHandler(eventBusOnMock: ReturnType, eventName: string) { + const calls = eventBusOnMock.mock.calls as Array<[string, (...args: unknown[]) => unknown]>; + + return calls.find(([name]) => name === eventName)?.[1] as ((...args: unknown[]) => unknown) | undefined; +} + +describe('handlers.test', () => { + beforeEach(() => { + mocks.remoteSyncController.getSyncStatus.mockReturnValue('IDLE'); + mocks.startRemoteSync.mockResolvedValue(undefined); + mocks.remoteSyncController.startRemoteSync.mockResolvedValue(undefined); + mocks.resyncRemoteSync.mockResolvedValue(undefined); + }); + + it('should register ipc handlers and event listeners', async () => { + // When + const { eventBusOnMock } = await loadHandlersModule(); + + // Then + expect(ipcMain.handle).toHaveBeenCalledWith('START_REMOTE_SYNC', expect.any(Function)); + expect(ipcMain.handle).toHaveBeenCalledWith('get-remote-sync-status', expect.any(Function)); + expect(eventBusOnMock).toHaveBeenCalledWith('RECEIVED_REMOTE_CHANGES', expect.any(Function)); + expect(eventBusOnMock).toHaveBeenCalledWith('APP_DATA_SOURCE_INITIALIZED', expect.any(Function)); + expect(eventBusOnMock).toHaveBeenCalledWith('USER_LOGGED_OUT', expect.any(Function)); + }); + + it('should start a sync from the ipc handler and emit remote changes synched', async () => { + // Given + const { eventBusEmitMock } = await loadHandlersModule(); + const handler = getIpcHandler('START_REMOTE_SYNC'); + + // When + await handler?.(); + + // Then + calls(mocks.startRemoteSync).toHaveLength(1); + call(eventBusEmitMock).toStrictEqual('REMOTE_CHANGES_SYNCHED'); + }); + + it('should return the current sync status from the ipc handler', async () => { + // Given + mocks.remoteSyncController.getSyncStatus.mockReturnValue('SYNCED'); + await loadHandlersModule(); + const handler = getIpcHandler('get-remote-sync-status'); + + // When + const result = await handler?.(); + + // Then + expect(result).toBe('SYNCED'); + }); + + it('should resync when remote changes are received', async () => { + // Given + const { eventBusOnMock } = await loadHandlersModule(); + const listener = getEventBusHandler(eventBusOnMock, 'RECEIVED_REMOTE_CHANGES'); + + // When + await listener?.(); + + // Then + calls(mocks.resyncRemoteSync).toHaveLength(1); + }); + + it('should start the sync when the data source is initialized', async () => { + // Given + const { eventBusOnMock } = await loadHandlersModule(); + const listener = getEventBusHandler(eventBusOnMock, 'APP_DATA_SOURCE_INITIALIZED'); + + // When + await listener?.(); + + // Then + calls(mocks.startRemoteSync).toHaveLength(1); + }); + + it('should log an error when sync startup fails during app data source initialization', async () => { + // Given + const error = new Error('sync failed'); + mocks.startRemoteSync.mockRejectedValue(error); + const { eventBusOnMock } = await loadHandlersModule(); + const listener = getEventBusHandler(eventBusOnMock, 'APP_DATA_SOURCE_INITIALIZED'); + + // When + await listener?.(); + + // Then + expect(logger.error).toHaveBeenCalledWith({ + tag: 'SYNC-ENGINE', + msg: 'Error starting remote sync controller', + error, + }); + }); + + it('should reset initial sync state and remote sync state on logout', async () => { + // Given + const { eventBusOnMock, setInitialSyncStateMock } = await loadHandlersModule(); + const listener = getEventBusHandler(eventBusOnMock, 'USER_LOGGED_OUT'); + + // When + listener?.(); + + // Then + call(setInitialSyncStateMock).toBe('NOT_READY'); + calls(mocks.remoteSyncController.resetRemoteSync).toHaveLength(1); + }); +}); diff --git a/src/apps/main/remote-sync/handlers.ts b/src/backend/features/remote-sync/handlers.ts similarity index 67% rename from src/apps/main/remote-sync/handlers.ts rename to src/backend/features/remote-sync/handlers.ts index 11136f773f..3c67ad37c9 100644 --- a/src/apps/main/remote-sync/handlers.ts +++ b/src/backend/features/remote-sync/handlers.ts @@ -1,15 +1,15 @@ import { ipcMain } from 'electron'; import { logger } from '@internxt/drive-desktop-core/build/backend'; -import eventBus from '../event-bus'; +import eventBus from '../../../apps/main/event-bus'; import { setInitialSyncState } from './InitialSyncReady'; -import { remoteSyncManager, resyncRemoteSync, startRemoteSync } from './service'; +import { remoteSyncController, resyncRemoteSync, startRemoteSync } from './service'; ipcMain.handle('START_REMOTE_SYNC', async () => { await startRemoteSync(); eventBus.emit('REMOTE_CHANGES_SYNCHED'); }); -ipcMain.handle('get-remote-sync-status', () => remoteSyncManager.getSyncStatus()); +ipcMain.handle('get-remote-sync-status', () => remoteSyncController.getSyncStatus()); eventBus.on('RECEIVED_REMOTE_CHANGES', async () => { // Wait before checking for updates, could be possible @@ -19,10 +19,10 @@ eventBus.on('RECEIVED_REMOTE_CHANGES', async () => { }); eventBus.on('APP_DATA_SOURCE_INITIALIZED', async () => { - await remoteSyncManager.startRemoteSync().catch((error) => { + await startRemoteSync().catch((error) => { logger.error({ tag: 'SYNC-ENGINE', - msg: 'Error starting remote sync manager', + msg: 'Error starting remote sync controller', error, }); }); @@ -30,5 +30,5 @@ eventBus.on('APP_DATA_SOURCE_INITIALIZED', async () => { eventBus.on('USER_LOGGED_OUT', () => { setInitialSyncState('NOT_READY'); - remoteSyncManager.resetRemoteSync(); + remoteSyncController.resetRemoteSync(); }); diff --git a/src/apps/main/remote-sync/helpers.ts b/src/backend/features/remote-sync/helpers.ts similarity index 100% rename from src/apps/main/remote-sync/helpers.ts rename to src/backend/features/remote-sync/helpers.ts diff --git a/src/backend/features/remote-sync/patch-drive-file-response-item.ts b/src/backend/features/remote-sync/patch-drive-file-response-item.ts new file mode 100644 index 0000000000..efe4fb0e94 --- /dev/null +++ b/src/backend/features/remote-sync/patch-drive-file-response-item.ts @@ -0,0 +1,15 @@ +import { RemoteSyncedFile } from './helpers'; + +type DriveFileResponseItem = Partial & { + fileId?: string | null; + size?: number | string; +}; + +export function patchDriveFileResponseItem(payload: DriveFileResponseItem) { + return { + ...payload, + fileId: payload.fileId ?? '', + size: typeof payload.size === 'string' ? Number.parseInt(payload.size) : payload.size, + name: payload.name ?? undefined, + }; +} diff --git a/src/backend/features/remote-sync/patch-drive-folder-response-item.ts b/src/backend/features/remote-sync/patch-drive-folder-response-item.ts new file mode 100644 index 0000000000..686c46f56a --- /dev/null +++ b/src/backend/features/remote-sync/patch-drive-folder-response-item.ts @@ -0,0 +1,28 @@ +import { RemoteSyncedFolder } from './helpers'; + +type DriveFolderResponseItem = Partial & { + removed?: boolean; + deleted?: boolean; +}; + +export function patchDriveFolderResponseItem(payload: DriveFolderResponseItem) { + let status: RemoteSyncedFolder['status'] = payload.status ?? ''; + + if (!status && !payload.removed) { + status = 'EXISTS'; + } + + if (!status && payload.removed) { + status = 'REMOVED'; + } + + if (!status && payload.deleted) { + status = 'DELETED'; + } + + return { + ...payload, + status, + name: payload.name ?? undefined, + }; +} diff --git a/src/backend/features/remote-sync/remote-sync-controller/change-status.test.ts b/src/backend/features/remote-sync/remote-sync-controller/change-status.test.ts new file mode 100644 index 0000000000..11f7f0dfda --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/change-status.test.ts @@ -0,0 +1,33 @@ +vi.mock('@internxt/drive-desktop-core/build/backend'); + +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { createControllerState } from './create-controller-state'; +import { changeStatus } from './change-status'; + +describe('change-status.test', () => { + it('should update the status and notify callbacks', () => { + // Given + const state = createControllerState(); + const callback = vi.fn(); + state.onStatusChangeCallbacks.push(callback); + + // When + changeStatus({ state, newStatus: 'SYNCING' }); + + // Then + expect(state.status).toBe('SYNCING'); + expect(callback).toHaveBeenCalledWith('SYNCING'); + expect(logger.debug).toHaveBeenCalledTimes(1); + }); + + it('should do nothing when the status is unchanged', () => { + // Given + const state = createControllerState(); + + // When + changeStatus({ state, newStatus: 'IDLE' }); + + // Then + expect(logger.debug).not.toHaveBeenCalled(); + }); +}); diff --git a/src/backend/features/remote-sync/remote-sync-controller/change-status.ts b/src/backend/features/remote-sync/remote-sync-controller/change-status.ts new file mode 100644 index 0000000000..08209ad5d2 --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/change-status.ts @@ -0,0 +1,28 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { RemoteSyncStatus } from '../helpers'; +import { RemoteSyncControllerState } from './types'; + +type ChangeStatusPops = { + state: RemoteSyncControllerState; + newStatus: RemoteSyncStatus; +}; + +export function changeStatus({ state, newStatus }: ChangeStatusPops) { + if (newStatus === state.status) { + return; + } + + logger.debug({ + tag: 'SYNC-ENGINE', + msg: `Remote sync controller ${state.status} -> ${newStatus}`, + }); + + state.status = newStatus; + state.onStatusChangeCallbacks.forEach((callback) => { + if (typeof callback !== 'function') { + return; + } + + callback(newStatus); + }); +} diff --git a/src/backend/features/remote-sync/remote-sync-controller/check-remote-sync-status.test.ts b/src/backend/features/remote-sync/remote-sync-controller/check-remote-sync-status.test.ts new file mode 100644 index 0000000000..60b81ed9a5 --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/check-remote-sync-status.test.ts @@ -0,0 +1,47 @@ +vi.mock('@internxt/drive-desktop-core/build/backend'); + +import { createControllerState } from './create-controller-state'; +import { checkRemoteSyncStatus } from './check-remote-sync-status'; + +describe('check-remote-sync-status.test', () => { + it('should move the controller to synced when file and folder sync are synced', () => { + // Given + const state = createControllerState(); + state.filesSyncStatus = 'SYNCED'; + state.foldersSyncStatus = 'SYNCED'; + + // When + checkRemoteSyncStatus({ + state, + config: { + fetchFilesLimitPerRequest: 1, + fetchFoldersLimitPerRequest: 1, + syncFiles: true, + syncFolders: true, + }, + }); + + // Then + expect(state.status).toBe('SYNCED'); + }); + + it('should keep the current state when there is no aggregated status yet', () => { + // Given + const state = createControllerState(); + state.status = 'SYNCING'; + + // When + checkRemoteSyncStatus({ + state, + config: { + fetchFilesLimitPerRequest: 1, + fetchFoldersLimitPerRequest: 1, + syncFiles: true, + syncFolders: true, + }, + }); + + // Then + expect(state.status).toBe('SYNCING'); + }); +}); diff --git a/src/backend/features/remote-sync/remote-sync-controller/check-remote-sync-status.ts b/src/backend/features/remote-sync/remote-sync-controller/check-remote-sync-status.ts new file mode 100644 index 0000000000..5c5adc631b --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/check-remote-sync-status.ts @@ -0,0 +1,26 @@ +import { resolveRemoteSyncStatus } from '../resolve-remote-sync-status'; +import { changeStatus } from './change-status'; +import { RemoteSyncControllerConfig, RemoteSyncControllerState } from './types'; + +type CheckRemoteSyncStatusPops = { + state: RemoteSyncControllerState; + config: RemoteSyncControllerConfig; +}; + +export function checkRemoteSyncStatus({ state, config }: CheckRemoteSyncStatusPops) { + const nextStatus = resolveRemoteSyncStatus({ + filesSyncStatus: state.filesSyncStatus, + foldersSyncStatus: state.foldersSyncStatus, + syncFiles: config.syncFiles, + syncFolders: config.syncFolders, + }); + + if (!nextStatus) { + return; + } + + changeStatus({ + state, + newStatus: nextStatus, + }); +} diff --git a/src/backend/features/remote-sync/remote-sync-controller/controller-methods.test.ts b/src/backend/features/remote-sync/remote-sync-controller/controller-methods.test.ts new file mode 100644 index 0000000000..8b89f727a1 --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/controller-methods.test.ts @@ -0,0 +1,38 @@ +import { createControllerState } from './create-controller-state'; +import { getSyncStatus, getTotalFilesSynced, onStatusChange } from './controller-methods'; + +describe('controller-methods.test', () => { + it('should return the status and total files from state', () => { + // Given + const state = createControllerState(); + state.status = 'SYNCED'; + state.totalFilesSynced = 12; + + // Then + expect(getSyncStatus({ state })).toBe('SYNCED'); + expect(getTotalFilesSynced({ state })).toBe(12); + }); + + it('should register status change callbacks', () => { + // Given + const state = createControllerState(); + const callback = vi.fn(); + + // When + onStatusChange({ state, callback }); + + // Then + expect(state.onStatusChangeCallbacks).toStrictEqual([callback]); + }); + + it('should ignore invalid callbacks', () => { + // Given + const state = createControllerState(); + + // When + onStatusChange({ state, callback: undefined as never }); + + // Then + expect(state.onStatusChangeCallbacks).toHaveLength(0); + }); +}); diff --git a/src/backend/features/remote-sync/remote-sync-controller/controller-methods.ts b/src/backend/features/remote-sync/remote-sync-controller/controller-methods.ts new file mode 100644 index 0000000000..1d041684de --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/controller-methods.ts @@ -0,0 +1,26 @@ +import { RemoteSyncStatus } from '../helpers'; +import { RemoteSyncControllerState } from './types'; + +type ControllerStatePops = { + state: RemoteSyncControllerState; +}; + +type OnStatusChangePops = ControllerStatePops & { + callback: (newStatus: RemoteSyncStatus) => void; +}; + +export function getTotalFilesSynced({ state }: ControllerStatePops) { + return state.totalFilesSynced; +} + +export function onStatusChange({ state, callback }: OnStatusChangePops) { + if (typeof callback !== 'function') { + return; + } + + state.onStatusChangeCallbacks.push(callback); +} + +export function getSyncStatus({ state }: ControllerStatePops) { + return state.status; +} diff --git a/src/backend/features/remote-sync/remote-sync-controller/create-controller-state.test.ts b/src/backend/features/remote-sync/remote-sync-controller/create-controller-state.test.ts new file mode 100644 index 0000000000..1a2acea40b --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/create-controller-state.test.ts @@ -0,0 +1,18 @@ +import { createControllerState } from './create-controller-state'; + +describe('create-controller-state.test', () => { + it('should create the initial controller state', () => { + // When + const result = createControllerState(); + + // Then + expect(result).toStrictEqual({ + foldersSyncStatus: 'IDLE', + filesSyncStatus: 'IDLE', + status: 'IDLE', + totalFilesSynced: 0, + totalFoldersSynced: 0, + onStatusChangeCallbacks: [], + }); + }); +}); diff --git a/src/backend/features/remote-sync/remote-sync-controller/create-controller-state.ts b/src/backend/features/remote-sync/remote-sync-controller/create-controller-state.ts new file mode 100644 index 0000000000..29705fb353 --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/create-controller-state.ts @@ -0,0 +1,12 @@ +import { RemoteSyncControllerState } from './types'; + +export function createControllerState(): RemoteSyncControllerState { + return { + foldersSyncStatus: 'IDLE', + filesSyncStatus: 'IDLE', + status: 'IDLE', + totalFilesSynced: 0, + totalFoldersSynced: 0, + onStatusChangeCallbacks: [], + }; +} diff --git a/src/backend/features/remote-sync/remote-sync-controller/index.test.ts b/src/backend/features/remote-sync/remote-sync-controller/index.test.ts new file mode 100644 index 0000000000..ba538c3d5b --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/index.test.ts @@ -0,0 +1,313 @@ +vi.mock('@internxt/drive-desktop-core/build/backend'); +vi.mock('../../../../infra/drive-server/client/drive-server.client.instance', () => ({ + driveServerClient: { + GET: vi.fn(), + }, +})); +vi.mock('../../../../infra/sqlite/services/file/create-or-update-file-by-batch', () => ({ + createOrUpdateFileByBatch: vi.fn().mockResolvedValue({ data: [] }), +})); +vi.mock('../../../../infra/sqlite/services/folder/create-or-update-folder-by-batch', () => ({ + createOrUpdateFolderByBatch: vi.fn().mockResolvedValue({ data: [] }), +})); + +import * as uuid from 'uuid'; +import { createRemoteSyncController, CreateRemoteSyncControllerPops, RemoteSyncController } from './index'; +import { RemoteSyncErrorHandler } from '../remote-sync-error-handler'; +import { RemoteSyncedFile, RemoteSyncedFolder } from '../helpers'; +import { DriveServerError } from '../../../../infra/drive-server/drive-server.error'; +import { RemoteSyncServerError } from '../errors'; +import { driveServerClient } from '../../../../infra/drive-server/client/drive-server.client.instance'; +import { DatabaseCollectionAdapter } from '../../../../apps/main/database/adapters/base'; +import { DriveFile } from '../../../../apps/main/database/entities/DriveFile'; +import { DriveFolder } from '../../../../apps/main/database/entities/DriveFolder'; +import { createOrUpdateFileByBatch } from '../../../../infra/sqlite/services/file/create-or-update-file-by-batch'; +import { createOrUpdateFolderByBatch } from '../../../../infra/sqlite/services/folder/create-or-update-folder-by-batch'; + +const mockedGet = vi.mocked( + driveServerClient.GET as (...args: unknown[]) => Promise<{ data?: unknown; error?: unknown }>, +); +const mockedCreateOrUpdateFileByBatch = vi.mocked(createOrUpdateFileByBatch); +const mockedCreateOrUpdateFolderByBatch = vi.mocked(createOrUpdateFolderByBatch); + +const inMemorySyncedFilesCollection: DatabaseCollectionAdapter = { + get: vi.fn(), + connect: vi.fn(), + update: vi.fn(), + create: vi.fn(), + remove: vi.fn(), + getLastUpdated: vi.fn(), +}; + +const inMemorySyncedFoldersCollection: DatabaseCollectionAdapter = { + get: vi.fn(), + connect: vi.fn(), + update: vi.fn(), + create: vi.fn(), + remove: vi.fn(), + getLastUpdated: vi.fn(), +}; + +function createRemoteSyncedFileFixture(payload: Partial): RemoteSyncedFile { + return { + status: 'EXISTS', + name: `name_${uuid.v4()}`, + plainName: `plainname_${Date.now()}`, + id: Date.now(), + uuid: uuid.v4(), + fileId: Date.now().toString(), + type: 'jpg', + size: 999, + bucket: `bucket_${Date.now()}`, + folderId: 555, + folderUuid: uuid.v4(), + userId: 567, + modificationTime: new Date().toISOString(), + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + ...payload, + }; +} + +function createRemoteSyncedFolderFixture(payload: Partial): RemoteSyncedFolder { + return { + name: `name_${uuid.v4()}`, + plainName: `folder_${Date.now()}`, + id: Date.now(), + type: 'folder', + bucket: `bucket_${Date.now()}`, + userId: 567, + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + parentId: null, + uuid: uuid.v4(), + status: 'EXISTS', + ...payload, + }; +} + +describe('index.test', () => { + let errorHandler: RemoteSyncErrorHandler; + let sut: RemoteSyncController; + let props: CreateRemoteSyncControllerPops; + + inMemorySyncedFilesCollection.getLastUpdated = () => Promise.resolve({ success: false, result: null }); + inMemorySyncedFoldersCollection.getLastUpdated = () => Promise.resolve({ success: false, result: null }); + + beforeEach(() => { + errorHandler = { + handleSyncError: vi.fn(), + }; + + props = { + db: { + folders: inMemorySyncedFoldersCollection, + files: inMemorySyncedFilesCollection, + }, + config: { + fetchFilesLimitPerRequest: 2, + fetchFoldersLimitPerRequest: 2, + syncFiles: true, + syncFolders: true, + }, + errorHandler, + }; + + sut = createRemoteSyncController(); + mockedGet.mockReset(); + mockedCreateOrUpdateFileByBatch.mockClear(); + mockedCreateOrUpdateFolderByBatch.mockClear(); + }); + + describe('When there are files in remote, should sync them with local', () => { + it('Should sync all the files', async () => { + // Given + const sut = createRemoteSyncController(); + const syncProps = { + ...props, + config: { + ...props.config, + syncFolders: false, + }, + }; + + mockedGet + .mockResolvedValueOnce({ + data: [ + createRemoteSyncedFileFixture({ plainName: 'file_1' }), + createRemoteSyncedFileFixture({ plainName: 'file_2' }), + ], + }) + .mockResolvedValueOnce({ + data: [createRemoteSyncedFileFixture({ plainName: 'file_3' })], + }); + + // When + await sut.startRemoteSync(syncProps); + + // Then + expect(mockedGet).toBeCalledTimes(2); + expect(sut.getSyncStatus()).toBe('SYNCED'); + }); + + it('Should sync all the folders', async () => { + // Given + const sut = createRemoteSyncController(); + const syncProps = { + ...props, + config: { + ...props.config, + syncFiles: false, + }, + }; + + mockedGet + .mockResolvedValueOnce({ + data: [ + createRemoteSyncedFolderFixture({ plainName: 'folder_1' }), + createRemoteSyncedFolderFixture({ plainName: 'folder_2' }), + ], + }) + .mockResolvedValueOnce({ + data: [ + createRemoteSyncedFolderFixture({ plainName: 'folder_3' }), + createRemoteSyncedFolderFixture({ plainName: 'folder_4' }), + ], + }) + .mockResolvedValueOnce({ + data: [createRemoteSyncedFolderFixture({ plainName: 'folder_5' })], + }); + + // When + await sut.startRemoteSync(syncProps); + + // Then + expect(mockedGet).toBeCalledTimes(3); + expect(sut.getSyncStatus()).toBe('SYNCED'); + }); + + it('Should save the files in the database', async () => { + // Given + const sut = createRemoteSyncController(); + const syncProps = { + ...props, + config: { + ...props.config, + syncFolders: false, + }, + }; + const file1 = createRemoteSyncedFileFixture({ plainName: 'file_1' }); + const file2 = createRemoteSyncedFileFixture({ plainName: 'file_2' }); + + mockedGet.mockResolvedValueOnce({ data: [file1, file2] }); + mockedGet.mockResolvedValueOnce({ data: [] }); + + // When + await sut.startRemoteSync(syncProps); + + // Then + expect(mockedGet).toBeCalledTimes(2); + expect(sut.getSyncStatus()).toBe('SYNCED'); + expect(mockedCreateOrUpdateFileByBatch).toBeCalledWith({ files: [file1, file2] }); + }); + + it('Should use the folders limit when syncing folders', async () => { + // Given + const sut = createRemoteSyncController(); + const syncProps = { + ...props, + config: { + ...props.config, + fetchFoldersLimitPerRequest: 7, + syncFiles: false, + }, + }; + + mockedGet.mockResolvedValueOnce({ data: [] }); + + // When + await sut.startRemoteSync(syncProps); + + // Then + expect(mockedGet).toBeCalledWith('/folders', { + query: { + limit: 7, + offset: 0, + status: 'ALL', + updatedAt: undefined, + }, + }); + }); + }); + + describe('When something fails during the sync', () => { + it('Should retry N times and then stop if sync does not succeed', async () => { + // Given + mockedGet.mockResolvedValue({ error: new DriveServerError('UNKNOWN', undefined, 'Fail on purpose') }); + + // When + await sut.startRemoteSync(props); + + // Then + expect(mockedGet).toBeCalledTimes(6); + expect(sut.getSyncStatus()).toBe('SYNC_FAILED'); + }); + + it('Should fail the sync if some files or folders cannot be retrieved', async () => { + // Given + mockedGet.mockResolvedValueOnce({ error: new RemoteSyncServerError(500, { message: 'Fail on purpose' }) }); + + // When + await sut.startRemoteSync(props); + + // Then + expect(mockedGet).toBeCalledTimes(6); + expect(sut.getSyncStatus()).toBe('SYNC_FAILED'); + }); + + it('should handle the error while syncing files by calling the error handler properly', async () => { + // Given + const sut = createRemoteSyncController(); + const syncProps = { + ...props, + config: { + ...props.config, + syncFolders: false, + }, + }; + mockedGet.mockResolvedValueOnce({ error: new RemoteSyncServerError(500, { message: 'Fail on purpose' }) }); + + // When + await sut.startRemoteSync(syncProps); + + // Then + expect(errorHandler.handleSyncError).toHaveBeenCalled(); + expect(vi.mocked(errorHandler.handleSyncError).mock.calls[0][0]).toMatchObject({ + syncItemType: 'files', + }); + }); + + it('should handle the error while syncing folders by calling the error handler properly', async () => { + // Given + const sut = createRemoteSyncController(); + const syncProps = { + ...props, + config: { + ...props.config, + syncFiles: false, + }, + }; + + mockedGet.mockResolvedValueOnce({ error: new RemoteSyncServerError(500, { message: 'Fail on purpose' }) }); + + // When + await sut.startRemoteSync(syncProps); + + // Then + expect(errorHandler.handleSyncError).toHaveBeenCalled(); + expect(vi.mocked(errorHandler.handleSyncError).mock.calls[0][0]).toMatchObject({ + syncItemType: 'folders', + }); + }); + }); +}); diff --git a/src/backend/features/remote-sync/remote-sync-controller/index.ts b/src/backend/features/remote-sync/remote-sync-controller/index.ts new file mode 100644 index 0000000000..52f1fa6034 --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/index.ts @@ -0,0 +1,29 @@ +import { createControllerState } from './create-controller-state'; +import { getSyncStatus, getTotalFilesSynced, onStatusChange } from './controller-methods'; +import { resetRemoteSync as resetRemoteSyncState } from './reset-remote-sync'; +import { startRemoteSync as startRemoteSyncProcess } from './start-remote-sync'; +import { CreateRemoteSyncControllerPops, RemoteSyncController } from './types'; + +export type { + CreateRemoteSyncControllerPops, + RemoteSyncController, + RemoteSyncControllerConfig, + RemoteSyncControllerDb, + RemoteSyncControllerState, +} from './types'; + +export function createRemoteSyncController(): RemoteSyncController { + const state = createControllerState(); + + return { + getTotalFilesSynced: () => getTotalFilesSynced({ state }), + onStatusChange: (callback) => onStatusChange({ state, callback }), + getSyncStatus: () => getSyncStatus({ state }), + resetRemoteSync: () => resetRemoteSyncState({ state }), + startRemoteSync: (props: CreateRemoteSyncControllerPops) => + startRemoteSyncProcess({ + state, + ...props, + }), + }; +} diff --git a/src/backend/features/remote-sync/remote-sync-controller/reset-remote-sync.test.ts b/src/backend/features/remote-sync/remote-sync-controller/reset-remote-sync.test.ts new file mode 100644 index 0000000000..76032bb4b2 --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/reset-remote-sync.test.ts @@ -0,0 +1,28 @@ +vi.mock('@internxt/drive-desktop-core/build/backend'); + +import { createControllerState } from './create-controller-state'; +import { resetRemoteSync } from './reset-remote-sync'; + +describe('reset-remote-sync.test', () => { + it('should reset sync state and counters', () => { + // Given + const state = createControllerState(); + state.status = 'SYNC_FAILED'; + state.filesSyncStatus = 'SYNC_FAILED'; + state.foldersSyncStatus = 'SYNCED'; + state.totalFilesSynced = 4; + state.totalFoldersSynced = 2; + + // When + resetRemoteSync({ state }); + + // Then + expect(state).toMatchObject({ + status: 'IDLE', + filesSyncStatus: 'IDLE', + foldersSyncStatus: 'IDLE', + totalFilesSynced: 0, + totalFoldersSynced: 0, + }); + }); +}); diff --git a/src/backend/features/remote-sync/remote-sync-controller/reset-remote-sync.ts b/src/backend/features/remote-sync/remote-sync-controller/reset-remote-sync.ts new file mode 100644 index 0000000000..4c5d301819 --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/reset-remote-sync.ts @@ -0,0 +1,17 @@ +import { changeStatus } from './change-status'; +import { RemoteSyncControllerState } from './types'; + +type ResetRemoteSyncPops = { + state: RemoteSyncControllerState; +}; + +export function resetRemoteSync({ state }: ResetRemoteSyncPops) { + changeStatus({ + state, + newStatus: 'IDLE', + }); + state.filesSyncStatus = 'IDLE'; + state.foldersSyncStatus = 'IDLE'; + state.totalFilesSynced = 0; + state.totalFoldersSynced = 0; +} diff --git a/src/backend/features/remote-sync/remote-sync-controller/start-remote-sync.ts b/src/backend/features/remote-sync/remote-sync-controller/start-remote-sync.ts new file mode 100644 index 0000000000..a9d059d141 --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/start-remote-sync.ts @@ -0,0 +1,76 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { changeStatus } from './change-status'; +import { syncRemoteFiles } from './sync-remote-files'; +import { syncRemoteFolders } from './sync-remote-folders'; +import { CreateRemoteSyncControllerPops, RemoteSyncControllerState } from './types'; + +type StartRemoteSyncPops = CreateRemoteSyncControllerPops & { + state: RemoteSyncControllerState; +}; + +export async function startRemoteSync({ state, db, config, errorHandler }: StartRemoteSyncPops) { + if (state.status === 'SYNCING') { + logger.warn({ + tag: 'SYNC-ENGINE', + msg: 'Remote sync controller is already in SYNCING status, not starting again', + }); + + return; + } + + state.totalFilesSynced = 0; + state.totalFoldersSynced = 0; + state.filesSyncStatus = 'IDLE'; + state.foldersSyncStatus = 'IDLE'; + + await db.files.connect(); + await db.folders.connect(); + + logger.debug({ tag: 'SYNC-ENGINE', msg: 'Starting' }); + changeStatus({ state, newStatus: 'SYNCING' }); + + try { + await Promise.all([ + config.syncFiles + ? syncRemoteFiles({ + state, + db, + config, + errorHandler, + syncConfig: { + retry: 1, + maxRetries: 3, + }, + }) + : Promise.resolve(), + config.syncFolders + ? syncRemoteFolders({ + state, + db, + config, + errorHandler, + syncConfig: { + retry: 1, + maxRetries: 3, + }, + }) + : Promise.resolve(), + ]); + } catch (error) { + changeStatus({ state, newStatus: 'SYNC_FAILED' }); + logger.error({ + tag: 'SYNC-ENGINE', + msg: 'Remote sync failed with uncontrolled error: ', + error, + }); + } finally { + logger.debug({ + tag: 'SYNC-ENGINE', + msg: `Total synced files: ${state.totalFilesSynced}`, + }); + logger.debug({ + tag: 'SYNC-ENGINE', + msg: `Total synced folders: ${state.totalFoldersSynced}`, + }); + } +} diff --git a/src/backend/features/remote-sync/remote-sync-controller/sync-remote-files.ts b/src/backend/features/remote-sync/remote-sync-controller/sync-remote-files.ts new file mode 100644 index 0000000000..852efaa43d --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/sync-remote-files.ts @@ -0,0 +1,50 @@ +import { SIX_HOURS_IN_MILLISECONDS } from '../helpers'; +import { createOrUpdateFileByBatch } from '../../../../infra/sqlite/services/file/create-or-update-file-by-batch'; +import { fetchRemoteFiles } from '../fetch-remote-files'; +import { getLastUpdatedCheckpoint } from '../get-last-updated-checkpoint'; +import { syncRemoteItems } from '../sync-remote-items'; +import { checkRemoteSyncStatus } from './check-remote-sync-status'; +import { SyncRemoteItemsPops } from './types'; + +export async function syncRemoteFiles({ state, db, config, errorHandler, syncConfig, from }: SyncRemoteItemsPops) { + await syncRemoteItems({ + from, + finishMessage: 'Remote files sync finished', + syncConfig, + syncItemType: 'files', + getCheckpoint: () => + getLastUpdatedCheckpoint({ + collection: db.files, + rewindMilliseconds: SIX_HOURS_IN_MILLISECONDS, + }), + fetchRemoteItems: async (updatedAtCheckpoint) => { + const { error, data } = await fetchRemoteFiles({ + limit: config.fetchFilesLimitPerRequest, + updatedAtCheckpoint, + }); + if (error) return { error }; + return { data: { hasMore: data.hasMore, result: data.files } }; + }, + persistRemoteItems: (items) => createOrUpdateFileByBatch({ files: items }), + onSyncFailed: () => { + state.filesSyncStatus = 'SYNC_FAILED'; + }, + onSyncFinished: () => { + state.filesSyncStatus = 'SYNCED'; + }, + onSyncProgress: (items) => { + state.totalFilesSynced += items.length; + }, + onSyncStateChanged: () => { + checkRemoteSyncStatus({ state, config }); + }, + handleSyncError: (error, itemName, checkpoint) => { + errorHandler.handleSyncError({ + error, + syncItemType: 'files', + itemName, + itemCheckpoint: checkpoint, + }); + }, + }); +} diff --git a/src/backend/features/remote-sync/remote-sync-controller/sync-remote-folders.ts b/src/backend/features/remote-sync/remote-sync-controller/sync-remote-folders.ts new file mode 100644 index 0000000000..db31dfef59 --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/sync-remote-folders.ts @@ -0,0 +1,50 @@ +import { SIX_HOURS_IN_MILLISECONDS } from '../helpers'; +import { createOrUpdateFolderByBatch } from '../../../../infra/sqlite/services/folder/create-or-update-folder-by-batch'; +import { fetchRemoteFolders } from '../fetch-remote-folders'; +import { getLastUpdatedCheckpoint } from '../get-last-updated-checkpoint'; +import { syncRemoteItems } from '../sync-remote-items'; +import { checkRemoteSyncStatus } from './check-remote-sync-status'; +import { SyncRemoteItemsPops } from './types'; + +export async function syncRemoteFolders({ state, db, config, errorHandler, syncConfig, from }: SyncRemoteItemsPops) { + await syncRemoteItems({ + from, + finishMessage: 'Remote folders sync finished', + syncConfig, + syncItemType: 'folders', + getCheckpoint: () => + getLastUpdatedCheckpoint({ + collection: db.folders, + rewindMilliseconds: SIX_HOURS_IN_MILLISECONDS, + }), + fetchRemoteItems: async (updatedAtCheckpoint) => { + const { error, data } = await fetchRemoteFolders({ + limit: config.fetchFoldersLimitPerRequest, + updatedAtCheckpoint, + }); + if (error) return { error }; + return { data: { hasMore: data.hasMore, result: data.folders } }; + }, + persistRemoteItems: (items) => createOrUpdateFolderByBatch({ folders: items }), + onSyncFailed: () => { + state.foldersSyncStatus = 'SYNC_FAILED'; + }, + onSyncFinished: () => { + state.foldersSyncStatus = 'SYNCED'; + }, + onSyncProgress: (items) => { + state.totalFoldersSynced += items.length; + }, + onSyncStateChanged: () => { + checkRemoteSyncStatus({ state, config }); + }, + handleSyncError: (error, itemName, checkpoint) => { + errorHandler.handleSyncError({ + error, + syncItemType: 'folders', + itemName, + itemCheckpoint: checkpoint, + }); + }, + }); +} diff --git a/src/backend/features/remote-sync/remote-sync-controller/types.ts b/src/backend/features/remote-sync/remote-sync-controller/types.ts new file mode 100644 index 0000000000..5ab968210e --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-controller/types.ts @@ -0,0 +1,49 @@ +import { SyncConfig, RemoteSyncStatus } from '../helpers'; +import { DatabaseCollectionAdapter } from '../../../../apps/main/database/adapters/base'; +import { DriveFile } from '../../../../apps/main/database/entities/DriveFile'; +import { DriveFolder } from '../../../../apps/main/database/entities/DriveFolder'; +import { RemoteSyncErrorHandler } from '../remote-sync-error-handler'; + +export type RemoteSyncControllerConfig = { + fetchFilesLimitPerRequest: number; + fetchFoldersLimitPerRequest: number; + syncFiles: boolean; + syncFolders: boolean; +}; + +export type RemoteSyncControllerDb = { + files: DatabaseCollectionAdapter; + folders: DatabaseCollectionAdapter; +}; + +export type CreateRemoteSyncControllerPops = { + db: RemoteSyncControllerDb; + config: RemoteSyncControllerConfig; + errorHandler: RemoteSyncErrorHandler; +}; + +export type RemoteSyncController = { + getTotalFilesSynced: () => number; + onStatusChange: (callback: (newStatus: RemoteSyncStatus) => void) => void; + getSyncStatus: () => RemoteSyncStatus; + resetRemoteSync: () => void; + startRemoteSync: (props: CreateRemoteSyncControllerPops) => Promise; +}; + +export type RemoteSyncControllerState = { + foldersSyncStatus: RemoteSyncStatus; + filesSyncStatus: RemoteSyncStatus; + status: RemoteSyncStatus; + totalFilesSynced: number; + totalFoldersSynced: number; + onStatusChangeCallbacks: Array<(newStatus: RemoteSyncStatus) => void>; +}; + +export type SyncRemoteItemsPops = { + state: RemoteSyncControllerState; + db: RemoteSyncControllerDb; + config: RemoteSyncControllerConfig; + errorHandler: RemoteSyncErrorHandler; + syncConfig: SyncConfig; + from?: Date; +}; diff --git a/src/backend/features/remote-sync/remote-sync-error-handler.test.ts b/src/backend/features/remote-sync/remote-sync-error-handler.test.ts new file mode 100644 index 0000000000..0c81bad5d8 --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-error-handler.test.ts @@ -0,0 +1,95 @@ +import * as virtualDriveIssuesModule from '../../../apps/main/issues/virtual-drive'; +import { call, calls, partialSpyOn } from '../../../../tests/vitest/utils.helper'; +import { createRemoteSyncErrorHandler } from './remote-sync-error-handler'; +import { + RemoteSyncError, + RemoteSyncInvalidResponseError, + RemoteSyncNetworkError, + RemoteSyncServerError, +} from './errors'; +import { loggerMock } from 'tests/vitest/mocks.helper'; + +describe('remote-sync-error-handler.test', () => { + const addVirtualDriveIssueMock = partialSpyOn(virtualDriveIssuesModule, 'addVirtualDriveIssue'); + + beforeEach(() => { + addVirtualDriveIssueMock.mockReset(); + }); + + it('should add a no-internet issue for file network errors', () => { + // Given + const sut = createRemoteSyncErrorHandler(); + + // When + sut.handleSyncError({ + error: new RemoteSyncNetworkError('connection lost'), + syncItemType: 'files', + itemName: 'Test File', + }); + + // Then + call(addVirtualDriveIssueMock).toStrictEqual({ + error: 'DOWNLOAD_ERROR', + cause: 'NO_INTERNET', + name: 'Test File', + }); + calls(loggerMock.error).toHaveLength(1); + }); + + it('should add a remote-connection issue for folder server errors', () => { + // Given + const sut = createRemoteSyncErrorHandler(); + + // When + sut.handleSyncError({ + error: new RemoteSyncServerError(500, { message: 'Server error occurred' }), + syncItemType: 'folders', + itemName: 'Test Folder', + itemCheckpoint: new Date('2025-02-24T00:00:00.000Z'), + }); + + // Then + call(addVirtualDriveIssueMock).toStrictEqual({ + error: 'FOLDER_CREATE_ERROR', + cause: 'NO_REMOTE_CONNECTION', + name: 'Test Folder', + }); + calls(loggerMock.error).toHaveLength(1); + }); + + it('should ignore invalid response errors', () => { + // Given + const sut = createRemoteSyncErrorHandler(); + + // When + sut.handleSyncError({ + error: new RemoteSyncInvalidResponseError({ invalid: true }), + syncItemType: 'files', + itemName: 'Test File', + }); + + // Then + calls(addVirtualDriveIssueMock).toHaveLength(0); + calls(loggerMock.error).toHaveLength(0); + }); + + it('should treat generic remote sync errors as remote connection issues', () => { + // Given + const sut = createRemoteSyncErrorHandler(); + + // When + sut.handleSyncError({ + error: new RemoteSyncError('generic failure'), + syncItemType: 'files', + itemName: 'Another File', + }); + + // Then + call(addVirtualDriveIssueMock).toStrictEqual({ + error: 'DOWNLOAD_ERROR', + cause: 'NO_REMOTE_CONNECTION', + name: 'Another File', + }); + calls(loggerMock.error).toHaveLength(1); + }); +}); diff --git a/src/backend/features/remote-sync/remote-sync-error-handler.ts b/src/backend/features/remote-sync/remote-sync-error-handler.ts new file mode 100644 index 0000000000..667e1c9f3e --- /dev/null +++ b/src/backend/features/remote-sync/remote-sync-error-handler.ts @@ -0,0 +1,44 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { addVirtualDriveIssue } from '../../../apps/main/issues/virtual-drive'; +import { RemoteSyncError } from './errors'; +import { getRemoteSyncErrorDetail, RemoteSyncItemType } from './get-remote-sync-error-detail'; + +export type HandleSyncErrorPops = { + error: RemoteSyncError; + syncItemType: RemoteSyncItemType; + itemName: string; + itemCheckpoint?: Date; +}; + +export type RemoteSyncErrorHandler = { + handleSyncError: (props: HandleSyncErrorPops) => void; +}; + +export function handleSyncError({ error, syncItemType, itemName, itemCheckpoint }: HandleSyncErrorPops) { + const errorDetail = getRemoteSyncErrorDetail({ + error, + syncItemType, + itemName, + }); + + if (!errorDetail) { + return; + } + + logger.error({ + tag: 'SYNC-ENGINE', + msg: `Remote ${syncItemType} sync failed`, + error, + errorLabel: errorDetail.errorLabel, + itemName, + itemCheckpoint, + }); + + addVirtualDriveIssue(errorDetail.issue); +} + +export function createRemoteSyncErrorHandler(): RemoteSyncErrorHandler { + return { + handleSyncError, + }; +} diff --git a/src/backend/features/remote-sync/resolve-remote-sync-status.ts b/src/backend/features/remote-sync/resolve-remote-sync-status.ts new file mode 100644 index 0000000000..0a3ecf5461 --- /dev/null +++ b/src/backend/features/remote-sync/resolve-remote-sync-status.ts @@ -0,0 +1,28 @@ +import { RemoteSyncStatus } from './helpers'; + +type Props = { + filesSyncStatus: RemoteSyncStatus; + foldersSyncStatus: RemoteSyncStatus; + syncFiles: boolean; + syncFolders: boolean; +}; + +export function resolveRemoteSyncStatus({ filesSyncStatus, foldersSyncStatus, syncFiles, syncFolders }: Props) { + if (syncFiles && !syncFolders && filesSyncStatus === 'SYNCED') { + return 'SYNCED' as const; + } + + if (!syncFiles && syncFolders && foldersSyncStatus === 'SYNCED') { + return 'SYNCED' as const; + } + + if (foldersSyncStatus === 'SYNCED' && filesSyncStatus === 'SYNCED') { + return 'SYNCED' as const; + } + + if (foldersSyncStatus === 'SYNC_FAILED' || filesSyncStatus === 'SYNC_FAILED') { + return 'SYNC_FAILED' as const; + } + + return undefined; +} diff --git a/src/backend/features/remote-sync/service.test.ts b/src/backend/features/remote-sync/service.test.ts new file mode 100644 index 0000000000..b2faa3404a --- /dev/null +++ b/src/backend/features/remote-sync/service.test.ts @@ -0,0 +1,217 @@ +import { vi } from 'vitest'; +import { call, calls, partialSpyOn } from '../../../../tests/vitest/utils.helper'; +import { UpdatedRemoteItemsDto } from '../../../context/shared/application/sync/remote-sync.contract'; + +function createUpdatedRemoteItemsDtoFixture(): UpdatedRemoteItemsDto { + return { + files: [ + { + bucket: 'bucket-1', + createdAt: '2024-01-01T00:00:00.000Z', + fileId: 'file-1', + folderId: 11, + id: 1, + modificationTime: '2024-01-02T00:00:00.000Z', + name: 'report.txt', + plainName: 'report', + size: 128, + type: 'txt', + updatedAt: '2024-01-03T00:00:00.000Z', + userId: 21, + status: 'EXISTS', + uuid: 'file-uuid-1', + }, + ], + folders: [ + { + bucket: 'bucket-2', + createdAt: '2024-02-01T00:00:00.000Z', + id: 2, + name: 'Projects', + parentId: null, + plainName: 'Projects', + status: 'EXISTS', + updatedAt: '2024-02-02T00:00:00.000Z', + uuid: 'folder-uuid-1', + }, + ], + }; +} + +const mocks = vi.hoisted(() => { + const filesCollection = { + getAll: vi.fn(), + getAllWhere: vi.fn(), + }; + + const foldersCollection = { + getAll: vi.fn(), + }; + + const controller = { + onStatusChange: vi.fn(), + startRemoteSync: vi.fn(), + getSyncStatus: vi.fn(), + resetRemoteSync: vi.fn(), + }; + + return { + debounce: vi.fn((fn: () => unknown) => fn), + filesCollection, + foldersCollection, + controller, + remoteSyncControllerFactory: vi.fn(() => controller), + }; +}); + +vi.mock('lodash', () => ({ + debounce: mocks.debounce, +})); + +vi.mock('../../../apps/main/database/collections/DriveFileCollection', () => ({ + DriveFilesCollection: vi.fn(() => mocks.filesCollection), +})); + +vi.mock('../../../apps/main/database/collections/DriveFolderCollection', () => ({ + DriveFoldersCollection: vi.fn(() => mocks.foldersCollection), +})); + +vi.mock('./remote-sync-controller', () => ({ + createRemoteSyncController: mocks.remoteSyncControllerFactory, +})); + +vi.mock('./remote-sync-error-handler', () => ({ + createRemoteSyncErrorHandler: vi.fn(() => ({ + handleSyncError: vi.fn(), + })), +})); + +type ServiceModule = typeof import('./service'); + +async function loadServiceModule() { + vi.resetModules(); + + const eventBusModule = await import('../../../apps/main/event-bus'); + const initialSyncReadyModule = await import('./InitialSyncReady'); + const windowsModule = await import('../../../apps/main/windows'); + + const eventBusEmitMock = partialSpyOn(eventBusModule.default, 'emit'); + const broadcastToWindowsMock = partialSpyOn(windowsModule, 'broadcastToWindows'); + const getIsInitialSyncReadyMock = partialSpyOn(initialSyncReadyModule, 'isInitialSyncReady'); + const setInitialSyncStateMock = partialSpyOn(initialSyncReadyModule, 'setInitialSyncState'); + + getIsInitialSyncReadyMock.mockReturnValue(false); + + const service = (await import('./service')) as ServiceModule; + + return { + broadcastToWindowsMock, + eventBusEmitMock, + getIsInitialSyncReadyMock, + service, + setInitialSyncStateMock, + }; +} + +function getRegisteredStatusChangeCallback() { + return mocks.controller.onStatusChange.mock.calls.at(-1)?.[0] as ((status: string) => Promise) | undefined; +} + +describe('service.test', () => { + beforeEach(() => { + mocks.controller.getSyncStatus.mockReturnValue('IDLE'); + const result = createUpdatedRemoteItemsDtoFixture(); + mocks.filesCollection.getAll.mockResolvedValue({ success: true, result: result.files }); + mocks.foldersCollection.getAll.mockResolvedValue({ success: true, result: result.folders }); + mocks.filesCollection.getAllWhere.mockResolvedValue({ result: [{ uuid: 'file-1' }] }); + }); + + it('should mark initial sync as ready and broadcast when the controller becomes synced for the first time', async () => { + // Given + const { eventBusEmitMock, broadcastToWindowsMock, setInitialSyncStateMock } = await loadServiceModule(); + const callback = getRegisteredStatusChangeCallback(); + + // When + await callback?.('SYNCED'); + + // Then + call(setInitialSyncStateMock).toBe('READY'); + call(eventBusEmitMock).toBe('INITIAL_SYNC_READY'); + call(broadcastToWindowsMock).toStrictEqual(['remote-sync-status-change', 'SYNCED']); + }); + + it('should only broadcast status changes when initial sync is already ready', async () => { + // Given + const { eventBusEmitMock, broadcastToWindowsMock, getIsInitialSyncReadyMock, setInitialSyncStateMock } = + await loadServiceModule(); + getIsInitialSyncReadyMock.mockReturnValue(true); + const callback = getRegisteredStatusChangeCallback(); + + // When + await callback?.('SYNCED'); + + // Then + expect(setInitialSyncStateMock).not.toHaveBeenCalled(); + expect(eventBusEmitMock).not.toHaveBeenCalledWith('INITIAL_SYNC_READY'); + call(broadcastToWindowsMock).toStrictEqual(['remote-sync-status-change', 'SYNCED']); + }); + + it('should return updated remote items from both collections', async () => { + // Given + const { service } = await loadServiceModule(); + const expected = createUpdatedRemoteItemsDtoFixture(); + + // When + const result = await service.getUpdatedRemoteItems(); + + // Then + expect(result).toStrictEqual(expected); + }); + + it('should throw when files cannot be retrieved from the local database', async () => { + // Given + mocks.filesCollection.getAll.mockResolvedValue({ success: false, result: [] }); + const { service } = await loadServiceModule(); + + // Then + await expect(service.getUpdatedRemoteItems()).rejects.toThrow( + 'Failed to retrieve all the drive files from local db', + ); + }); + + it('should delegate startRemoteSync to the controller', async () => { + // Given + const { service } = await loadServiceModule(); + + // When + await service.startRemoteSync(); + + // Then + calls(mocks.controller.startRemoteSync).toHaveLength(1); + }); + + it('should run a debounced resync and emit remote changes synched', async () => { + // Given + const { eventBusEmitMock, service } = await loadServiceModule(); + + // When + await service.resyncRemoteSync(); + + // Then + calls(mocks.debounce).toHaveLength(1); + calls(mocks.controller.startRemoteSync).toHaveLength(1); + call(eventBusEmitMock).toBe('REMOTE_CHANGES_SYNCHED'); + }); + + it('should return only existing files from the local collection', async () => { + // Given + const { service } = await loadServiceModule(); + + // When + const result = await service.getExistingFiles(); + + // Then + call(mocks.filesCollection.getAllWhere).toStrictEqual({ status: 'EXISTS' }); + expect(result).toStrictEqual([{ uuid: 'file-1' }]); + }); +}); diff --git a/src/apps/main/remote-sync/service.ts b/src/backend/features/remote-sync/service.ts similarity index 55% rename from src/apps/main/remote-sync/service.ts rename to src/backend/features/remote-sync/service.ts index f2468cd661..e96e984ac1 100644 --- a/src/apps/main/remote-sync/service.ts +++ b/src/backend/features/remote-sync/service.ts @@ -1,33 +1,37 @@ import { debounce } from 'lodash'; -import eventBus from '../event-bus'; -import { DriveFilesCollection } from '../database/collections/DriveFileCollection'; -import { DriveFoldersCollection } from '../database/collections/DriveFolderCollection'; -import { RemoteSyncManager } from './RemoteSyncManager'; -import { broadcastToWindows } from '../windows'; +import eventBus from '../../../apps/main/event-bus'; +import { DriveFilesCollection } from '../../../apps/main/database/collections/DriveFileCollection'; +import { DriveFoldersCollection } from '../../../apps/main/database/collections/DriveFolderCollection'; +import { createRemoteSyncController } from './remote-sync-controller'; +import { broadcastToWindows } from '../../../apps/main/windows'; import { isInitialSyncReady, setInitialSyncState } from './InitialSyncReady'; -import { RemoteSyncErrorHandler } from './RemoteSyncErrorHandler/RemoteSyncErrorHandler'; +import { createRemoteSyncErrorHandler } from './remote-sync-error-handler'; +import { registerRemoteSyncService } from '../../../context/shared/application/sync/remote-sync-service'; +import { toRemoteSyncFileDto } from './to-remote-sync-file-dto'; +import { toRemoteSyncFolderDto } from './to-remote-sync-folder-dto'; const SYNC_DEBOUNCE_DELAY = 3_000; const driveFilesCollection = new DriveFilesCollection(); const driveFoldersCollection = new DriveFoldersCollection(); -const errorHandler = new RemoteSyncErrorHandler(); - -export const remoteSyncManager = new RemoteSyncManager( - { +const errorHandler = createRemoteSyncErrorHandler(); +const remoteSyncControllerPops = { + db: { files: driveFilesCollection, folders: driveFoldersCollection, }, - { + config: { fetchFilesLimitPerRequest: 1000, fetchFoldersLimitPerRequest: 1000, syncFiles: true, syncFolders: true, }, errorHandler, -); +}; + +export const remoteSyncController = createRemoteSyncController(); -remoteSyncManager.onStatusChange(async (newStatus) => { +remoteSyncController.onStatusChange(async (newStatus) => { if (!isInitialSyncReady() && newStatus === 'SYNCED') { setInitialSyncState('READY'); eventBus.emit('INITIAL_SYNC_READY'); @@ -44,14 +48,15 @@ export async function getUpdatedRemoteItems() { if (!allDriveFiles.success) throw new Error('Failed to retrieve all the drive files from local db'); if (!allDriveFolders.success) throw new Error('Failed to retrieve all the drive folders from local db'); + return { - files: allDriveFiles.result, - folders: allDriveFolders.result, + files: allDriveFiles.result.map(toRemoteSyncFileDto), + folders: allDriveFolders.result.map(toRemoteSyncFolderDto), }; } export async function startRemoteSync(): Promise { - await remoteSyncManager.startRemoteSync(); + await remoteSyncController.startRemoteSync(remoteSyncControllerPops); } const debouncedSynchronization = debounce(async () => { @@ -63,6 +68,12 @@ export async function resyncRemoteSync() { await debouncedSynchronization(); } +registerRemoteSyncService({ + getUpdatedRemoteItems, + startRemoteSync, + resyncRemoteSync, +}); + export async function getExistingFiles() { const allExisting = await driveFilesCollection.getAllWhere({ status: 'EXISTS', diff --git a/src/backend/features/remote-sync/sync-remote-items.ts b/src/backend/features/remote-sync/sync-remote-items.ts new file mode 100644 index 0000000000..cf4efeb381 --- /dev/null +++ b/src/backend/features/remote-sync/sync-remote-items.ts @@ -0,0 +1,95 @@ +import { logger } from '@internxt/drive-desktop-core/build/backend'; +import { Nullable } from '../../../apps/shared/types/Nullable'; +import { RemoteSyncError } from './errors'; +import { SyncConfig } from './helpers'; +import { waitBeforeRetry } from './wait-before-retry'; +import { Result } from '../../../context/shared/domain/Result'; + +type RemoteSyncItem = { + updatedAt: string; + name?: string; +}; + +type Props = { + from?: Date; + finishMessage: string; + syncConfig: SyncConfig; + syncItemType: 'files' | 'folders'; + getCheckpoint: () => Promise>; + fetchRemoteItems: (updatedAtCheckpoint?: Date) => Promise>; + persistRemoteItems: (items: TItem[]) => Promise; + onSyncFailed: () => void; + onSyncFinished: () => void; + onSyncProgress: (items: TItem[]) => void; + onSyncStateChanged: () => void; + handleSyncError: (error: RemoteSyncError, itemName: string, checkpoint?: Date) => void; +}; + +export async function syncRemoteItems({ + from, + finishMessage, + syncConfig, + syncItemType, + getCheckpoint, + fetchRemoteItems, + persistRemoteItems, + onSyncFailed, + onSyncFinished, + onSyncProgress, + onSyncStateChanged, + handleSyncError, +}: Props) { + let checkpoint = from ?? (await getCheckpoint()); + let hasMore = true; + let retryCount = 0; + + while (hasMore && retryCount < syncConfig.maxRetries) { + let lastSyncedItem: TItem | null = null; + + try { + const { error, data } = await fetchRemoteItems(checkpoint); + if (error) throw error; + + const { hasMore: moreAvailable, result } = data; + + await persistRemoteItems(result); + onSyncProgress(result); + + lastSyncedItem = result.at(-1) ?? null; + hasMore = moreAvailable; + + if (hasMore && lastSyncedItem) { + checkpoint = new Date(lastSyncedItem.updatedAt); + } + + retryCount = 0; + } catch (error) { + retryCount++; + + if (error instanceof RemoteSyncError) { + handleSyncError(error, lastSyncedItem?.name ?? 'unknown', checkpoint); + } else { + logger.error({ + tag: 'SYNC-ENGINE', + msg: `Remote ${syncItemType} sync failed with uncontrolled error: `, + error, + }); + } + + if (retryCount >= syncConfig.maxRetries) { + onSyncFailed(); + onSyncStateChanged(); + return; + } + + await waitBeforeRetry({ retryCount }); + } + } + + logger.debug({ + tag: 'SYNC-ENGINE', + msg: finishMessage, + }); + onSyncFinished(); + onSyncStateChanged(); +} diff --git a/src/backend/features/remote-sync/to-remote-sync-file-dto.ts b/src/backend/features/remote-sync/to-remote-sync-file-dto.ts new file mode 100644 index 0000000000..ff8a12ee32 --- /dev/null +++ b/src/backend/features/remote-sync/to-remote-sync-file-dto.ts @@ -0,0 +1,21 @@ +import { DriveFile } from '../../../apps/main/database/entities/DriveFile'; +import { RemoteSyncFileDto } from '../../../context/shared/application/sync/remote-sync.contract'; + +export function toRemoteSyncFileDto(file: DriveFile): RemoteSyncFileDto { + return { + bucket: file.bucket, + createdAt: file.createdAt, + fileId: file.fileId, + folderId: file.folderId, + id: file.id, + modificationTime: file.modificationTime, + name: file.name ?? '', + plainName: file.plainName, + size: file.size, + type: file.type ?? null, + updatedAt: file.updatedAt, + userId: file.userId, + status: file.status, + uuid: file.uuid, + }; +} diff --git a/src/backend/features/remote-sync/to-remote-sync-folder-dto.ts b/src/backend/features/remote-sync/to-remote-sync-folder-dto.ts new file mode 100644 index 0000000000..1482d0e4c1 --- /dev/null +++ b/src/backend/features/remote-sync/to-remote-sync-folder-dto.ts @@ -0,0 +1,16 @@ +import { DriveFolder } from '../../../apps/main/database/entities/DriveFolder'; +import { RemoteSyncFolderDto } from '../../../context/shared/application/sync/remote-sync.contract'; + +export function toRemoteSyncFolderDto(folder: DriveFolder): RemoteSyncFolderDto { + return { + bucket: folder.bucket ?? null, + createdAt: folder.createdAt, + id: folder.id, + name: folder.name ?? '', + parentId: folder.parentId ?? null, + updatedAt: folder.updatedAt, + plainName: folder.plainName ?? null, + status: folder.status, + uuid: folder.uuid, + }; +} diff --git a/src/backend/features/remote-sync/wait-before-retry.ts b/src/backend/features/remote-sync/wait-before-retry.ts new file mode 100644 index 0000000000..f94c6ab1ff --- /dev/null +++ b/src/backend/features/remote-sync/wait-before-retry.ts @@ -0,0 +1,7 @@ +type Props = { + retryCount: number; +}; + +export async function waitBeforeRetry({ retryCount }: Props) { + await new Promise((resolve) => setTimeout(resolve, 1000 * retryCount)); +} diff --git a/src/context/shared/application/sync/TriggerRemoteSyncOnFileOverridden.ts b/src/context/shared/application/sync/TriggerRemoteSyncOnFileOverridden.ts index d352376ae0..84f1a0dca2 100644 --- a/src/context/shared/application/sync/TriggerRemoteSyncOnFileOverridden.ts +++ b/src/context/shared/application/sync/TriggerRemoteSyncOnFileOverridden.ts @@ -2,7 +2,7 @@ import { Service } from 'diod'; import { FileOverriddenDomainEvent } from '../../../virtual-drive/files/domain/events/FileOverriddenDomainEvent'; import { DomainEventSubscriber } from '../../domain/DomainEventSubscriber'; import { DomainEventClass } from '../../domain/DomainEvent'; -import { resyncRemoteSync } from '../../../../apps/main/remote-sync/service'; +import { getRemoteSyncService } from './remote-sync-service'; @Service() export class TriggerRemoteSyncOnFileOverridden implements DomainEventSubscriber { @@ -11,6 +11,6 @@ export class TriggerRemoteSyncOnFileOverridden implements DomainEventSubscriber< } async on(): Promise { - await resyncRemoteSync(); + await getRemoteSyncService().resyncRemoteSync(); } } diff --git a/src/context/shared/application/sync/remote-sync-service.test.ts b/src/context/shared/application/sync/remote-sync-service.test.ts new file mode 100644 index 0000000000..3764002f63 --- /dev/null +++ b/src/context/shared/application/sync/remote-sync-service.test.ts @@ -0,0 +1,30 @@ +describe('remote-sync-service.test', () => { + it('should throw when the remote sync service has not been registered', async () => { + // Given + vi.resetModules(); + const { getRemoteSyncService } = await import('./remote-sync-service'); + + // When + const getService = () => getRemoteSyncService(); + + // Then + expect(getService).toThrow('Remote sync service has not been registered'); + }); + + it('should return the registered remote sync service', async () => { + // Given + vi.resetModules(); + const { getRemoteSyncService, registerRemoteSyncService } = await import('./remote-sync-service'); + const service = { + getUpdatedRemoteItems: vi.fn(), + startRemoteSync: vi.fn(), + resyncRemoteSync: vi.fn(), + }; + + // When + registerRemoteSyncService(service); + + // Then + expect(getRemoteSyncService()).toBe(service); + }); +}); diff --git a/src/context/shared/application/sync/remote-sync-service.ts b/src/context/shared/application/sync/remote-sync-service.ts new file mode 100644 index 0000000000..c903f37454 --- /dev/null +++ b/src/context/shared/application/sync/remote-sync-service.ts @@ -0,0 +1,21 @@ +import { UpdatedRemoteItemsDto } from './remote-sync.contract'; + +export type RemoteSyncService = { + getUpdatedRemoteItems: () => Promise; + startRemoteSync: () => Promise; + resyncRemoteSync: () => Promise; +}; + +let remoteSyncService: RemoteSyncService | undefined; + +export function registerRemoteSyncService(service: RemoteSyncService) { + remoteSyncService = service; +} + +export function getRemoteSyncService() { + if (!remoteSyncService) { + throw new Error('Remote sync service has not been registered'); + } + + return remoteSyncService; +} diff --git a/src/context/shared/application/sync/remote-sync.contract.ts b/src/context/shared/application/sync/remote-sync.contract.ts new file mode 100644 index 0000000000..efa6b3a23b --- /dev/null +++ b/src/context/shared/application/sync/remote-sync.contract.ts @@ -0,0 +1,33 @@ +export type RemoteSyncFileDto = { + bucket: string; + createdAt: string; + fileId: string; + folderId: number; + id: number; + modificationTime: string; + name: string; + plainName: string; + size: number; + type: string | null; + updatedAt: string; + userId: number; + status: string; + uuid: string; +}; + +export type RemoteSyncFolderDto = { + bucket: string | null; + createdAt: string; + id: number; + name: string; + parentId: number | null; + updatedAt: string; + plainName: string | null; + status: string; + uuid: string; +}; + +export type UpdatedRemoteItemsDto = { + files: RemoteSyncFileDto[]; + folders: RemoteSyncFolderDto[]; +}; diff --git a/src/context/virtual-drive/remoteTree/infrastructure/SQLiteRemoteItemsGenerator.ts b/src/context/virtual-drive/remoteTree/infrastructure/SQLiteRemoteItemsGenerator.ts index 2e75852734..cc685dcda0 100644 --- a/src/context/virtual-drive/remoteTree/infrastructure/SQLiteRemoteItemsGenerator.ts +++ b/src/context/virtual-drive/remoteTree/infrastructure/SQLiteRemoteItemsGenerator.ts @@ -1,13 +1,13 @@ import { Service } from 'diod'; -import { getUpdatedRemoteItems, startRemoteSync } from '../../../../apps/main/remote-sync/service'; import { ServerFile, ServerFileStatus } from '../../../shared/domain/ServerFile'; import { ServerFolder, ServerFolderStatus } from '../../../shared/domain/ServerFolder'; import { RemoteItemsGenerator } from '../domain/RemoteItemsGenerator'; +import { getRemoteSyncService } from '../../../shared/application/sync/remote-sync-service'; @Service() export class SQLiteRemoteItemsGenerator implements RemoteItemsGenerator { async getAll(): Promise<{ files: ServerFile[]; folders: ServerFolder[] }> { - const result = await getUpdatedRemoteItems(); + const result = await getRemoteSyncService().getUpdatedRemoteItems(); const files = result.files.map((updatedFile) => { return { @@ -21,7 +21,7 @@ export class SQLiteRemoteItemsGenerator implements RemoteItemsGenerator { name: updatedFile.name, plainName: updatedFile.plainName, size: updatedFile.size, - type: updatedFile.type ?? null, + type: updatedFile.type ?? '', updatedAt: updatedFile.updatedAt, userId: updatedFile.userId, status: updatedFile.status as ServerFileStatus, @@ -47,6 +47,6 @@ export class SQLiteRemoteItemsGenerator implements RemoteItemsGenerator { } async forceRefresh(): Promise { - await startRemoteSync(); + await getRemoteSyncService().startRemoteSync(); } } diff --git a/src/infra/sqlite/services/file/create-or-update-file-by-batch.ts b/src/infra/sqlite/services/file/create-or-update-file-by-batch.ts index 221897d4c1..26feaf6bdf 100644 --- a/src/infra/sqlite/services/file/create-or-update-file-by-batch.ts +++ b/src/infra/sqlite/services/file/create-or-update-file-by-batch.ts @@ -2,7 +2,7 @@ import { fileRepository } from '../drive-file'; import { logger } from '@internxt/drive-desktop-core/build/backend'; import { parseData } from './parse-data'; import { SqliteError } from '../common/sqlite-error'; -import { RemoteSyncedFile } from 'src/apps/main/remote-sync/helpers'; +import { RemoteSyncedFile } from 'src/backend/features/remote-sync/helpers'; const BATCH_SIZE = 500; diff --git a/src/infra/sqlite/services/file/parse-data.ts b/src/infra/sqlite/services/file/parse-data.ts index 9b64a510af..f1594465b0 100644 --- a/src/infra/sqlite/services/file/parse-data.ts +++ b/src/infra/sqlite/services/file/parse-data.ts @@ -1,4 +1,4 @@ -import { RemoteSyncedFile } from 'src/apps/main/remote-sync/helpers'; +import { RemoteSyncedFile } from 'src/backend/features/remote-sync/helpers'; import { FileUuid, SimpleDriveFile, ContentsId } from '../../../../apps/main/database/entities/DriveFile'; type TProps = { diff --git a/src/infra/sqlite/services/folder/create-or-update-folder-by-batch.ts b/src/infra/sqlite/services/folder/create-or-update-folder-by-batch.ts index 6debdfa3c5..3c95f9db09 100644 --- a/src/infra/sqlite/services/folder/create-or-update-folder-by-batch.ts +++ b/src/infra/sqlite/services/folder/create-or-update-folder-by-batch.ts @@ -2,7 +2,7 @@ import { folderRepository } from '../drive-folder'; import { logger } from '@internxt/drive-desktop-core/build/backend'; import { parseData } from './parse-data'; import { SqliteError } from '../common/sqlite-error'; -import { RemoteSyncedFolder } from 'src/apps/main/remote-sync/helpers'; +import { RemoteSyncedFolder } from 'src/backend/features/remote-sync/helpers'; const BATCH_SIZE = 500; diff --git a/src/infra/sqlite/services/folder/parse-data.ts b/src/infra/sqlite/services/folder/parse-data.ts index 43409909e2..5e99ea5fd0 100644 --- a/src/infra/sqlite/services/folder/parse-data.ts +++ b/src/infra/sqlite/services/folder/parse-data.ts @@ -1,4 +1,4 @@ -import { RemoteSyncedFolder } from 'src/apps/main/remote-sync/helpers'; +import { RemoteSyncedFolder } from 'src/backend/features/remote-sync/helpers'; import { FolderUuid, SimpleDriveFolder } from '../../../../apps/main/database/entities/DriveFolder'; type TProps = { diff --git a/tests/vitest/ipc.helper.ts b/tests/vitest/ipc.helper.ts new file mode 100644 index 0000000000..97f955f3d7 --- /dev/null +++ b/tests/vitest/ipc.helper.ts @@ -0,0 +1,12 @@ +import { ipcMain } from 'electron'; + +export function getIpcHandler(eventName: string, useOn = false) { + const ipc = ipcMain as unknown as { + handle: { mock: { calls: Array<[string, (...args: unknown[]) => unknown]> } }; + on: { mock: { calls: Array<[string, (...args: unknown[]) => unknown]> } }; + }; + + const calls = useOn ? ipc.on.mock.calls : ipc.handle.mock.calls; + + return calls.find(([name]) => name === eventName)?.[1]; +} \ No newline at end of file