From c869a17a8e29b5127e95aa01fd603eea5a0380f0 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Tue, 26 May 2026 14:07:18 -0700 Subject: [PATCH] Port helper apps from aj-cortex --- helper-apps/cortex-file-handler/.npmrc | 1 + helper-apps/cortex-file-handler/Dockerfile | 12 +- .../cortex-file-handler/package-lock.json | 4 +- helper-apps/cortex-file-handler/package.json | 2 +- .../cortex-file-handler/src/blobHandler.js | 440 ++++- .../cortex-file-handler/src/constants.js | 64 + .../cortex-file-handler/src/fileChunker.js | 23 +- helper-apps/cortex-file-handler/src/index.js | 1486 +++++++++++++++-- helper-apps/cortex-file-handler/src/redis.js | 49 +- .../src/services/ConversionService.js | 9 +- .../src/services/FileConversionService.js | 26 +- .../services/storage/AzureStorageProvider.js | 353 +++- .../services/storage/GCSStorageProvider.js | 148 +- .../src/services/storage/StorageFactory.js | 12 +- .../src/services/storage/StorageService.js | 332 +++- helper-apps/cortex-file-handler/src/start.js | 4 +- .../src/utils/filenameUtils.js | 4 + .../utils/legacyWorkspacePrivateResolver.js | 460 +++++ .../src/utils/targetBlobPathUtils.js | 32 + .../tests/blobHandler.test.js | 77 + .../tests/checkHashShortLived.test.js | 79 + .../cortex-file-handler/tests/cleanup.test.js | 8 +- .../tests/containerConversionFlow.test.js | 1 - .../tests/fileChunker.test.js | 15 + .../tests/fileUpload.test.js | 7 +- .../tests/filenameUtils.test.js | 16 + .../tests/folderStorage.test.js | 698 ++++++++ .../tests/getOperations.test.js | 220 +++ .../legacyWorkspacePrivateResolver.test.js | 357 ++++ .../tests/postOperations.test.js | 7 +- .../tests/renameTargetBlobPath.test.js | 74 + .../tests/setRetention.test.js | 2 +- .../cortex-file-handler/tests/start.test.js | 2 +- .../storage/AzureStorageProvider.test.js | 478 +++++- .../tests/storage/StorageFactory.test.js | 44 +- .../tests/storage/StorageService.test.js | 9 +- .../tests/testUtils.helper.js | 18 +- helper-apps/cortex-workspace/.npmrc | 1 + helper-apps/cortex-workspace/Dockerfile | 41 + helper-apps/cortex-workspace/entrypoint.sh | 136 ++ helper-apps/cortex-workspace/lib/auth.js | 42 + helper-apps/cortex-workspace/lib/files.js | 201 +++ helper-apps/cortex-workspace/lib/shell.js | 214 +++ helper-apps/cortex-workspace/lib/system.js | 881 ++++++++++ .../cortex-workspace/package-lock.json | 831 +++++++++ helper-apps/cortex-workspace/package.json | 18 + helper-apps/cortex-workspace/server.js | 343 ++++ .../cortex-workspace/tests/files.test.js | 138 ++ .../cortex-workspace/tests/shell.test.js | 96 ++ .../cortex-workspace/tests/system.test.js | 210 +++ 50 files changed, 8295 insertions(+), 430 deletions(-) create mode 100644 helper-apps/cortex-file-handler/.npmrc create mode 100644 helper-apps/cortex-file-handler/src/utils/legacyWorkspacePrivateResolver.js create mode 100644 helper-apps/cortex-file-handler/src/utils/targetBlobPathUtils.js create mode 100644 helper-apps/cortex-file-handler/tests/filenameUtils.test.js create mode 100644 helper-apps/cortex-file-handler/tests/folderStorage.test.js create mode 100644 helper-apps/cortex-file-handler/tests/legacyWorkspacePrivateResolver.test.js create mode 100644 helper-apps/cortex-file-handler/tests/renameTargetBlobPath.test.js create mode 100644 helper-apps/cortex-workspace/.npmrc create mode 100644 helper-apps/cortex-workspace/Dockerfile create mode 100755 helper-apps/cortex-workspace/entrypoint.sh create mode 100644 helper-apps/cortex-workspace/lib/auth.js create mode 100644 helper-apps/cortex-workspace/lib/files.js create mode 100644 helper-apps/cortex-workspace/lib/shell.js create mode 100644 helper-apps/cortex-workspace/lib/system.js create mode 100644 helper-apps/cortex-workspace/package-lock.json create mode 100644 helper-apps/cortex-workspace/package.json create mode 100644 helper-apps/cortex-workspace/server.js create mode 100644 helper-apps/cortex-workspace/tests/files.test.js create mode 100644 helper-apps/cortex-workspace/tests/shell.test.js create mode 100644 helper-apps/cortex-workspace/tests/system.test.js diff --git a/helper-apps/cortex-file-handler/.npmrc b/helper-apps/cortex-file-handler/.npmrc new file mode 100644 index 00000000..7253a5ce --- /dev/null +++ b/helper-apps/cortex-file-handler/.npmrc @@ -0,0 +1 @@ +min-release-age=7 diff --git a/helper-apps/cortex-file-handler/Dockerfile b/helper-apps/cortex-file-handler/Dockerfile index f89889ee..66c4b096 100644 --- a/helper-apps/cortex-file-handler/Dockerfile +++ b/helper-apps/cortex-file-handler/Dockerfile @@ -1,8 +1,16 @@ FROM node:22-alpine +RUN wget -qO /tmp/npm-11.14.1.tgz https://registry.npmjs.org/npm/-/npm-11.14.1.tgz \ + && mkdir -p /opt/npm-11.14.1 \ + && tar -xzf /tmp/npm-11.14.1.tgz -C /opt/npm-11.14.1 --strip-components=1 \ + && node /opt/npm-11.14.1/bin/npm-cli.js --version \ + && printf '%s\n' '#!/bin/sh' 'exec node /opt/npm-11.14.1/bin/npm-cli.js "$@"' > /usr/local/bin/npm \ + && printf '%s\n' '#!/bin/sh' 'exec node /opt/npm-11.14.1/bin/npx-cli.js "$@"' > /usr/local/bin/npx \ + && chmod +x /usr/local/bin/npm /usr/local/bin/npx \ + && rm /tmp/npm-11.14.1.tgz WORKDIR /usr/src/app -COPY package*.json ./ +COPY .npmrc package*.json ./ RUN npm install @@ -16,4 +24,4 @@ EXPOSE 7071 # RUN npm run build -CMD [ "npm", "start" ] \ No newline at end of file +CMD [ "npm", "start" ] diff --git a/helper-apps/cortex-file-handler/package-lock.json b/helper-apps/cortex-file-handler/package-lock.json index 71165792..dd1accf6 100644 --- a/helper-apps/cortex-file-handler/package-lock.json +++ b/helper-apps/cortex-file-handler/package-lock.json @@ -1,12 +1,12 @@ { "name": "@aj-archipelago/cortex-file-handler", - "version": "2.8.1", + "version": "2.9.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@aj-archipelago/cortex-file-handler", - "version": "2.8.1", + "version": "2.9.1", "dependencies": { "@azure/storage-blob": "^12.13.0", "@distube/ytdl-core": "^4.14.3", diff --git a/helper-apps/cortex-file-handler/package.json b/helper-apps/cortex-file-handler/package.json index 638ce7ba..6d79114a 100644 --- a/helper-apps/cortex-file-handler/package.json +++ b/helper-apps/cortex-file-handler/package.json @@ -1,6 +1,6 @@ { "name": "@aj-archipelago/cortex-file-handler", - "version": "2.8.1", + "version": "2.9.1", "description": "File handling service for Cortex - handles file uploads, media chunking, and document processing", "type": "module", "main": "src/index.js", diff --git a/helper-apps/cortex-file-handler/src/blobHandler.js b/helper-apps/cortex-file-handler/src/blobHandler.js index 07becac5..c4dfa251 100644 --- a/helper-apps/cortex-file-handler/src/blobHandler.js +++ b/helper-apps/cortex-file-handler/src/blobHandler.js @@ -7,7 +7,6 @@ import { v4 as uuidv4 } from "uuid"; import Busboy from "busboy"; import { PassThrough } from "stream"; import { Storage } from "@google-cloud/storage"; -import { BlobServiceClient } from "@azure/storage-blob"; import axios from "axios"; import mime from "mime-types"; @@ -17,10 +16,11 @@ import { generateBlobName, } from "./utils/filenameUtils.js"; import { publicFolder, port, ipAddress } from "./start.js"; -import { - CONVERTED_EXTENSIONS, +import { + CONVERTED_EXTENSIONS, AZURITE_ACCOUNT_NAME, getDefaultContainerName, + getUserContainerName, GCS_BUCKETNAME, AZURE_STORAGE_CONTAINER_NAME } from "./constants.js"; @@ -82,6 +82,244 @@ function isEncoded(str) { return /%[0-9A-Fa-f]{2}/.test(str); } +/** + * Construct folder path for file storage based on user/chat context. + * MIRROR: Keep in sync with vendor/cortex/lib/fileUtils.js constructFolderPath() + * @param {Object} options - Options for folder path construction + * @param {string} options.userId - Container owner ID + * @param {string} options.chatId - Chat ID (if file is chat-scoped) + * @param {string} options.workspaceId - Workspace ID (for workspace-user-legacy or workspace-shared-legacy scopes) + * @param {string} options.appletId - Applet ID (for applet-user scopes) + * @param {string} options.contextId - Optional scoped context ID + * @param {string} options.fileScope - File scope: 'all', 'global', 'media', 'chat', 'workspace-user-legacy', 'applet-user', 'applet-shared', 'profile', 'articles', 'workspace-shared-legacy' + * @returns {string|null} Folder path or null if no valid path can be constructed + */ +// IDs must be alphanumeric, hyphens, or underscores — rejects traversal and injection. +const SAFE_ID = /^[A-Za-z0-9_-]+$/; + +function isValidId(id) { + return typeof id === 'string' && id.length > 0 && id.length <= 128 && SAFE_ID.test(id); +} + +function sanitizeSubPath(subPath) { + if (!subPath || typeof subPath !== 'string') return null; + + const segments = subPath + .replace(/\\/g, '/') + .split('/') + .map((segment) => segment.trim()) + .filter(Boolean); + + if (segments.length === 0 || segments.some((segment) => !isValidId(segment))) { + return null; + } + + return segments.join('/'); +} + +function parseAppletUserContextId(contextId = null) { + if (typeof contextId !== "string" || !contextId.startsWith("applet-user:")) { + return { appletId: null, userId: null }; + } + + const parts = contextId.split(":"); + if (parts.length < 3) { + return { appletId: null, userId: null }; + } + + return { + appletId: parts[1] || null, + userId: parts.slice(2).join(":") || null, + }; +} + +function resolveAppletScopeId({ + appletId = null, + workspaceId = null, + contextId = null, +} = {}) { + if (appletId && isValidId(appletId)) { + return appletId; + } + + const parsed = parseAppletUserContextId(contextId); + if (parsed.appletId && isValidId(parsed.appletId)) { + return parsed.appletId; + } + + if (workspaceId && isValidId(workspaceId)) { + return workspaceId; + } + + return null; +} + +function buildAppletUserContextId(userId = null, appletId = null) { + if (!userId || !appletId) { + return null; + } + return `applet-user:${appletId}:${userId}`; +} + +function buildAppletSharedContextId(appletId = null) { + if (!appletId) { + return null; + } + return `applet-shared:${appletId}`; +} + +function constructFolderPath({ + userId, + chatId, + workspaceId, + appletId = null, + contextId = null, + fileScope, +}) { + // For workspace-shared-legacy scope, userId is not required — workspaceId is the owner + if (fileScope === 'workspace-shared-legacy') { + if (!workspaceId || !isValidId(workspaceId)) return null; + return ''; // root of per-workspace container + } + + const ownerId = contextId || userId || null; + if (!ownerId) { + return null; + } + + // Validate path-part IDs to prevent traversal / cross-tenant access. + // ownerId is used only for container selection and may be a compound scoped context ID. + if (chatId && !isValidId(chatId)) return null; + if (workspaceId && !isValidId(workspaceId)) return null; + if (appletId && !isValidId(appletId)) return null; + + // Folder names encode the logical scope within the selected container. + // applet-user stays in the user's container under an applet-specific folder. + switch (fileScope) { + case 'all': + // Root of the scoped container — lists everything + return ''; + case 'global': + return 'global'; + case 'media': + return 'media'; + case 'chat': + if (!chatId) { + return 'global'; + } + return `chats/${chatId}`; + case 'workspace-user-legacy': + if (!workspaceId) { + return 'global'; + } + return `applets/${workspaceId}`; + case 'applet-user': { + const scopedAppletId = resolveAppletScopeId({ + appletId, + workspaceId, + contextId, + }); + if (!scopedAppletId) { + return null; + } + return `applets/${scopedAppletId}`; + } + case 'applet-shared': + return 'applet-shared'; + case 'profile': + return 'profile'; + case 'articles': + return 'articles'; + case 'applets': + return 'applets'; + case 'skills': + return 'skills'; + case 'automations': + return 'automations'; + default: + return 'global'; + } +} + +function getScopedLogicalContextId({ + contextId = null, + userId = null, + workspaceId = null, + appletId = null, + fileScope = null, +} = {}) { + if (fileScope === 'workspace-shared-legacy') { + return workspaceId || contextId || null; + } + + if (fileScope === 'applet-user') { + if ( + typeof contextId === 'string' + && contextId.startsWith('applet-user:') + ) { + return contextId; + } + + const parsedContext = parseAppletUserContextId(contextId); + const parsedUser = parseAppletUserContextId(userId); + const resolvedUserId = + parsedContext.userId + || parsedUser.userId + || userId + || contextId + || null; + const resolvedAppletId = resolveAppletScopeId({ + appletId: parsedContext.appletId || parsedUser.appletId || appletId, + workspaceId, + contextId, + }); + + return buildAppletUserContextId(resolvedUserId, resolvedAppletId); + } + + if (fileScope === 'applet-shared') { + if (contextId) { + return contextId; + } + + if (typeof userId === 'string' && userId.startsWith('applet-shared:')) { + return userId; + } + + return buildAppletSharedContextId( + resolveAppletScopeId({ appletId, workspaceId, contextId }), + ); + } + + return contextId || userId || null; +} + +function getScopedContainerOwnerId({ + contextId = null, + userId = null, + workspaceId = null, + appletId = null, + fileScope = null, +} = {}) { + if (fileScope === 'workspace-shared-legacy') { + return workspaceId || null; + } + + if (fileScope === 'applet-user') { + const parsed = parseAppletUserContextId(contextId); + const scopedUserId = parsed.userId || userId || null; + return scopedUserId; + } + + return getScopedLogicalContextId({ + contextId, + userId, + workspaceId, + appletId, + fileScope, + }); +} + // Helper function to ensure GCS URLs are never encoded function ensureUnencodedGcsUrl(url) { if (!url || !url.startsWith("gs://")) { @@ -216,9 +454,8 @@ async function generateShortLivedUrlForConvertedFile(context, convertedUrl, logS if (primaryProvider.generateShortLivedSASToken && primaryProvider.extractBlobNameFromUrl) { const blobName = primaryProvider.extractBlobNameFromUrl(convertedUrl); if (blobName) { - const { containerClient } = await primaryProvider.getBlobClient(); + await primaryProvider.ensureInitialized(); const sasToken = primaryProvider.generateShortLivedSASToken( - containerClient, blobName, 5 ); @@ -236,38 +473,10 @@ async function generateShortLivedUrlForConvertedFile(context, convertedUrl, logS } export const getBlobClient = async () => { - const connectionString = process.env.AZURE_STORAGE_CONNECTION_STRING; - // Always use default container from env var - const finalContainerName = getDefaultContainerName(); - - if (!connectionString || !finalContainerName) { - throw new Error( - "Missing Azure Storage connection string or container name environment variable", - ); - } - - const blobServiceClient = - BlobServiceClient.fromConnectionString(connectionString); - - const serviceProperties = await blobServiceClient.getProperties(); - if (!serviceProperties.defaultServiceVersion) { - serviceProperties.defaultServiceVersion = "2020-02-10"; - await blobServiceClient.setProperties(serviceProperties); - } - - const containerClient = blobServiceClient.getContainerClient(finalContainerName); - - return { blobServiceClient, containerClient }; + const provider = await StorageFactory.getInstance().getAzureProvider(); + return await provider.getBlobClient(); }; -async function saveFileToBlob(chunkPath, requestId, filename = null) { - // Use provider for consistency with cache control headers - // Container parameter is ignored - always uses default container from env var - const storageFactory = StorageFactory.getInstance(); - const provider = await storageFactory.getAzureProvider(); - return await provider.uploadFile({}, chunkPath, requestId, null, filename); -} - //deletes blob that has the requestId async function deleteBlob(requestId) { if (!requestId) throw new Error("Missing requestId parameter"); @@ -359,9 +568,16 @@ function uploadBlob( fields[fieldname] = value; // Store all fields }); + // Promise that resolves once busboy has finished parsing the + // entire multipart stream (all fields + file data consumed). + // Used inside processFile to ensure late-arriving fields + // (e.g. hash appended after the file) are available. + let resolveBusboyFinished; + const busboyFinished = new Promise((r) => { resolveBusboyFinished = r; }); + busboy.on("file", async (fieldname, file, info) => { if (errorOccurred) return; - + hasFile = true; // Validate file @@ -373,14 +589,10 @@ function uploadBlob( return; } - // Simple approach: small delay to allow container field to be processed - console.log("File received, giving fields time to process..."); - await new Promise(resolve => setTimeout(resolve, 20)); - // Container parameter is ignored - always uses default container from env var - - if (errorOccurred) return; // Check again after waiting - - // Container parameter is ignored - always uses default container from env var + // Fields that precede the file part are already available. + // Fields after the file part (e.g. hash) will arrive once + // the file data is consumed — processFile handles this by + // awaiting busboyFinished after the upload completes. await processFile(fieldname, file, info); }); @@ -396,11 +608,45 @@ function uploadBlob( return; } + // Extract folder-related fields from form data + const userId = fields.userId || null; + const chatId = fields.chatId || null; + const workspaceId = fields.workspaceId || null; + const contextId = fields.contextId || null; + const appletId = fields.appletId || null; + const fileScope = fields.fileScope || null; + const logicalContextId = getScopedLogicalContextId({ + contextId, + userId, + workspaceId, + appletId, + fileScope, + }); + + // Construct folder path for folder-based storage + let folderPath = constructFolderPath({ + userId, + chatId, + workspaceId, + appletId, + contextId: logicalContextId, + fileScope, + }); + + // Optional subPath appends a subdirectory within the fileScope folder. + const subPath = fields.subPath || null; + if (subPath && folderPath !== null) { + const sanitizedSub = sanitizeSubPath(subPath); + if (sanitizedSub) { + folderPath = folderPath ? `${folderPath}/${sanitizedSub}` : sanitizedSub; + } + } + // Prepare for streaming to cloud destinations const displayFilename = info.filename; // Preserve original filename for metadata const fileExtension = path.extname(displayFilename); const shortId = generateShortId(); - const uploadName = `${shortId}${fileExtension}`; + const uploadName = folderPath ? sanitizeFilename(displayFilename) : `${shortId}${fileExtension}`; // Extract content-type from busboy info (preserves charset if provided) const contentType = info.mimeType || null; const azureStream = !saveToLocal ? new PassThrough() : null; @@ -475,6 +721,17 @@ function uploadBlob( } }); + const containerOwnerId = getScopedContainerOwnerId({ + contextId: logicalContextId, + userId, + workspaceId, + appletId, + fileScope, + }); + const userContainerName = containerOwnerId + ? getUserContainerName(getDefaultContainerName(), containerOwnerId) + : null; + // Start cloud uploads immediately let azurePromise; if (!saveToLocal) { @@ -482,8 +739,9 @@ function uploadBlob( context, uploadName, azureStream, - null, // containerName ignored + userContainerName, contentType, + folderPath, // Pass folder path for folder-based storage ).catch(async (err) => { cloudUploadError = err; // Fallback: try from disk if available @@ -493,18 +751,24 @@ function uploadBlob( highWaterMark: 1024 * 1024, autoClose: true, }); - return saveToAzureStorage(context, uploadName, diskStream, null, contentType); + return saveToAzureStorage(context, uploadName, diskStream, userContainerName, contentType, folderPath); } throw err; }); } let gcsPromise; if (gcsStream) { + // GCS uses a shared bucket, so prefix with the scoped container owner + // to preserve the same isolation boundaries as Azure containers. + const gcsFolderPath = containerOwnerId + ? `${containerOwnerId}/${folderPath || ''}`.replace(/\/+$/, '') + : folderPath; gcsPromise = saveToGoogleStorage( context, uploadName, gcsStream, contentType, + gcsFolderPath, ).catch(async (err) => { cloudUploadError = err; if (diskWritePromise) { @@ -513,7 +777,7 @@ function uploadBlob( highWaterMark: 1024 * 1024, autoClose: true, }); - return saveToGoogleStorage(context, uploadName, diskStream, contentType); + return saveToGoogleStorage(context, uploadName, diskStream, contentType, gcsFolderPath); } throw err; }); @@ -549,18 +813,42 @@ function uploadBlob( return acc; }, {}), }; + // Wait for busboy to finish parsing the entire request so + // that form fields appended after the file (e.g. hash) are + // available. The file data is already consumed, so busboy + // can parse the remaining fields without backpressure. + await busboyFinished; + if (hash) result.hash = hash; - + // Store MIME type from upload (used by Cortex for file type detection) if (contentType) { result.mimeType = contentType; } - // Extract contextId from form fields if present - if (fields && fields.contextId) { - result.contextId = fields.contextId; + // Persist metadata in the same scoped Redis namespace that owns + // the uploaded blob, even when callers only send userId/workspaceId. + const uploadContextId = getScopedContainerOwnerId({ + contextId: logicalContextId, + userId, + workspaceId, + appletId, + fileScope, + }); + const mapContextId = logicalContextId || uploadContextId; + if (mapContextId) { + result.contextId = mapContextId; } - + + // Include folder metadata in result (for transparency/debugging) + if (folderPath) { + result.folderPath = folderPath; + } + if (userId) result.userId = userId; + if (workspaceId) result.workspaceId = workspaceId; + if (appletId) result.appletId = appletId; + if (fileScope) result.fileScope = fileScope; + // All uploads default to temporary (permanent: false) to match file collection logic result.permanent = false; @@ -638,12 +926,13 @@ function uploadBlob( if (conversion.converted) { context.log("Saving converted file (busboy)..."); // Save converted file to primary storage + // Pass folderPath so converted file is stored next to original const convertedSaveResult = await conversionService._saveConvertedFile( conversion.convertedPath, requestId, null, - null, // containerName ignored + folderPath, ); // Optionally save to GCS @@ -653,6 +942,8 @@ function uploadBlob( await conversionService._uploadChunkToGCS( conversion.convertedPath, requestId, + null, + folderPath, ); } @@ -709,6 +1000,7 @@ function uploadBlob( }; busboy.on("error", (error) => { + resolveBusboyFinished(); if (errorOccurred) return; errorOccurred = true; const err = new Error("No file provided in request"); @@ -717,6 +1009,7 @@ function uploadBlob( }); busboy.on("finish", () => { + resolveBusboyFinished(); if (!hasFile) { errorOccurred = true; const err = new Error("No file provided in request"); @@ -774,14 +1067,14 @@ async function saveToLocalStorage(context, requestId, encodedFilename, file) { } // Helper function to handle Azure blob storage -async function saveToAzureStorage(context, encodedFilename, file, containerName = null, contentType = null) { +async function saveToAzureStorage(context, encodedFilename, file, containerName = null, contentType = null, folderPath = null) { const storageFactory = StorageFactory.getInstance(); const provider = await storageFactory.getAzureProvider(containerName); - return await provider.uploadStream(context, encodedFilename, file, contentType); + return await provider.uploadStream(context, encodedFilename, file, contentType, 'temporary', folderPath); } // Wrapper that checks if GCS is configured -async function saveToGoogleStorage(context, encodedFilename, file, contentType = null) { +async function saveToGoogleStorage(context, encodedFilename, file, contentType = null, folderPath = null) { if (!gcs) { throw new Error("Google Cloud Storage is not initialized"); } @@ -790,7 +1083,7 @@ async function saveToGoogleStorage(context, encodedFilename, file, contentType = if (!gcsProvider) { throw new Error("GCS provider not available"); } - return await gcsProvider.uploadStream(context, encodedFilename, file, contentType); + return await gcsProvider.uploadStream(context, encodedFilename, file, contentType, 'temporary', folderPath); } async function uploadFile( @@ -955,7 +1248,7 @@ async function uploadFile( conversion.convertedPath, requestId, null, - containerName, + null, ); context.log("Converted file saved to primary storage"); @@ -1249,17 +1542,24 @@ async function ensureGCSUpload(context, existingFile) { return existingFile; } -async function uploadChunkToGCS(chunkPath, requestId, filename = null) { +async function uploadChunkToGCS(chunkPath, requestId, filename = null, folderPath = null) { if (!gcs) return null; - const dirName = requestId || uuidv4(); - // Use provided filename or generate LLM-friendly naming let gcsFileName; - if (filename) { - gcsFileName = `${dirName}/${filename}`; + if (folderPath) { + // Store in the same folder structure as the original file + const normalizedFolder = folderPath.replace(/^\/+|\/+$/g, ''); + const name = filename || path.basename(chunkPath); + gcsFileName = normalizedFolder ? `${normalizedFolder}/${name}` : name; } else { - const fileExtension = path.extname(chunkPath); - const shortId = generateShortId(); - gcsFileName = `${dirName}/${shortId}${fileExtension}`; + const dirName = requestId || uuidv4(); + // Use provided filename or generate LLM-friendly naming + if (filename) { + gcsFileName = `${dirName}/${filename}`; + } else { + const fileExtension = path.extname(chunkPath); + const shortId = generateShortId(); + gcsFileName = `${dirName}/${shortId}${fileExtension}`; + } } await gcs .bucket(GCS_BUCKETNAME) @@ -1268,7 +1568,6 @@ async function uploadChunkToGCS(chunkPath, requestId, filename = null) { } export { - saveFileToBlob, deleteBlob, deleteGCS, uploadBlob, @@ -1280,8 +1579,13 @@ export { uploadChunkToGCS, downloadFromGCS, getMimeTypeFromUrl, + constructFolderPath, + sanitizeSubPath, + getScopedLogicalContextId, + getScopedContainerOwnerId, // Re-export container constants getDefaultContainerName, + getUserContainerName, GCS_BUCKETNAME, AZURE_STORAGE_CONTAINER_NAME, }; diff --git a/helper-apps/cortex-file-handler/src/constants.js b/helper-apps/cortex-file-handler/src/constants.js index 00df7b02..762230e3 100644 --- a/helper-apps/cortex-file-handler/src/constants.js +++ b/helper-apps/cortex-file-handler/src/constants.js @@ -78,6 +78,7 @@ export const ACCEPTED_MIME_TYPES = { // Audio types "audio/wav": [".wav"], + "audio/x-wav": [".wav"], "audio/mpeg": [".mp3"], "audio/aac": [".aac"], "audio/ogg": [".ogg"], @@ -171,3 +172,66 @@ export const getDefaultContainerName = () => { // Export constant - evaluated at module load time, but getContainerName() handles defaults export const AZURE_STORAGE_CONTAINER_NAME = getContainerName(); export const GCS_BUCKETNAME = process.env.GCS_BUCKETNAME || "cortextempfiles"; + +function buildContainerName(baseName, sanitized) { + if (!sanitized) return baseName; + return `${baseName}-${sanitized}`; +} + +function sanitizeContainerContextId(contextId) { + return contextId + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/-+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 50); +} + +function sanitizeLegacyContainerContextId(contextId) { + return contextId + .toLowerCase() + .replace(/[^a-z0-9-]/g, '') + .slice(0, 50); +} + +/** + * Derive a per-user blob container name from the base name and a contextId. + * MIRROR: Keep in sync with vendor/cortex/lib/blobContainerUtils.js getUserContainerName() + * Returns baseName unchanged if no contextId is provided. + * @param {string} baseName - Base container name (e.g. 'cortexfiles-local') + * @param {string} [contextId] - User/entity context ID + * @returns {string} Per-user container name + */ +export function getUserContainerName(baseName, contextId) { + if (!contextId) return baseName; + return buildContainerName(baseName, sanitizeContainerContextId(contextId)); +} + +/** + * Legacy compound-context container naming used before scoped contexts preserved + * separators. Keep this for storage compatibility when probing older blobs. + * @param {string} baseName - Base container name (e.g. 'cortexfiles-local') + * @param {string} [contextId] - Legacy compound context ID + * @returns {string} Legacy container name + */ +export function getLegacyUserContainerName(baseName, contextId) { + if (!contextId) return baseName; + return buildContainerName( + baseName, + sanitizeLegacyContainerContextId(contextId), + ); +} + +/** + * Return current and legacy-compatible container names for the same context ID. + * Current naming is listed first; legacy aliases are included only when distinct. + * @param {string} baseName - Base container name + * @param {string} [contextId] - Context ID + * @returns {string[]} Candidate container names + */ +export function getUserContainerNameCandidates(baseName, contextId) { + if (!contextId) return [baseName]; + const current = getUserContainerName(baseName, contextId); + const legacy = getLegacyUserContainerName(baseName, contextId); + return legacy === current ? [current] : [current, legacy]; +} diff --git a/helper-apps/cortex-file-handler/src/fileChunker.js b/helper-apps/cortex-file-handler/src/fileChunker.js index 4da77c11..430bb16e 100644 --- a/helper-apps/cortex-file-handler/src/fileChunker.js +++ b/helper-apps/cortex-file-handler/src/fileChunker.js @@ -140,6 +140,7 @@ async function splitMediaFile( inputPath, chunkDurationInSeconds = 500, requestId = uuidv4(), + chunkOverlapSeconds = 0, ) { let tempPath = null; let uniqueOutputPath = null; @@ -186,7 +187,14 @@ async function splitMediaFile( } const duration = metadata.format.duration; - const numChunks = Math.ceil((duration - 1) / chunkDurationInSeconds); + const numChunks = Math.max( + 1, + Math.ceil((duration - 1) / chunkDurationInSeconds), + ); + const overlapSeconds = Math.max( + 0, + Math.min(Number(chunkOverlapSeconds) || 0, chunkDurationInSeconds / 2), + ); console.log( `Processing ${numChunks} chunks of ${chunkDurationInSeconds} seconds each`, ); @@ -212,14 +220,23 @@ async function splitMediaFile( uniqueOutputPath, `chunk-${chunkIndex + 1}-${chunkBaseName}`, ); - const offset = chunkIndex * chunkDurationInSeconds; + const idealOffset = chunkIndex * chunkDurationInSeconds; + const offset = Math.max( + 0, + idealOffset - (chunkIndex > 0 ? overlapSeconds : 0), + ); + const chunkEnd = Math.min( + duration, + (chunkIndex + 1) * chunkDurationInSeconds, + ); + const chunkLength = Math.max(0.1, chunkEnd - offset); chunkBatch.push( processChunk( inputPath, outputFileName, offset, - chunkDurationInSeconds, + chunkLength, ) .then((result) => { chunkResults[chunkIndex] = result; // Store in correct position diff --git a/helper-apps/cortex-file-handler/src/index.js b/helper-apps/cortex-file-handler/src/index.js index 1c7fd85c..20093dbd 100644 --- a/helper-apps/cortex-file-handler/src/index.js +++ b/helper-apps/cortex-file-handler/src/index.js @@ -4,13 +4,20 @@ import path from "path"; import { v4 as uuidv4 } from "uuid"; import mime from "mime-types"; -import { DOC_EXTENSIONS, AZURITE_ACCOUNT_NAME } from "./constants.js"; +import { + DOC_EXTENSIONS, + AZURITE_ACCOUNT_NAME, + getDefaultContainerName, + getUserContainerName, + getUserContainerNameCandidates, +} from "./constants.js"; import { easyChunker } from "./docHelper.js"; import { downloadFile, splitMediaFile } from "./fileChunker.js"; import { ensureEncoded, ensureFileExtension, urlExists } from "./helper.js"; import { cleanupRedisFileStoreMap, getFileStoreMap, + getAllFilesForContext, publishRequestProgress, removeFromFileStoreMap, setFileStoreMap, @@ -18,15 +25,288 @@ import { } from "./redis.js"; import { FileConversionService } from "./services/FileConversionService.js"; import { StorageService } from "./services/storage/StorageService.js"; -import { uploadBlob, getMimeTypeFromUrl } from "./blobHandler.js"; -import { generateShortId } from "./utils/filenameUtils.js"; +import { + uploadBlob, + getMimeTypeFromUrl, + constructFolderPath, + sanitizeSubPath, + getScopedContainerOwnerId, + getScopedLogicalContextId, +} from "./blobHandler.js"; +import { StorageFactory } from "./services/storage/StorageFactory.js"; +import { generateShortId, sanitizeFilename } from "./utils/filenameUtils.js"; +import { sanitizeTargetBlobPath } from "./utils/targetBlobPathUtils.js"; import { redactContextId, redactSasToken, sanitizeForLogging } from "./utils/logSecurity.js"; - -// Hybrid cleanup approach: -// 1. Lazy cleanup: Check file existence when cache entries are accessed (in getFileStoreMap) -// 2. Age cleanup: Remove old entries every 100 requests to prevent cache bloat +import { + resolveHashRecordWithLegacyWorkspacePrivateFallback, + migrateHashRecordToScopedStorage, + resolveBlobPathWithLegacyFallback, +} from "./utils/legacyWorkspacePrivateResolver.js"; + +// Lazy cleanup remains in getFileStoreMap for entries whose backing files are +// actually gone. Age/container cleanup is opt-in because Redis hash records are +// needed to resolve legacy files whose blobs still exist. let requestCount = 0; +function isEnabled(value) { + return /^(1|true|yes)$/i.test(String(value || "")); +} + +/** + * Extract the container name from an Azure blob URL. + * Handles both real Azure URLs (/{container}/blob) and Azurite URLs (/devstoreaccount1/{container}/blob). + */ +function extractContainerFromUrl(url) { + try { + const urlObj = new URL(url); + let pathParts = urlObj.pathname.split('/').filter(p => p.length > 0); + // Azurite: /devstoreaccount1/{container}/blob → skip account name + if (pathParts[0] === AZURITE_ACCOUNT_NAME) { + pathParts = pathParts.slice(1); + } + return pathParts[0] || null; + } catch { return null; } +} + +/** + * Extract the blob name (everything after the container) from an Azure blob URL. + */ +function extractBlobNameFromUrl(url) { + try { + const urlObj = new URL(url); + const decodedPath = decodeURIComponent(urlObj.pathname); + let pathParts = decodedPath.split('/').filter(p => p.length > 0); + if (pathParts[0] === AZURITE_ACCOUNT_NAME) { + pathParts = pathParts.slice(1); + } + // First part is container, rest is blob name + return pathParts.length > 1 ? pathParts.slice(1).join('/') : null; + } catch { return null; } +} + +async function getScopedProvider({ + storageService, + resolvedContextId = null, + userId = null, + workspaceId = null, + appletId = null, + fileScope = null, +} = {}) { + const containerOwnerId = getScopedContainerOwnerId({ + contextId: resolvedContextId, + userId, + workspaceId, + appletId, + fileScope, + }); + const containerName = containerOwnerId + ? getUserContainerName(getDefaultContainerName(), containerOwnerId) + : null; + const useScopedAzureProvider = + containerName + && storageService.primaryProvider?.constructor?.name === "AzureStorageProvider"; + + return { + containerOwnerId, + containerName, + provider: useScopedAzureProvider + ? await StorageFactory.getInstance().getAzureProvider(containerName) + : storageService.primaryProvider, + }; +} + +function isNotFoundError(error) { + const message = error?.message || ""; + return error?.statusCode === 404 || /not found/i.test(message); +} + +function getLegacyScopedContainerNames(containerOwnerId = null) { + if (!containerOwnerId) { + return []; + } + + const [, ...legacyContainerNames] = getUserContainerNameCandidates( + getDefaultContainerName(), + containerOwnerId, + ); + return legacyContainerNames; +} + +function getListedFileKey(file = {}) { + return file.hash || file.name || file.url || file.filename || null; +} + +function mergeListedFiles(primaryFiles = [], fallbackFiles = []) { + const merged = [...primaryFiles]; + const seen = new Set(primaryFiles.map((file) => getListedFileKey(file)).filter(Boolean)); + + for (const file of fallbackFiles) { + const key = getListedFileKey(file); + if (key && seen.has(key)) { + continue; + } + if (key) { + seen.add(key); + } + merged.push(file); + } + + return merged; +} + +async function listAzureFolderIfContainerExists(provider, folderPath) { + if (!provider?.ensureInitialized) { + return []; + } + + try { + await provider.ensureInitialized(); + const containerClient = provider._containerClient; + if (!containerClient) { + return []; + } + + const prefix = folderPath === '' ? undefined : (folderPath.endsWith('/') ? folderPath : `${folderPath}/`); + const results = []; + + for await (const blob of containerClient.listBlobsFlat({ prefix })) { + const rawFilename = blob.name.split('/').pop(); + let filename; + try { + filename = decodeURIComponent(rawFilename); + } catch { + filename = rawFilename; + } + + const hashMatch = filename.match(/^([a-f0-9]+)_/i); + const blockBlobClient = containerClient.getBlockBlobClient(blob.name); + const sasToken = provider.generateShortLivedSASToken(blob.name, 60); + + results.push({ + name: blob.name, + filename: hashMatch ? filename.replace(/^[a-f0-9]+_/i, '') : filename, + hash: hashMatch ? hashMatch[1] : null, + lastModified: blob.properties.lastModified, + contentType: blob.properties.contentType, + size: blob.properties.contentLength, + url: `${blockBlobClient.url}?${sasToken}`, + }); + } + + return results; + } catch (error) { + if (isNotFoundError(error)) { + return []; + } + throw error; + } +} + +export async function listLegacyScopedFolderFiles(containerOwnerId, folderPath) { + const files = []; + for (const containerName of getLegacyScopedContainerNames(containerOwnerId)) { + try { + const provider = await StorageFactory.getInstance().getAzureProvider( + containerName, + ); + const legacyFiles = await listAzureFolderIfContainerExists( + provider, + folderPath, + ); + files.push(...legacyFiles); + } catch (error) { + if (/Missing Azure Storage connection string or container name/i.test(error?.message || "")) { + return files; + } + throw error; + } + } + return files; +} + +export async function resolveLegacyScopedBlobClient(containerOwnerId, blobPath) { + if (!containerOwnerId || !blobPath) { + return null; + } + + for (const containerName of getLegacyScopedContainerNames(containerOwnerId)) { + let provider; + try { + provider = await StorageFactory.getInstance().getAzureProvider( + containerName, + ); + } catch (error) { + if (/Missing Azure Storage connection string or container name/i.test(error?.message || "")) { + return null; + } + throw error; + } + + try { + await provider.ensureInitialized(); + const containerClient = provider._containerClient; + if (!containerClient) { + continue; + } + + const blockBlobClient = containerClient.getBlockBlobClient(blobPath); + if (await blockBlobClient.exists()) { + return { + containerName, + provider, + containerClient, + blockBlobClient, + }; + } + } catch (error) { + if (isNotFoundError(error)) { + continue; + } + throw error; + } + } + + return null; +} + +/** + * Delete up to `limit` empty per-user blob containers. + * Per-user containers follow the naming pattern `{baseName}-{userId}`. + * Fire-and-forget — errors are logged, never thrown. + */ +async function cullEmptyContainers(context, limit = 20) { + const { BlobServiceClient } = await import('@azure/storage-blob'); + const connStr = process.env.AZURE_STORAGE_CONNECTION_STRING; + if (!connStr) return; + + const baseName = getDefaultContainerName(); + const prefix = `${baseName}-`; + const blobService = BlobServiceClient.fromConnectionString(connStr); + + let deleted = 0; + for await (const container of blobService.listContainers({ prefix })) { + if (deleted >= limit) break; + try { + const client = blobService.getContainerClient(container.name); + const iter = client.listBlobsFlat().byPage({ maxPageSize: 1 }); + const page = await iter.next(); + const hasBlobs = page.value?.segment?.blobItems?.length > 0; + if (!hasBlobs) { + await client.delete(); + deleted++; + context.log(`Culled empty container: ${container.name}`); + } + } catch (err) { + if (err.statusCode !== 404) { + console.log(`Container cull error (${container.name}): ${err.message}`); + } + } + } + if (deleted > 0) { + context.log(`Container cull: deleted ${deleted} empty container(s)`); + } +} + /** * Lightweight age-based cleanup - removes old cache entries to prevent bloat * Only removes entries older than 7 days and only checks a small sample @@ -34,14 +314,23 @@ let requestCount = 0; */ async function cleanupInactive(context) { try { - // Only run age cleanup every 100 requests to avoid overhead requestCount++; - if (requestCount % 100 === 0) { + if ( + isEnabled(process.env.CFH_ENABLE_FILESTORE_AGE_CLEANUP) + && requestCount % 100 === 0 + ) { const cleaned = await cleanupRedisFileStoreMapAge(7, 10); // 7 days, max 10 entries if (cleaned.length > 0) { context.log(`Age cleanup: Removed ${cleaned.length} old cache entries`); } } + // Empty-container culling is opt-in; it can race with newly created upload containers. + if ( + isEnabled(process.env.CFH_ENABLE_EMPTY_CONTAINER_CULL) + && requestCount % 100 === 0 + ) { + cullEmptyContainers(context).catch(() => {}); + } } catch (error) { console.log("Error occurred during age-based cleanup:", error); } @@ -58,17 +347,17 @@ async function CortexFileHandler(context, req) { parsedBody = {}; } } - + // For GET requests, prioritize query string. For other methods, check body first, then query // Also check if parsedBody actually has content (not just empty object) const hasBodyContent = parsedBody && typeof parsedBody === 'object' && Object.keys(parsedBody).length > 0; const bodySource = hasBodyContent ? (parsedBody.params || parsedBody) : {}; const querySource = req.query || {}; - + // Merge sources: for GET, query takes priority; for others, body takes priority const isGet = req.method?.toLowerCase() === 'get'; const source = isGet ? { ...bodySource, ...querySource } : { ...querySource, ...bodySource }; - + const { uri, requestId, @@ -81,10 +370,35 @@ async function CortexFileHandler(context, req) { load, restore, setRetention, + rename, + newFilename, + filename: clientFilename, contextId, + chunkOverlapSeconds, + blobPath, + // Folder-based storage parameters + listFolder, + userId, + chatId, + workspaceId, + appletId, + fileScope, + subPath, } = source; - // Container parameter is ignored - always uses default container from env var - const resolvedContextId = contextId || null; + const logicalContextId = getScopedLogicalContextId({ + contextId: contextId || null, + userId, + workspaceId, + appletId, + fileScope, + }); + const storageOwnerId = getScopedContainerOwnerId({ + contextId: logicalContextId, + userId, + workspaceId, + appletId, + fileScope, + }); // Normalize boolean parameters const shouldSave = save === true || save === "true"; @@ -95,30 +409,48 @@ async function CortexFileHandler(context, req) { - const shouldSetRetention = setRetention === true || setRetention === "true" || + const shouldSetRetention = setRetention === true || setRetention === "true" || (req.query?.operation === "setRetention") || (parsedBody?.operation === "setRetention"); - - const operation = shouldSave - ? "save" - : shouldCheckHash - ? "checkHash" - : shouldClearHash - ? "clearHash" - : shouldSetRetention - ? "setRetention" - : shouldFetchRemote - ? "remoteFile" - : req.method.toLowerCase() === "delete" || - (req.query?.operation === "delete") || (parsedBody?.operation === "delete") - ? "delete" - : uri - ? DOC_EXTENSIONS.some((ext) => uri.toLowerCase().endsWith(ext)) - ? "document_processing" - : "media_chunking" - : "upload"; + const shouldRename = rename === true || rename === "true" || + (req.query?.operation === "rename") || (parsedBody?.operation === "rename"); + const shouldListFolder = listFolder === true || listFolder === "true" || + (req.query?.operation === "listFolder") || (parsedBody?.operation === "listFolder"); + + // Determine operation using explicit if-else chain + let operation; + if (shouldSave) { + operation = "save"; + } else if (shouldCheckHash) { + operation = "checkHash"; + } else if (shouldClearHash) { + operation = "clearHash"; + } else if (shouldSetRetention) { + operation = "setRetention"; + } else if (shouldRename) { + operation = "rename"; + } else if (shouldListFolder) { + operation = "listFolder"; + } else if (shouldFetchRemote) { + operation = "remoteFile"; + } else if (req.method.toLowerCase() === "delete" || + (req.query?.operation === "delete") || (parsedBody?.operation === "delete")) { + operation = "delete"; + } else if (uri) { + if (DOC_EXTENSIONS.some((ext) => uri.toLowerCase().endsWith(ext))) { + operation = "document_processing"; + } else { + operation = "media_chunking"; + } + } else if (blobPath && isGet) { + operation = "blobLookup"; + } else if (hash && isGet) { + operation = "hashLookup"; + } else { + operation = "upload"; + } context.log( - `Processing ${req.method} request - ${requestId ? `requestId: ${requestId}, ` : ""}${uri ? `uri: ${redactSasToken(uri)}, ` : ""}${hash ? `hash: ${hash}, ` : ""}${resolvedContextId ? `contextId: ${redactContextId(resolvedContextId)}, ` : ""}operation: ${operation}`, + `Processing ${req.method} request - ${requestId ? `requestId: ${requestId}, ` : ""}${uri ? `uri: ${redactSasToken(uri)}, ` : ""}${hash ? `hash: ${hash}, ` : ""}${blobPath ? `blobPath: ${blobPath}, ` : ""}${logicalContextId ? `contextId: ${redactContextId(logicalContextId)}, ` : ""}operation: ${operation}`, ); // Trigger lightweight age-based cleanup (runs every 100 requests) @@ -168,20 +500,25 @@ async function CortexFileHandler(context, req) { if (operation === "delete") { // Check both query string and body params for delete parameters // Handle both req.body.params.hash and req.body.hash formats - // Note: container is already extracted from source above (line 82), same as checkHash const deleteRequestId = req.query.requestId || parsedBody?.params?.requestId || parsedBody?.requestId || requestId; const deleteHash = req.query.hash || parsedBody?.params?.hash || parsedBody?.hash || hash; - + // If only hash is provided, delete single file by hash if (deleteHash && !deleteRequestId) { try { - // Container parameter is ignored - always uses default container from env var - const deleted = await storageService.deleteFileByHash(deleteHash, resolvedContextId); + const deleted = await storageService.deleteFileByHash(deleteHash, logicalContextId); + if (deleted.alreadyDeleted) { + context.res = { + status: 404, + body: `File with hash ${deleteHash} not found`, + }; + return; + } context.res = { status: 200, - body: { + body: { message: `File with hash ${deleteHash} deleted successfully`, - deleted + deleted }, }; return; @@ -193,22 +530,81 @@ async function CortexFileHandler(context, req) { return; } } - + + // Delete by blobPath: directly delete the blob from storage without Redis lookup + const deleteBlobPath = req.query.blobPath || parsedBody?.params?.blobPath || parsedBody?.blobPath || blobPath; + if (deleteBlobPath && !deleteRequestId) { + try { + const { provider } = await getScopedProvider({ + storageService, + resolvedContextId: logicalContextId, + userId, + workspaceId, + appletId, + fileScope, + }); + const { containerClient } = await provider.getBlobClient(); + const blockBlobClient = containerClient.getBlockBlobClient(deleteBlobPath); + await blockBlobClient.delete(); + context.log(`Deleted blob by blobPath: ${deleteBlobPath}`); + context.res = { + status: 200, + body: { + message: `File deleted successfully`, + blobPath: deleteBlobPath, + }, + }; + return; + } catch (error) { + if (isNotFoundError(error)) { + try { + const legacyBlob = await resolveLegacyScopedBlobClient( + logicalContextId || storageOwnerId, + deleteBlobPath, + ); + if (legacyBlob) { + await legacyBlob.blockBlobClient.delete(); + context.log(`Deleted legacy blob by blobPath: ${deleteBlobPath} (${legacyBlob.containerName})`); + context.res = { + status: 200, + body: { + message: `File deleted successfully`, + blobPath: deleteBlobPath, + }, + }; + return; + } + } catch (legacyError) { + context.res = { + status: legacyError.statusCode === 404 ? 404 : 500, + body: `Error deleting blob ${deleteBlobPath}: ${legacyError.message}`, + }; + return; + } + } + context.res = { + status: error.statusCode === 404 ? 404 : 500, + body: `Error deleting blob ${deleteBlobPath}: ${error.message}`, + }; + return; + } + } + // If requestId is provided, use the existing multi-file delete flow if (!deleteRequestId) { context.res = { status: 400, - body: "Please pass either a requestId or hash in the query string or request body", + body: "Please pass either a requestId, hash, or blobPath in the query string or request body", }; return; } // First, get the hash from the map if it exists if (deleteHash) { - const hashResult = await getFileStoreMap(deleteHash, false, resolvedContextId); + const hashResult = await getFileStoreMap(deleteHash, false, logicalContextId); if (hashResult) { - context.log(`Found hash in map for deletion: ${deleteHash}${resolvedContextId ? ` (contextId: ${redactContextId(resolvedContextId)})` : ""}`); - await removeFromFileStoreMap(deleteHash, resolvedContextId); + context.log(`Found hash in map for deletion: ${deleteHash}${logicalContextId ? ` (contextId: ${redactContextId(logicalContextId)})` : ""}`); + await removeFromFileStoreMap(deleteHash, logicalContextId); } } @@ -224,16 +620,17 @@ async function CortexFileHandler(context, req) { if (operation === "setRetention") { // Extract parameters from query string or body const fileHash = req.query.hash || parsedBody?.params?.hash || parsedBody?.hash || hash; + const fileBlobPath = req.query.blobPath || parsedBody?.params?.blobPath || parsedBody?.blobPath || blobPath; const retention = req.query.retention || parsedBody?.params?.retention || parsedBody?.retention; - - if (!fileHash) { + + if (!fileHash && !fileBlobPath) { context.res = { status: 400, - body: "Missing hash parameter. Please provide hash in query string or request body.", + body: "Missing identifier. Please provide hash or blobPath in query string or request body.", }; return; } - + if (!retention) { context.res = { status: 400, @@ -241,7 +638,7 @@ async function CortexFileHandler(context, req) { }; return; } - + // Validate retention value if (retention !== 'temporary' && retention !== 'permanent') { context.res = { @@ -251,17 +648,388 @@ async function CortexFileHandler(context, req) { return; } + // Prefer hash-based lookup (updates Redis + blob tags), fall back to blobPath (blob tags only) + if (fileHash) { + try { + const result = await storageService.setRetention(fileHash, retention, context, logicalContextId); + context.res = { + status: 200, + body: result, + }; + return; + } catch (error) { + // If hash lookup failed but we have blobPath, fall through to blobPath handler + if (!fileBlobPath) { + context.res = { + status: error.message.includes("not found") ? 404 : 500, + body: error.message, + }; + return; + } + context.log(`Hash-based setRetention failed for ${fileHash}, trying blobPath: ${fileBlobPath}`); + } + } + + // Set retention directly by blobPath (no Redis lookup needed) + if (fileBlobPath) { + try { + const { provider } = await getScopedProvider({ + storageService, + resolvedContextId: logicalContextId, + userId, + workspaceId, + appletId, + fileScope, + }); + if (provider && provider.updateBlobTags) { + await provider.updateBlobTags(fileBlobPath, retention); + context.log(`Set retention to ${retention} for blobPath: ${fileBlobPath}`); + context.res = { + status: 200, + body: { + message: `Retention set to ${retention}`, + blobPath: fileBlobPath, + retention, + }, + }; + return; + } + context.res = { + status: 500, + body: "Storage provider does not support blob tags", + }; + return; + } catch (error) { + if (isNotFoundError(error)) { + try { + const legacyBlob = await resolveLegacyScopedBlobClient( + logicalContextId || storageOwnerId, + fileBlobPath, + ); + if (legacyBlob?.provider?.updateBlobTags) { + await legacyBlob.provider.updateBlobTags(fileBlobPath, retention); + context.log(`Set retention to ${retention} for legacy blobPath: ${fileBlobPath} (${legacyBlob.containerName})`); + context.res = { + status: 200, + body: { + message: `Retention set to ${retention}`, + blobPath: fileBlobPath, + retention, + }, + }; + return; + } + } catch (legacyError) { + context.res = { + status: legacyError.statusCode === 404 ? 404 : 500, + body: `Error setting retention for ${fileBlobPath}: ${legacyError.message}`, + }; + return; + } + } + context.res = { + status: error.statusCode === 404 ? 404 : 500, + body: `Error setting retention for ${fileBlobPath}: ${error.message}`, + }; + return; + } + } + } + + // Rename a file (rename blob in cloud storage + update Redis) + if (operation === "rename") { + const fileHash = req.query.hash || parsedBody?.params?.hash || parsedBody?.hash || hash; + const fileBlobPath = req.query.blobPath || parsedBody?.params?.blobPath || parsedBody?.blobPath || blobPath; + const targetFilename = req.query.newFilename || parsedBody?.params?.newFilename || parsedBody?.newFilename || newFilename; + const targetBlobPath = req.query.targetBlobPath || parsedBody?.params?.targetBlobPath || parsedBody?.targetBlobPath; + const sanitizedTargetBlobPath = targetBlobPath + ? sanitizeTargetBlobPath(targetBlobPath) + : ""; + + if (!fileHash && !fileBlobPath) { + context.res = { + status: 400, + body: "Missing identifier. Please provide hash or blobPath in query string or request body.", + }; + return; + } + + if (!targetFilename || !targetFilename.trim()) { + context.res = { + status: 400, + body: "Missing newFilename parameter. Please provide newFilename in query string or request body.", + }; + return; + } + + if (targetBlobPath && !sanitizedTargetBlobPath) { + context.res = { + status: 400, + body: "Invalid targetBlobPath parameter.", + }; + return; + } + + const renameDirectlyByBlobPath = async () => { + try { + const { provider } = await getScopedProvider({ + storageService, + resolvedContextId: logicalContextId, + userId, + workspaceId, + appletId, + fileScope, + }); + + const sanitized = sanitizeFilename(targetFilename.trim()); + const newBlobName = sanitizedTargetBlobPath + || storageService._computeNewBlobName(fileBlobPath, sanitized); + + context.log(`Renaming blob by blobPath: ${fileBlobPath} → ${newBlobName}`); + const result = await provider.renameBlob(fileBlobPath, newBlobName); + + context.res = { + status: 200, + body: { + blobPath: newBlobName, + filename: sanitized, + url: result.url, + shortLivedUrl: result.shortLivedUrl || result.url, + message: `File renamed to "${targetFilename.trim()}"`, + }, + }; + return; + } catch (error) { + if (isNotFoundError(error)) { + try { + const legacyBlob = await resolveLegacyScopedBlobClient( + logicalContextId || storageOwnerId, + fileBlobPath, + ); + if (legacyBlob?.provider?.renameBlob) { + const sanitized = sanitizeFilename(targetFilename.trim()); + const newBlobName = sanitizedTargetBlobPath + || storageService._computeNewBlobName( + fileBlobPath, + sanitized, + ); + + context.log(`Renaming legacy blob by blobPath: ${fileBlobPath} → ${newBlobName} (${legacyBlob.containerName})`); + const result = await legacyBlob.provider.renameBlob( + fileBlobPath, + newBlobName, + ); + + context.res = { + status: 200, + body: { + blobPath: newBlobName, + filename: sanitized, + url: result.url, + shortLivedUrl: result.shortLivedUrl || result.url, + message: `File renamed to "${targetFilename.trim()}"`, + }, + }; + return; + } + } catch (legacyError) { + context.log(`Error renaming legacy blob by blobPath: ${legacyError.message}`); + context.res = { + status: legacyError.message.includes("not found") || legacyError.statusCode === 404 ? 404 : 500, + body: `Error renaming file: ${legacyError.message}`, + }; + return; + } + } + context.log(`Error renaming blob by blobPath: ${error.message}`); + context.res = { + status: error.message.includes("not found") || error.statusCode === 404 ? 404 : 500, + body: `Error renaming file: ${error.message}`, + }; + } + }; + + // Prefer blobPath when provided; hash is retained for legacy callers and Redis updates. + if (fileBlobPath) { + if (fileHash) { + try { + const result = await storageService.renameFile( + fileHash, + targetFilename, + context, + logicalContextId, + { + sourceBlobPath: fileBlobPath, + targetBlobPath: sanitizedTargetBlobPath, + }, + ); + + context.log(`Renamed blob ${fileBlobPath} to "${targetFilename.trim()}" via hash ${fileHash}${logicalContextId ? ` (contextId: ${redactContextId(logicalContextId)})` : ""}`); + + context.res = { + status: 200, + body: result, + }; + return; + } catch (error) { + if (!isNotFoundError(error)) { + context.log(`Error renaming file: ${error.message}`); + context.res = { + status: 500, + body: `Error renaming file: ${error.message}`, + }; + return; + } + context.log(`Hash-based rename failed for ${fileHash}, trying blobPath: ${fileBlobPath}`); + } + } + await renameDirectlyByBlobPath(); + return; + } + + // Legacy hash-only rename path. + if (fileHash) { + try { + const result = await storageService.renameFile( + fileHash, + targetFilename, + context, + logicalContextId, + { targetBlobPath: sanitizedTargetBlobPath }, + ); + + context.log(`Renamed file ${fileHash} to "${targetFilename.trim()}"${logicalContextId ? ` (contextId: ${redactContextId(logicalContextId)})` : ""}`); + + context.res = { + status: 200, + body: result, + }; + return; + } catch (error) { + context.log(`Error renaming file: ${error.message}`); + const status = error.message.includes("not found") ? 404 : 500; + context.res = { + status, + body: `Error renaming file: ${error.message}`, + }; + return; + } + } + } + + // List files in a folder (folder-based storage) + if (operation === "listFolder") { + // Construct folder path from provided parameters + let folderPath = constructFolderPath({ + userId, + chatId, + workspaceId, + appletId, + contextId: logicalContextId, + fileScope, + }); + + // Optional subPath appends a subdirectory within the fileScope folder. + if (subPath && folderPath !== null) { + const sanitizedSub = sanitizeSubPath(subPath); + if (sanitizedSub) { + folderPath = folderPath ? `${folderPath}/${sanitizedSub}` : sanitizedSub; + } + } + + if (folderPath === null) { + context.res = { + status: 400, + body: "Missing required parameters. Provide contextId or userId with optional chatId/workspaceId/appletId/fileScope, or workspaceId with fileScope='workspace-shared-legacy'", + }; + return; + } + try { - const result = await storageService.setRetention(fileHash, retention, context, resolvedContextId); + // Derive the correct per-user or per-workspace container + const { provider, containerOwnerId } = await getScopedProvider({ + storageService, + resolvedContextId: logicalContextId, + userId, + workspaceId, + appletId, + fileScope, + }); + + if (!provider || typeof provider.listFolder !== 'function') { + context.res = { + status: 500, + body: "Storage provider does not support folder listing", + }; + return; + } + + let files = await provider.listFolder(folderPath); + if (containerOwnerId) { + files = mergeListedFiles( + files, + await listLegacyScopedFolderFiles(containerOwnerId, folderPath), + ); + } + + // Enrich listing with hash, gcs, displayFilename, and permanent from Redis + const enrichContextId = logicalContextId || containerOwnerId || storageOwnerId; + if (enrichContextId) { + // Load all Redis records for this context to match files without hashes + const allRedisRecords = await getAllFilesForContext(enrichContextId); + + // Build a filename→{hash, record} lookup from Redis for matching hashless files + const redisFilenameMap = new Map(); + for (const [hash, record] of Object.entries(allRedisRecords)) { + if (record && record.filename) { + redisFilenameMap.set(record.filename.toLowerCase(), { hash, record }); + } + } + + for (const file of files) { + // If file has no hash from blob name, try to find it in Redis by filename + if (!file.hash && file.filename) { + const match = redisFilenameMap.get(file.filename.toLowerCase()); + if (match) { + file.hash = match.hash; + if (match.record.gcs) { + file.gcs = match.record.gcs; + } + } + } + + // Enrich files that have a hash (either from blob name or Redis match above) + if (file.hash) { + try { + const stored = allRedisRecords[file.hash] || await getFileStoreMap(file.hash, true, enrichContextId); + if (stored) { + if (stored.displayFilename) { + file.displayFilename = stored.displayFilename; + } + if (stored.gcs && !file.gcs) { + file.gcs = stored.gcs; + } + file.permanent = stored.permanent || false; + } + } catch { /* skip enrichment for this file */ } + } + } + } + context.res = { status: 200, - body: result, + body: { + folderPath, + files, + count: files.length + }, }; return; } catch (error) { + context.log(`Error listing folder: ${error.message}`); context.res = { - status: error.message.includes("not found") ? 404 : 500, - body: error.message, + status: 500, + body: `Error listing folder: ${error.message}`, }; return; } @@ -284,9 +1052,9 @@ async function CortexFileHandler(context, req) { // Check if file already exists (using hash or URL as the key) // Always respect contextId if provided, even for URL-based lookups - const exists = hash - ? await getFileStoreMap(hash, false, resolvedContextId) - : await getFileStoreMap(remoteUrl, false, resolvedContextId); + const exists = hash + ? await getFileStoreMap(hash, false, logicalContextId) + : await getFileStoreMap(remoteUrl, false, logicalContextId); if (exists) { context.res = { status: 200, @@ -294,26 +1062,85 @@ async function CortexFileHandler(context, req) { }; //update redis timestamp with current time if (hash) { - await setFileStoreMap(hash, exists, resolvedContextId); + await setFileStoreMap(hash, exists, logicalContextId); } else { - await setFileStoreMap(remoteUrl, exists, resolvedContextId); + await setFileStoreMap(remoteUrl, exists, logicalContextId); } return; } // Download the file first const urlObj = new URL(remoteUrl); - // Use LLM-friendly naming for temp files instead of original filename const fileExtension = path.extname(urlObj.pathname) || ".mp3"; - const shortId = generateShortId(); - const tempFileName = `${shortId}${fileExtension}`; + // Use client-provided filename when available (for folder-based storage); + // fall back to shortId-based name for legacy flat storage + const folderPath = constructFolderPath({ + userId, + chatId, + workspaceId, + appletId, + contextId: logicalContextId, + fileScope, + }); + const tempFileName = (folderPath && clientFilename) + ? sanitizeFilename(clientFilename) + : `${generateShortId()}${fileExtension}`; filename = path.join(os.tmpdir(), tempFileName); await downloadFile(remoteUrl, filename); - // For remote files, we don't need a requestId folder structure since it's just a single file - // Pass empty string to store the file directly in the root - // Container parameter is ignored - always uses default container from env var - const res = await storageService.uploadFile(context, filename, '', null, null); + const finalFilename = path.basename(filename); + const { provider, containerOwnerId } = await getScopedProvider({ + storageService, + resolvedContextId: logicalContextId, + userId, + workspaceId, + appletId, + fileScope, + }); + const fileStream = fs.createReadStream(filename); + // Prefer the content type from the remote server's HEAD response; + // uploadStream falls back to mime.lookup(filename) if null. + const remoteContentType = urlCheck.contentType || null; + const primaryUploadResult = await provider.uploadStream( + context, + finalFilename, + fileStream, + remoteContentType, + "temporary", + folderPath, + ); + + let backupUploadUrl = null; + if ( + storageService.backupProvider && + typeof storageService.backupProvider.uploadStream === "function" + ) { + // Prefix GCS backup path with the scoped container owner to preserve + // isolation in the shared bucket backend. + const gcsFolderPath = containerOwnerId + ? `${containerOwnerId}/${folderPath || ''}`.replace(/\/+$/, '') + : folderPath; + const backupStream = fs.createReadStream(filename); + backupUploadUrl = await storageService.backupProvider.uploadStream( + context, + finalFilename, + backupStream, + remoteContentType, + "temporary", + gcsFolderPath, + ); + } + + const primaryBlobName = + provider?.extractBlobNameFromUrl?.(primaryUploadResult.url) || + primaryUploadResult.blobName || + null; + const res = { + ...primaryUploadResult, + ...(primaryBlobName && { blobName: primaryBlobName }), + ...(primaryBlobName && { blobPath: primaryBlobName }), + ...(backupUploadUrl && { gcs: backupUploadUrl.url || backupUploadUrl }), + }; // All uploads default to temporary (permanent: false) to match file collection logic res.permanent = false; @@ -321,9 +1148,9 @@ async function CortexFileHandler(context, req) { //Update Redis (using hash or URL as the key) // Always respect contextId if provided, even for URL-based lookups if (hash) { - await setFileStoreMap(hash, res, resolvedContextId); + await setFileStoreMap(hash, res, logicalContextId); } else { - await setFileStoreMap(remoteUrl, res, resolvedContextId); + await setFileStoreMap(remoteUrl, res, logicalContextId); } // Return the file URL @@ -352,9 +1179,9 @@ async function CortexFileHandler(context, req) { if (hash && clearHash) { try { - const hashValue = await getFileStoreMap(hash, false, resolvedContextId); + const hashValue = await getFileStoreMap(hash, false, logicalContextId); if (hashValue) { - await removeFromFileStoreMap(hash, resolvedContextId); + await removeFromFileStoreMap(hash, logicalContextId); context.res = { status: 200, body: `Hash ${hash} removed`, @@ -376,10 +1203,65 @@ async function CortexFileHandler(context, req) { } if (hash && checkHash) { - let hashResult = await getFileStoreMap(hash, true, resolvedContextId); // Skip lazy cleanup to handle it ourselves + let mapContextId = logicalContextId || null; + let hashResult = await getFileStoreMap(hash, true, mapContextId); // Skip lazy cleanup to handle it ourselves + + // Self-healing fallback for old applet-private layout: + // if current context misses, probe legacy compound context and migrate. + if (!hashResult && logicalContextId) { + try { + const legacyRecord = + await resolveHashRecordWithLegacyWorkspacePrivateFallback({ + hash, + resolvedContextId: logicalContextId, + userId, + workspaceId, + fileScope, + getFileStoreMap, + }); + + if (legacyRecord?.hashResult) { + hashResult = legacyRecord.hashResult; + mapContextId = legacyRecord.sourceContextId || mapContextId; + context.log( + `Recovered hash from legacy context for self-heal: ${hash}${mapContextId ? ` (contextId: ${redactContextId(mapContextId)})` : ""}`, + ); + + try { + hashResult = await migrateHashRecordToScopedStorage({ + context, + hash, + hashResult, + sourceContextId: mapContextId, + resolvedContextId: logicalContextId, + userId, + chatId, + workspaceId, + appletId, + fileScope, + storageService, + setFileStoreMap, + removeFromFileStoreMap, + }); + mapContextId = logicalContextId; + context.log( + `Legacy hash self-healed into current context: ${hash} (contextId: ${redactContextId(logicalContextId)})`, + ); + } catch (migrationError) { + context.log( + `Legacy hash migration failed (continuing with legacy location): ${migrationError.message}`, + ); + } + } + } catch (legacyLookupError) { + context.log( + `Legacy hash lookup failed for ${hash}: ${legacyLookupError.message}`, + ); + } + } if (hashResult) { - context.log(`File exists in map: ${hash}${resolvedContextId ? ` (contextId: ${redactContextId(resolvedContextId)})` : ""}`); + context.log(`File exists in map: ${hash}${mapContextId ? ` (contextId: ${redactContextId(mapContextId)})` : ""}`); // Log the URL retrieved from Redis before checking existence context.log(`Checking existence of URL from Redis: ${redactSasToken(hashResult?.url || '')}`); @@ -398,7 +1280,7 @@ async function CortexFileHandler(context, req) { context.log( `File not found in any storage. Removing from map: ${hash}`, ); - await removeFromFileStoreMap(hash, resolvedContextId); + await removeFromFileStoreMap(hash, mapContextId); context.res = { status: 404, body: `Hash ${hash} not found in storage`, @@ -417,7 +1299,7 @@ async function CortexFileHandler(context, req) { } catch (error) { context.log(`Error restoring to GCS: ${error}`); // If restoration fails, remove the hash from the map - await removeFromFileStoreMap(hash, resolvedContextId); + await removeFromFileStoreMap(hash, mapContextId); context.res = { status: 404, body: `Hash ${hash} not found`, @@ -448,15 +1330,36 @@ async function CortexFileHandler(context, req) { // Download from GCS await storageService.downloadFile(hashResult.gcs, downloadedFile); - // Upload to primary storage - // Container parameter is ignored - always uses default container from env var - const res = await storageService.uploadFile( - context, - downloadedFile, - hash, - null, - null, - ); + // Restore to the ORIGINAL container and folder path (not the default container). + // Extract container name and blob path from the original URL stored in Redis. + let res; + const originalUrl = hashResult.url; + if (originalUrl && originalUrl.startsWith('http')) { + const containerName = storageService._extractContainerFromUrl(originalUrl); + const provider = containerName + ? await StorageFactory.getInstance().getAzureProvider(containerName) + : storageService.primaryProvider; + const originalBlobName = provider.extractBlobNameFromUrl(originalUrl); + + // Extract folder path and filename from the original blob name + const lastSlash = originalBlobName ? originalBlobName.lastIndexOf('/') : -1; + const folderPath = lastSlash >= 0 ? originalBlobName.substring(0, lastSlash) : null; + const originalFilePart = lastSlash >= 0 ? originalBlobName.substring(lastSlash + 1) : originalBlobName; + // Use the original filename (with hash prefix) for the restored blob + const filename = originalFilePart || hashResult.filename || path.basename(hashResult.gcs); + + const stream = fs.createReadStream(downloadedFile); + res = await provider.uploadStream(context, filename, stream, null, 'temporary', folderPath); + } else { + // Fallback: no original URL, restore to default container + res = await storageService.uploadFile( + context, + downloadedFile, + hash, + null, + null, + ); + } // Update the hash result with the new primary storage URL hashResult.url = res.url; @@ -475,7 +1378,7 @@ async function CortexFileHandler(context, req) { } catch (error) { console.error("Error restoring from GCS:", error); // If restoration fails, remove the hash from the map - await removeFromFileStoreMap(hash, resolvedContextId); + await removeFromFileStoreMap(hash, mapContextId); context.res = { status: 404, body: `Hash ${hash} not found`, @@ -493,7 +1396,7 @@ async function CortexFileHandler(context, req) { : false; if (!finalPrimaryCheck && !finalGCSCheck) { context.log(`Failed to restore file. Removing from map: ${hash}`); - await removeFromFileStoreMap(hash, resolvedContextId); + await removeFromFileStoreMap(hash, mapContextId); context.res = { status: 404, body: `Hash ${hash} not found`, @@ -516,11 +1419,177 @@ async function CortexFileHandler(context, req) { context.log(`Error extracting filename from URL: ${error.message}`); } } - - // Ensure hash is set if missing + + // Ensure hash/blobPath are set if missing if (!hashResult.hash) { hashResult.hash = hash; } + if (!hashResult.blobPath && hashResult.url) { + const inferredBlobPath = extractBlobNameFromUrl(hashResult.url); + if (inferredBlobPath) { + hashResult.blobPath = inferredBlobPath; + } + } + + // === Lazy migration: move old shared-container files to per-user container === + if (userId && hashResult.url) { + try { + const defaultContainer = getDefaultContainerName(); + const urlContainer = extractContainerFromUrl(hashResult.url); + const perUserContainer = getUserContainerName(defaultContainer, userId); + + // Only migrate if file is in old shared container, not already in per-user + if (urlContainer === defaultContainer && perUserContainer !== defaultContainer) { + context.log(`Migrating file from shared to per-user container: ${hash}`); + const factory = StorageFactory.getInstance(); + const perUserProvider = await factory.getAzureProvider(perUserContainer); + const { containerClient: destContainerClient } = await perUserProvider.getBlobClient(); + + const oldBlobName = extractBlobNameFromUrl(hashResult.url); + if (oldBlobName) { + // Strip "users/{id}/" prefix to get new blob name + const newBlobName = oldBlobName.replace(/^users\/[^/]+\//, ''); + + // Download from old URL (has valid SAS) and upload to per-user container + const downloadResp = await globalThis.fetch(hashResult.url); + if (!downloadResp.ok) throw new Error(`Download failed: ${downloadResp.status}`); + const blobBuffer = Buffer.from(await downloadResp.arrayBuffer()); + const contentType = downloadResp.headers.get('content-type'); + const destBlob = destContainerClient.getBlockBlobClient(newBlobName); + await destBlob.upload(blobBuffer, blobBuffer.length, { + blobHTTPHeaders: { + ...(contentType ? { blobContentType: contentType } : {}), + blobCacheControl: 'public, max-age=2592000, immutable', + }, + }); + + // Generate new long-lived SAS token and update URL + const newSasToken = perUserProvider.generateSASToken(destContainerClient, newBlobName); + hashResult.url = `${destBlob.url}?${newSasToken}`; + hashResult.blobPath = newBlobName; + hashResult.blobName = newBlobName; + + // Migrate converted file if it exists + if (hashResult.converted?.url) { + const oldConvertedBlob = extractBlobNameFromUrl(hashResult.converted.url); + if (oldConvertedBlob) { + const newConvertedBlob = oldConvertedBlob.replace(/^users\/[^/]+\//, ''); + const convResp = await globalThis.fetch(hashResult.converted.url); + if (convResp.ok) { + const convBuffer = Buffer.from(await convResp.arrayBuffer()); + const convContentType = convResp.headers.get('content-type'); + const destConvBlob = destContainerClient.getBlockBlobClient(newConvertedBlob); + await destConvBlob.upload(convBuffer, convBuffer.length, { + blobHTTPHeaders: { + ...(convContentType ? { blobContentType: convContentType } : {}), + blobCacheControl: 'public, max-age=2592000, immutable', + }, + }); + const convSasToken = perUserProvider.generateSASToken(destContainerClient, newConvertedBlob); + hashResult.converted.url = `${destConvBlob.url}?${convSasToken}`; + hashResult.converted.blobPath = newConvertedBlob; + hashResult.converted.blobName = newConvertedBlob; + } + } + } + + // Strip users/ prefix from GCS path if present + if (hashResult.gcs) { + hashResult.gcs = hashResult.gcs.replace(/\/users\/[^/]+\//, '/'); + } + + // Persist updated record to Redis + await setFileStoreMap(hash, hashResult, mapContextId); + context.log(`Migration complete for hash: ${hash}`); + } + } + } catch (migrationError) { + context.log(`Migration failed (using existing URL): ${migrationError.message}`); + } + } + + // === Copy blob to target folder if checkHash matched from a different folder === + if (hashResult.url) { + try { + const targetFolder = constructFolderPath({ + userId, + chatId, + workspaceId, + appletId, + contextId: logicalContextId, + fileScope, + }); + if (targetFolder !== null) { + const currentBlobName = extractBlobNameFromUrl(hashResult.url); + if (currentBlobName) { + // Extract the current folder and filename from the blob name + const lastSlash = currentBlobName.lastIndexOf('/'); + const currentFolder = lastSlash >= 0 ? currentBlobName.substring(0, lastSlash) : ''; + const filenameOnly = lastSlash >= 0 ? currentBlobName.substring(lastSlash + 1) : currentBlobName; + + // Normalize for comparison (empty string means root) + const normalizedTarget = targetFolder.replace(/^\/+|\/+$/g, ''); + + if (currentFolder !== normalizedTarget) { + context.log(`Copying blob from folder "${currentFolder}" to "${normalizedTarget}" for hash: ${hash}`); + + const urlContainer = extractContainerFromUrl(hashResult.url); + const factory = StorageFactory.getInstance(); + const provider = urlContainer + ? await factory.getAzureProvider(urlContainer) + : storageService.primaryProvider; + + await provider.ensureInitialized(); + const { containerClient } = await provider.getBlobClient(); + + const newBlobName = normalizedTarget ? `${normalizedTarget}/${filenameOnly}` : filenameOnly; + const srcBlobClient = containerClient.getBlockBlobClient(currentBlobName); + const destBlobClient = containerClient.getBlockBlobClient(newBlobName); + + // Copy using short-lived SAS for source auth + const sourceSas = provider.generateShortLivedSASToken(currentBlobName, 10); + const sourceUrl = `${srcBlobClient.url}?${sourceSas}`; + const copyPoller = await destBlobClient.beginCopyFromURL(sourceUrl); + await copyPoller.pollUntilDone(); + + // Generate new long-lived SAS for the copied blob + const newSasToken = provider.generateSASToken(newBlobName); + hashResult.url = `${destBlobClient.url}?${newSasToken}`; + hashResult.blobPath = newBlobName; + hashResult.blobName = newBlobName; + + // Copy converted file if it exists and is in a different folder too + if (hashResult.converted?.url) { + const convBlobName = extractBlobNameFromUrl(hashResult.converted.url); + if (convBlobName) { + const convLastSlash = convBlobName.lastIndexOf('/'); + const convFilename = convLastSlash >= 0 ? convBlobName.substring(convLastSlash + 1) : convBlobName; + const newConvBlobName = normalizedTarget ? `${normalizedTarget}/${convFilename}` : convFilename; + + const srcConvClient = containerClient.getBlockBlobClient(convBlobName); + const destConvClient = containerClient.getBlockBlobClient(newConvBlobName); + const convSas = provider.generateShortLivedSASToken(convBlobName, 10); + const convSourceUrl = `${srcConvClient.url}?${convSas}`; + const convPoller = await destConvClient.beginCopyFromURL(convSourceUrl); + await convPoller.pollUntilDone(); + + const convSasToken = provider.generateSASToken(newConvBlobName); + hashResult.converted.url = `${destConvClient.url}?${convSasToken}`; + hashResult.converted.blobPath = newConvBlobName; + hashResult.converted.blobName = newConvBlobName; + } + } + + // Persist updated record to Redis + await setFileStoreMap(hash, hashResult, mapContextId); + context.log(`Folder copy complete for hash: ${hash}`); + } + } + } + } catch (folderCopyError) { + context.log(`Folder copy failed (using existing URL): ${folderCopyError.message}`); + } + } // Create the response object const response = { @@ -529,9 +1598,10 @@ async function CortexFileHandler(context, req) { url: hashResult.url, gcs: hashResult.gcs, hash: hashResult.hash || hash, + ...(hashResult.blobPath ? { blobPath: hashResult.blobPath } : {}), timestamp: new Date().toISOString(), }; - + // Include displayFilename if it exists in Redis record if (hashResult.displayFilename) { response.displayFilename = hashResult.displayFilename; @@ -539,7 +1609,6 @@ async function CortexFileHandler(context, req) { // Ensure converted version exists and is synced across storage providers try { - // Container parameter is ignored - always uses default container from env var hashResult = await conversionService.ensureConvertedVersion( hashResult, requestId, @@ -557,21 +1626,24 @@ async function CortexFileHandler(context, req) { // Helper function to generate short-lived URL for a given URL const generateShortLivedUrlForUrl = async (urlToProcess) => { if (!urlToProcess) return null; - + try { // Extract blob name from the URL to generate new SAS token let blobName; try { const url = new URL(urlToProcess); let path = url.pathname.substring(1); - + // For Azurite URLs, the path includes account name: devstoreaccount1/container/blob // For real Azure URLs, the path is: container/blob if (path.startsWith(`${AZURITE_ACCOUNT_NAME}/`)) { path = path.substring(`${AZURITE_ACCOUNT_NAME}/`.length); } - - const pathSegments = path.split('/').filter(segment => segment.length > 0); + + // Decode each segment so double-encoded names (e.g. %2520 → %20) + // resolve to the actual blob name used in Azure storage. + const pathSegments = path.split('/').filter(segment => segment.length > 0) + .map(s => decodeURIComponent(s)); if (pathSegments.length >= 2) { blobName = pathSegments.slice(1).join('/'); } else if (pathSegments.length === 1) { @@ -583,26 +1655,35 @@ async function CortexFileHandler(context, req) { } if (blobName) { - const provider = storageService.primaryProvider; - + // Use correct provider based on URL container + const urlContainer = extractContainerFromUrl(urlToProcess); + const defaultContainer = getDefaultContainerName(); + let provider; + if (urlContainer && urlContainer !== defaultContainer) { + provider = await StorageFactory.getInstance().getAzureProvider(urlContainer); + } else { + provider = storageService.primaryProvider; + } + if (provider && provider.generateShortLivedSASToken) { - const blobClientResult = await provider.getBlobClient(); - const containerClient = blobClientResult.containerClient; - + await provider.ensureInitialized(); + const sasToken = provider.generateShortLivedSASToken( - containerClient, - blobName, + blobName, shortLivedDuration ); - - const baseUrl = urlToProcess.split('?')[0]; - return `${baseUrl}?${sasToken}`; + + // Build URL from the blob client so path encoding matches + // the decoded blobName used for SAS generation. + const { containerClient } = await provider.getBlobClient(); + const blockBlobClient = containerClient.getBlockBlobClient(blobName); + return `${blockBlobClient.url}?${sasToken}`; } } } catch (error) { context.log(`Error generating short-lived URL: ${error}`); } - + return null; }; @@ -636,18 +1717,21 @@ async function CortexFileHandler(context, req) { // Attach converted info to response if present (include shortLivedUrl in response only) if (hashResult.converted) { + const convertedBlobPath = hashResult.converted.blobPath + || extractBlobNameFromUrl(hashResult.converted.url || ""); response.converted = { url: hashResult.converted.url, shortLivedUrl: convertedShortLivedUrl || hashResult.converted.url, gcs: hashResult.converted.gcs, mimeType: hashResult.converted.mimeType || null, + ...(convertedBlobPath ? { blobPath: convertedBlobPath } : {}), }; } // Update redis timestamp with current time // Note: setFileStoreMap will remove shortLivedUrl fields before storing // hashResult has already been enriched with filename/hash above if missing - await setFileStoreMap(hash, hashResult, resolvedContextId); + await setFileStoreMap(hash, hashResult, mapContextId); context.res = { status: 200, @@ -657,7 +1741,7 @@ async function CortexFileHandler(context, req) { } catch (error) { context.log(`Error checking file existence: ${error}`); // If there's an error checking file existence, remove the hash from the map - await removeFromFileStoreMap(hash, resolvedContextId); + await removeFromFileStoreMap(hash, mapContextId); context.res = { status: 404, body: `Hash ${hash} not found`, @@ -666,11 +1750,168 @@ async function CortexFileHandler(context, req) { } } - context.res = { - status: 404, - body: `Hash ${hash} not found`, - }; - return; + // If blobPath is available, fall through to blobPath-based lookup + // instead of returning 404 — the file may still exist in storage + // even though its hash expired from Redis. + if (!blobPath) { + context.res = { + status: 404, + body: `Hash ${hash} not found`, + }; + return; + } + + try { + const legacyBlobResult = await resolveBlobPathWithLegacyFallback({ + context, + hash, + blobPath, + resolvedContextId: logicalContextId, + userId, + chatId, + workspaceId, + appletId, + fileScope, + storageService, + setFileStoreMap, + }); + if (legacyBlobResult?.url) { + context.res = { + status: 200, + body: { + url: legacyBlobResult.url, + shortLivedUrl: legacyBlobResult.shortLivedUrl || legacyBlobResult.url, + hash: legacyBlobResult.hash || hash, + blobPath: legacyBlobResult.blobPath || blobPath, + filename: legacyBlobResult.filename || null, + message: "File found by legacy blobPath fallback", + }, + }; + return; + } + } catch (legacyBlobError) { + context.log(`Legacy blobPath fallback failed for ${blobPath}: ${legacyBlobError.message}`); + } + + context.log(`Hash ${hash} not found in Redis, falling back to blobPath: ${blobPath}`); + } + + // Handle blobPath-based lookups: generate a short-lived SAS URL directly + // from the blob path, without needing a hash in Redis. + if (blobPath) { + try { + const { provider } = await getScopedProvider({ + storageService, + resolvedContextId: logicalContextId, + userId, + workspaceId, + appletId, + fileScope, + }); + + // Azure storage: use SDK to check existence and generate SAS token + if (provider && provider.getBlobClient && provider.generateShortLivedSASToken) { + const { containerClient } = await provider.getBlobClient(); + const blockBlobClient = containerClient.getBlockBlobClient(blobPath); + + const exists = await blockBlobClient.exists(); + if (exists) { + const sasToken = provider.generateShortLivedSASToken(blobPath, shortLivedDuration); + const shortLivedUrl = `${blockBlobClient.url}?${sasToken}`; + let gcsUrl = null; + + try { + const ensuredFile = await storageService.ensureGCSUpload(context, { + url: shortLivedUrl, + blobName: blobPath, + }); + gcsUrl = ensuredFile?.gcs || null; + } catch (ensureGcsError) { + context.log( + `Warning: Could not ensure GCS backup for blobPath ${blobPath}: ${ensureGcsError.message}`, + ); + } + + context.log(`Generated short-lived URL for blobPath: ${blobPath} (expires in ${shortLivedDuration} minutes)`); + context.res = { + status: 200, + body: { + url: shortLivedUrl, + shortLivedUrl: shortLivedUrl, + ...(gcsUrl ? { gcs: gcsUrl } : {}), + ...(hash ? { hash } : {}), + blobPath, + expiresInMinutes: shortLivedDuration, + message: "File found by blobPath", + }, + }; + return; + } + } else if (provider && provider.fileExists) { + // Local/other storage: check if a file with this path exists + const localUrl = `http://localhost:${process.env.PORT || 7071}/files/${blobPath}`; + const exists = await provider.fileExists(localUrl); + if (exists) { + context.log(`Found local file for blobPath: ${blobPath}`); + context.res = { + status: 200, + body: { + url: localUrl, + shortLivedUrl: localUrl, + message: "File found by blobPath", + }, + }; + return; + } + } + + try { + const legacyBlobResult = await resolveBlobPathWithLegacyFallback({ + context, + hash, + blobPath, + resolvedContextId: logicalContextId, + userId, + chatId, + workspaceId, + appletId, + fileScope, + storageService, + setFileStoreMap, + }); + if (legacyBlobResult?.url) { + context.res = { + status: 200, + body: { + url: legacyBlobResult.shortLivedUrl || legacyBlobResult.url, + shortLivedUrl: legacyBlobResult.shortLivedUrl || legacyBlobResult.url, + ...(legacyBlobResult.hash ? { hash: legacyBlobResult.hash } : {}), + blobPath: legacyBlobResult.blobPath || blobPath, + filename: legacyBlobResult.filename || null, + expiresInMinutes: shortLivedDuration, + message: "File found by legacy blobPath fallback", + }, + }; + return; + } + } catch (legacyBlobError) { + context.log(`Legacy blobPath fallback failed for ${blobPath}: ${legacyBlobError.message}`); + } + + context.log(`Blob not found for blobPath: ${blobPath}`); + context.res = { + status: 404, + body: `Blob not found: ${blobPath}`, + }; + return; + } catch (error) { + context.log(`Error looking up blobPath ${blobPath}: ${error}`); + context.res = { + status: 404, + body: `Blob not found: ${blobPath}`, + }; + return; + } } if (req.method.toLowerCase() === "post") { @@ -679,11 +1920,21 @@ async function CortexFileHandler(context, req) { storageService.primaryProvider.constructor.name === "LocalStorageProvider"; // Use uploadBlob to handle multipart/form-data - // Container parameter is ignored - always uses default container from env var const result = await uploadBlob(context, req, saveToLocal, null, hash); if (result?.hash && context?.res?.body) { - // Use contextId from result (extracted from form fields) or from resolvedContextId (query/body) - const uploadContextId = result.contextId || resolvedContextId; + // Use the explicit scoped context when available, otherwise derive it + // from the folder-storage routing inputs so uploads and lookups share + // the same Redis namespace. + const uploadContextId = + result.contextId + || getScopedLogicalContextId({ + contextId: null, + userId: result.userId || null, + workspaceId: result.workspaceId || null, + appletId: result.appletId || null, + fileScope: result.fileScope || null, + }) + || logicalContextId; // Store contextId alongside the entry for debugging/traceability if (uploadContextId && typeof context.res.body === "object" && context.res.body) { context.res.body.contextId = uploadContextId; @@ -750,7 +2001,6 @@ async function CortexFileHandler(context, req) { } // Save the converted file - // Container parameter is ignored - always uses default container from env var const convertedSaveResult = await conversionService._saveConvertedFile( conversion.convertedPath, @@ -768,7 +2018,6 @@ async function CortexFileHandler(context, req) { }; } else { // File doesn't need conversion, save the original file - // Container parameter is ignored - always uses default container from env var const saveResult = await conversionService._saveConvertedFile( downloadedFile, requestId, @@ -830,7 +2079,7 @@ async function CortexFileHandler(context, req) { } } else { const { chunkPromises, chunkOffsets, uniqueOutputPath, chunkBaseName } = - await splitMediaFile(file); + await splitMediaFile(file, 500, requestId, chunkOverlapSeconds); numberOfChunks = chunkPromises.length; // for progress reporting totalCount += chunkPromises.length * 4; // 4 steps for each chunk (download and upload) @@ -848,7 +2097,6 @@ async function CortexFileHandler(context, req) { const chunkPath = chunks[index]; // Use the same base filename for all chunks to ensure consistency const chunkFilename = `chunk-${index + 1}-${chunkBaseName}`; - // Container parameter is ignored - always uses default container from env var const chunkResult = await storageService.uploadFile( context, chunkPath, diff --git a/helper-apps/cortex-file-handler/src/redis.js b/helper-apps/cortex-file-handler/src/redis.js index b19d33c4..7abe22a1 100644 --- a/helper-apps/cortex-file-handler/src/redis.js +++ b/helper-apps/cortex-file-handler/src/redis.js @@ -14,9 +14,15 @@ const createMockClient = () => { const store = new Map(); const hashMap = new Map(); const locks = new Map(); // For lock simulation - + return { connected: false, + // Reset all in-memory state (used between test files to prevent leakage) + _reset() { + store.clear(); + hashMap.clear(); + locks.clear(); + }, async connect() { return Promise.resolve(); }, async publish() { return Promise.resolve(); }, async hgetall(hashName) { @@ -141,33 +147,7 @@ if (connectionString && process.env.NODE_ENV !== 'test') { const channel = "requestProgress"; -const connectClient = async () => { - // ioredis connects automatically; this function is kept for backwards - // compatibility and for the mock client. - try { - // Mock client uses `connected`; ioredis uses `status`. - if (typeof client?.connected === "boolean") { - if (!client.connected && typeof client.connect === "function") { - await client.connect(); - } - return; - } - - // ioredis states: "wait" | "connecting" | "connect" | "ready" | "close" | "end" - if (client?.status && client.status !== "ready") { - // If the caller explicitly wants to ensure connectivity, we can ping. - // If Redis is down, ping will throw and we handle it. - await client.ping(); - } - } catch (error) { - console.error( - `[redis] Not ready (status=${client?.status || "unknown"}): ${error?.message || error}`, - ); - } -}; - const publishRequestProgress = async (data) => { - // await connectClient(); try { const message = JSON.stringify(data); console.log(`Publishing message ${message} to channel ${channel}`); @@ -364,10 +344,10 @@ const getFileStoreMap = async (hash, skipLazyCleanup = false, contextId = null) // Remove stale entry if both primary and backup are missing // Need to extract contextId from the key if it was scoped if (shouldRemove) { - // For lazy cleanup, we don't have contextId, so try unscoped first - // If the key was scoped, we'd need contextId, but lazy cleanup doesn't have it - // So we'll just try to remove from unscoped map - await removeFromFileStoreMap(hash, null); + // Use contextId if available (passed to getFileStoreMap), otherwise + // fall back to unscoped map. Context-scoped entries found without a + // contextId cannot be cleaned up here. + await removeFromFileStoreMap(hash, contextId || null); console.log( `Lazy cleanup: Removed stale cache entry for hash ${hash}`, ); @@ -524,9 +504,9 @@ const acquireLock = async (lockKey, ttlSeconds = 300) => { return result === "OK"; } catch (error) { console.error(`Error acquiring lock for ${lockKey}:`, error); - // In case of error, allow operation to proceed (fail open) - // This prevents Redis issues from blocking operations - return true; + // Fail closed: callers must handle the case where the lock was NOT acquired + // rather than proceeding unsafely without mutual exclusion + return false; } }; @@ -547,7 +527,6 @@ const releaseLock = async (lockKey) => { export { publishRequestProgress, - connectClient, setFileStoreMap, getFileStoreMap, removeFromFileStoreMap, diff --git a/helper-apps/cortex-file-handler/src/services/ConversionService.js b/helper-apps/cortex-file-handler/src/services/ConversionService.js index f138b00d..e46ee1f1 100644 --- a/helper-apps/cortex-file-handler/src/services/ConversionService.js +++ b/helper-apps/cortex-file-handler/src/services/ConversionService.js @@ -106,7 +106,7 @@ export class ConversionService { * @returns {Promise} - Updated file info with conversion if needed */ async ensureConvertedVersion(fileInfo, requestId) { - const { url, gcs } = fileInfo; + const { url, gcs, folderPath } = fileInfo; // Remove any query parameters before extension check const extension = path.extname(url.split("?")[0]).toLowerCase(); @@ -167,6 +167,7 @@ export class ConversionService { conversion.convertedPath, requestId, null, + folderPath, ); if (!convertedSaveResult) { throw new Error("Failed to save converted file to primary storage"); @@ -178,6 +179,8 @@ export class ConversionService { gcsUrl = await this._uploadChunkToGCS( conversion.convertedPath, requestId, + null, + folderPath, ); } @@ -377,11 +380,11 @@ export class ConversionService { throw new Error("Method _downloadFile must be implemented"); } - async _saveConvertedFile(filePath, requestId, filename = null) { + async _saveConvertedFile(filePath, requestId, filename = null, folderPath = null) { throw new Error("Method _saveConvertedFile must be implemented"); } - async _uploadChunkToGCS(filePath, requestId) { + async _uploadChunkToGCS(filePath, requestId, filename = null, folderPath = null) { throw new Error("Method _uploadChunkToGCS must be implemented"); } diff --git a/helper-apps/cortex-file-handler/src/services/FileConversionService.js b/helper-apps/cortex-file-handler/src/services/FileConversionService.js index a941f857..f9bb8daa 100644 --- a/helper-apps/cortex-file-handler/src/services/FileConversionService.js +++ b/helper-apps/cortex-file-handler/src/services/FileConversionService.js @@ -1,3 +1,7 @@ +import { createReadStream } from "fs"; +import path from "path"; +import mime from "mime-types"; + import { ConversionService } from "./ConversionService.js"; import { getFileStoreMap, setFileStoreMap } from "../redis.js"; import { urlExists } from "../helper.js"; @@ -34,24 +38,34 @@ export class FileConversionService extends ConversionService { return downloadFile(url, destination); } - async _saveConvertedFile(filePath, requestId, filename = null) { + async _saveConvertedFile(filePath, requestId, filename = null, folderPath = null) { // Generate a fallback requestId if none supplied (e.g. during checkHash calls) const reqId = requestId || uuidv4(); let fileUrl; if (this.useAzure) { - // Container parameter is ignored - always uses default container from env var const provider = await this.storageFactory.getAzureProvider(); - const result = await provider.uploadFile({}, filePath, reqId, null, filename); - fileUrl = result.url; + if (folderPath) { + // Use uploadStream which supports folderPath to store converted file + // next to the original in the user's folder + const uploadName = filename || path.basename(filePath); + const stream = createReadStream(filePath); + const contentType = mime.lookup(uploadName) || null; + const result = await provider.uploadStream({}, uploadName, stream, contentType, 'temporary', folderPath); + fileUrl = result.url; + } else { + // Container parameter is ignored - always uses default container from env var + const result = await provider.uploadFile({}, filePath, reqId, null, filename); + fileUrl = result.url; + } } else { fileUrl = await moveFileToPublicFolder(filePath, reqId); } return { url: fileUrl }; } - async _uploadChunkToGCS(filePath, requestId, filename = null) { - return uploadChunkToGCS(filePath, requestId, filename); + async _uploadChunkToGCS(filePath, requestId, filename = null, folderPath = null) { + return uploadChunkToGCS(filePath, requestId, filename, folderPath); } _isGCSConfigured() { diff --git a/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js b/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js index fd6dc871..396546a5 100644 --- a/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js +++ b/helper-apps/cortex-file-handler/src/services/storage/AzureStorageProvider.js @@ -27,9 +27,29 @@ export class AzureStorageProvider extends StorageProvider { this.connectionString = connectionString; this.containerName = containerName; this.sasTokenLifeDays = process.env.SAS_TOKEN_LIFE_DAYS || 30; + this._containerEnsured = false; + + // Cached clients — lazily initialized by _doInitialize() + this._blobServiceClient = null; + this._containerClient = null; + this._sharedKeyCredential = null; + this._initPromise = null; + } + + /** + * Lazy one-time initialization: creates BlobServiceClient, checks service + * version, creates containerClient, and caches StorageSharedKeyCredential. + * Uses promise coalescence so concurrent callers share the same init. + */ + async ensureInitialized() { + if (this._sharedKeyCredential) return; // already done + if (!this._initPromise) { + this._initPromise = this._doInitialize(); + } + await this._initPromise; } - async getBlobClient() { + async _doInitialize() { const blobServiceClient = BlobServiceClient.fromConnectionString( this.connectionString, ); @@ -44,35 +64,87 @@ export class AzureStorageProvider extends StorageProvider { const containerClient = blobServiceClient.getContainerClient( this.containerName, ); - return { blobServiceClient, containerClient }; - } - generateSASToken(containerClient, blobName, options = {}) { - // Handle Azurite (development storage) credentials + // Extract and cache the shared key credential let accountName, accountKey; - - // Note: Debug logging removed for production - if (containerClient.credential && containerClient.credential.accountName) { - // Regular Azure Storage credentials accountName = containerClient.credential.accountName; - - // Handle Buffer case (Azurite) vs string case (real Azure) if (Buffer.isBuffer(containerClient.credential.accountKey)) { accountKey = containerClient.credential.accountKey.toString('base64'); } else { accountKey = containerClient.credential.accountKey; } } else { - // Azurite development storage fallback - accountName = AZURITE_ACCOUNT_NAME; - accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + // Azurite development storage fallback — only if explicitly detected + const isAzurite = process.env.AZURITE_ACCOUNTS + || process.env.AZURE_STORAGE_EMULATOR + || this.connectionString.includes(AZURITE_ACCOUNT_NAME); + + if (isAzurite) { + accountName = AZURITE_ACCOUNT_NAME; + // Well-known default Azurite development key (publicly documented) + accountKey = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw=="; + } else { + throw new Error( + "Could not extract Azure Storage credentials from the connection string and Azurite was not detected. " + + "Set AZURITE_ACCOUNTS or AZURE_STORAGE_EMULATOR for local development, or provide a valid connection string.", + ); + } } - - const sharedKeyCredential = new StorageSharedKeyCredential( + + this._blobServiceClient = blobServiceClient; + this._containerClient = containerClient; + this._sharedKeyCredential = new StorageSharedKeyCredential( accountName, accountKey, ); + } + + async getBlobClient({ createContainer = true } = {}) { + await this.ensureInitialized(); + + // Create container if it doesn't exist (only checked once per instance). + // Only cache success — a failed create must retry on the next call so a + // transient error doesn't permanently poison this provider instance. + if (createContainer && !this._containerEnsured) { + try { + await this._containerClient.createIfNotExists(); + this._containerEnsured = true; + } catch (e) { + // 409 = already exists, which is fine + if (e.statusCode === 409) { + this._containerEnsured = true; + } else { + console.error(`Failed to ensure container ${this.containerName}: ${e.message}`); + throw e; + } + } + } + + return { blobServiceClient: this._blobServiceClient, containerClient: this._containerClient }; + } + + /** + * Generate a SAS token for a blob. + * Dual-signature for backward compat: + * New: generateSASToken(blobName, options?) + * Old: generateSASToken(containerClient, blobName, options?) — containerClient is ignored + */ + generateSASToken(firstArg, secondArg, thirdArg) { + let blobName, options; + if (typeof firstArg === 'string') { + // New signature: generateSASToken(blobName, options?) + blobName = firstArg; + options = secondArg || {}; + } else { + // Old signature: generateSASToken(containerClient, blobName, options?) + blobName = secondArg; + options = thirdArg || {}; + } + + if (!this._sharedKeyCredential) { + throw new Error('AzureStorageProvider not initialized — call ensureInitialized() or getBlobClient() first'); + } // Support custom duration: minutes, hours, or fall back to default days let expirationTime; @@ -90,7 +162,7 @@ export class AzureStorageProvider extends StorageProvider { } const sasOptions = { - containerName: containerClient.containerName, + containerName: this.containerName, blobName: blobName, permissions: options.permissions || "r", startsOn: new Date(), @@ -99,12 +171,23 @@ export class AzureStorageProvider extends StorageProvider { return generateBlobSASQueryParameters( sasOptions, - sharedKeyCredential, + this._sharedKeyCredential, ).toString(); } - generateShortLivedSASToken(containerClient, blobName, minutes = 5) { - return this.generateSASToken(containerClient, blobName, { minutes }); + /** + * Generate a short-lived SAS token. + * Dual-signature for backward compat: + * New: generateShortLivedSASToken(blobName, minutes?) + * Old: generateShortLivedSASToken(containerClient, blobName, minutes?) — containerClient is ignored + */ + generateShortLivedSASToken(firstArg, secondArg, thirdArg) { + if (typeof firstArg === 'string') { + // New signature: generateShortLivedSASToken(blobName, minutes?) + return this.generateSASToken(firstArg, { minutes: secondArg || 5 }); + } + // Old signature: generateShortLivedSASToken(containerClient, blobName, minutes?) + return this.generateSASToken(secondArg, { minutes: thirdArg || 5 }); } async uploadFile(context, filePath, requestId, hash = null, filename = null, retention = 'temporary') { @@ -181,10 +264,16 @@ export class AzureStorageProvider extends StorageProvider { }; } - async uploadStream(context, encodedFilename, stream, providedContentType = null, retention = 'temporary') { + async uploadStream(context, encodedFilename, stream, providedContentType = null, retention = 'temporary', folderPath = null) { const { containerClient } = await this.getBlobClient(); let contentType = providedContentType || mime.lookup(encodedFilename); + // mime-types@3 maps .mp4 to 'application/mp4' (IANA-registered) which + // browsers don't play inline. Override to the widely-supported type. + if (contentType === 'application/mp4') { + contentType = 'video/mp4'; + } + // For text MIME types, ensure charset=utf-8 is included if not already present if (contentType && this.isTextMimeType(contentType)) { if (!contentType.includes('charset=')) { @@ -192,9 +281,20 @@ export class AzureStorageProvider extends StorageProvider { } } - // Normalize the blob name: sanitizeFilename decodes, cleans, then we encode for Azure + // Normalize the blob name: sanitizeFilename decodes and cleans. + // Do NOT encodeURIComponent — Azure SDK handles URL-encoding internally + // when constructing blockBlobClient.url. Encoding here would double-encode + // (e.g., spaces become %20 in the blob name, then %2520 in the URL). let blobName = sanitizeFilename(encodedFilename); - blobName = encodeURIComponent(blobName); + + // If folderPath is provided, prepend it to create folder hierarchy + if (folderPath) { + // Normalize folder path: remove leading/trailing slashes + const normalizedFolder = folderPath.replace(/^\/+|\/+$/g, ''); + if (normalizedFolder) { + blobName = `${normalizedFolder}/${blobName}`; + } + } // Validate blobName is not empty if (!blobName || blobName.trim().length === 0) { @@ -218,16 +318,42 @@ export class AzureStorageProvider extends StorageProvider { blockSize: 8 * 1024 * 1024, }; - const blockBlobClient = containerClient.getBlockBlobClient(blobName); + let activeContainerClient = containerClient; + let blockBlobClient = activeContainerClient.getBlockBlobClient(blobName); if (context.log) { context.log(`Uploading to Azure... ${blobName}`); context.log(`Setting content-type: ${contentType}`); } + + try { + await blockBlobClient.uploadStream(stream, undefined, undefined, options); + } catch (error) { + const code = error?.code || error?.details?.errorCode; + const missingContainer = error?.statusCode === 404 || code === "ContainerNotFound"; + const streamPath = typeof stream?.path === "string" ? stream.path : null; + + if (!missingContainer || !streamPath || !fs.existsSync(streamPath)) { + throw error; + } + + // A background empty-container cull or transient create failure can remove + // the just-created scoped container before the first upload block lands. + // Recreate the container and retry with a fresh file stream. + console.warn(`Azure container ${this.containerName} missing during upload; recreating and retrying once`); + this._containerEnsured = false; + const retryClient = await this.getBlobClient(); + activeContainerClient = retryClient.containerClient; + blockBlobClient = activeContainerClient.getBlockBlobClient(blobName); + await blockBlobClient.uploadStream( + fs.createReadStream(streamPath), + undefined, + undefined, + options, + ); + } - await blockBlobClient.uploadStream(stream, undefined, undefined, options); - - const sasToken = this.generateSASToken(containerClient, blobName); - const shortLivedSasToken = this.generateShortLivedSASToken(containerClient, blobName, 5); + const sasToken = this.generateSASToken(activeContainerClient, blobName); + const shortLivedSasToken = this.generateShortLivedSASToken(activeContainerClient, blobName, 5); const url = `${blockBlobClient.url}?${sasToken}`; const shortLivedUrl = `${blockBlobClient.url}?${shortLivedSasToken}`; @@ -280,61 +406,18 @@ export class AzureStorageProvider extends StorageProvider { try { const { containerClient } = await this.getBlobClient(); - - // Extract blob name from URL - const urlObj = new URL(url); - let blobName = urlObj.pathname.substring(1); // Remove leading slash - - // Handle different URL formats: - // 1. Azurite: /devstoreaccount1/container/blobname (3 segments) - // 2. Standard Azure: /container/blobname (2 segments) - // 3. Container-only: /container or /container/ (invalid) - - if (blobName.includes('/')) { - const pathSegments = blobName.split('/').filter(segment => segment.length > 0); - - if (pathSegments.length === 1) { - // Only container name, no blob name - this is invalid - console.warn(`Invalid blob URL (container-only): ${url}`); - return null; - } else if (pathSegments.length === 2) { - // Standard Azure format: container/blobname - // Check if first segment matches container name - if (pathSegments[0] === this.containerName) { - blobName = pathSegments[1]; - } else { - // Container name doesn't match, but assume second segment is blob name - blobName = pathSegments[1]; - } - } else if (pathSegments.length >= 3) { - // Azurite format: devstoreaccount1/container/blobname - // Skip the account and container segments to get the actual blob name - // Check if second segment matches container name - if (pathSegments[1] === this.containerName) { - blobName = pathSegments.slice(2).join('/'); - } else { - // Container name doesn't match, but assume remaining segments are blob name - blobName = pathSegments.slice(2).join('/'); - } - } - } else { - // No slashes - could be just container name or just blob name - if (blobName === this.containerName || blobName === this.containerName + '/') { - // URL is just the container name - invalid blob URL - console.warn(`Invalid blob URL (container-only): ${url}`); - return null; - } - // Otherwise assume it's a blob name at root level (unlikely but possible) - } - - // Validate that we have a non-empty blob name + + // Use extractBlobNameFromUrl which correctly handles both standard Azure + // and Azurite URL formats by finding the container name index + const blobName = this.extractBlobNameFromUrl(url); + if (!blobName || blobName.trim().length === 0) { - console.warn(`Invalid blob URL (empty blob name): ${url}`); + console.warn(`Invalid blob URL (could not extract blob name): ${url}`); return null; } - + const blockBlobClient = containerClient.getBlockBlobClient(blobName); - + try { await blockBlobClient.delete(); return blobName; @@ -413,7 +496,12 @@ export class AzureStorageProvider extends StorageProvider { extractBlobNameFromUrl(url) { try { const urlObj = new URL(url); - const pathParts = urlObj.pathname.split("/"); + // Decode the pathname to reverse URL encoding applied by the Azure SDK. + // Blob names may contain percent-encoded characters (e.g., %20 for spaces), + // and the SDK further encodes the % as %25 when building the URL (%20 → %2520). + // A single decodeURIComponent reverses that to get the actual blob name. + const decodedPath = decodeURIComponent(urlObj.pathname); + const pathParts = decodedPath.split("/"); const containerIndex = pathParts.indexOf(this.containerName); if (containerIndex === -1) return null; @@ -424,6 +512,109 @@ export class AzureStorageProvider extends StorageProvider { } } + /** + * List all files in a folder path + * @param {string} folderPath - The folder path to list (e.g., 'users/123/global') + * @returns {Promise} Array of file objects with name, filename, hash, lastModified + */ + async listFolder(folderPath) { + const { containerClient } = await this.getBlobClient({ createContainer: false }); + + // Ensure folder path ends with / for proper prefix matching + // Empty string means "list everything" — use undefined prefix (no filter) + const prefix = folderPath === '' ? undefined : (folderPath.endsWith('/') ? folderPath : `${folderPath}/`); + const results = []; + + try { + for await (const blob of containerClient.listBlobsFlat({ prefix })) { + // Extract just the filename from the full blob path and decode + // (blob names are URL-encoded by uploadStream via encodeURIComponent) + const rawFilename = blob.name.split('/').pop(); + let filename; + try { filename = decodeURIComponent(rawFilename); } catch { filename = rawFilename; } + + // Extract hash from filename if it matches pattern {hash}_{filename} + // Hash is a hex string (xxhash64 produces 16-char hex strings) + const hashMatch = filename.match(/^([a-f0-9]+)_/i); + + // Generate a short-lived SAS URL for direct download (60 min) + const blockBlobClient = containerClient.getBlockBlobClient(blob.name); + const sasToken = this.generateShortLivedSASToken(containerClient, blob.name, 60); + + results.push({ + name: blob.name, // Full blob path + filename: hashMatch ? filename.replace(/^[a-f0-9]+_/i, '') : filename, // Original filename without hash prefix + hash: hashMatch ? hashMatch[1] : null, + lastModified: blob.properties.lastModified, + contentType: blob.properties.contentType, + size: blob.properties.contentLength, + url: `${blockBlobClient.url}?${sasToken}`, + }); + } + } catch (e) { + // A user with no uploads yet has no per-user container — treat as empty. + const code = e?.code || e?.details?.errorCode; + if (e?.statusCode === 404 || code === "ContainerNotFound") { + return []; + } + throw e; + } + + return results; + } + + /** + * Rename a blob by copying to a new name and deleting the old one. + * @param {string} oldBlobName - The current blob name + * @param {string} newBlobName - The new blob name + * @returns {Promise<{url: string, shortLivedUrl: string, blobName: string}>} + */ + async renameBlob(oldBlobName, newBlobName) { + const { containerClient } = await this.getBlobClient(); + + // If the blob names are identical, skip the copy-delete cycle + // (otherwise copy-to-self then delete would destroy the file) + if (oldBlobName === newBlobName) { + const blobClient = containerClient.getBlockBlobClient(oldBlobName); + const sasToken = this.generateSASToken(oldBlobName); + const shortLivedSasToken = this.generateShortLivedSASToken(oldBlobName, 5); + return { + url: `${blobClient.url}?${sasToken}`, + shortLivedUrl: `${blobClient.url}?${shortLivedSasToken}`, + blobName: oldBlobName, + }; + } + + const oldBlobClient = containerClient.getBlockBlobClient(oldBlobName); + const newBlobClient = containerClient.getBlockBlobClient(newBlobName); + + // Generate a short SAS token so the copy source is accessible + const sourceSas = this.generateShortLivedSASToken(oldBlobName, 10); + const sourceUrl = `${oldBlobClient.url}?${sourceSas}`; + + // Copy old blob to new name + const copyPoller = await newBlobClient.beginCopyFromURL(sourceUrl); + await copyPoller.pollUntilDone(); + + // Delete the old blob — if this fails the old blob is orphaned but + // the rename still succeeds (the new blob exists). + try { + await oldBlobClient.delete(); + } catch (deleteErr) { + console.error(`Orphaned blob after rename: ${oldBlobName} (new: ${newBlobName}) — ${deleteErr.message}`); + } + + // Generate SAS tokens for the new blob + const sasToken = this.generateSASToken(newBlobName); + const shortLivedSasToken = this.generateShortLivedSASToken(newBlobName, 5); + + return { + url: `${newBlobClient.url}?${sasToken}`, + shortLivedUrl: `${newBlobClient.url}?${shortLivedSasToken}`, + blobName: newBlobName, + }; + } + /** * Update blob index tags (specifically the retention tag) * @param {string} blobName - The blob name @@ -445,7 +636,7 @@ export class AzureStorageProvider extends StorageProvider { const tagsResponse = await blockBlobClient.getTags(); // Tags response might be an object with a tags property or a plain object if (tagsResponse && typeof tagsResponse === 'object') { - currentTags = tagsResponse.tags || tagsResponse; + currentTags = tagsResponse?.tags ?? {}; } } catch (error) { // If getTags fails (e.g., no tags exist or Azurite doesn't support it), start with empty object diff --git a/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js b/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js index 11c78c42..67d387dc 100644 --- a/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js +++ b/helper-apps/cortex-file-handler/src/services/storage/GCSStorageProvider.js @@ -39,7 +39,7 @@ export class GCSStorageProvider extends StorageProvider { return `gs://${bucket}/${pathParts.map((part) => decodeURIComponent(part)).join("/")}`; } - async uploadFile(context, filePath, requestId, hash = null, filename = null) { + async uploadFile(context, filePath, requestId, hash = null, filename = null, folderPath = null) { const bucket = this.storage.bucket(this.bucketName); // Use provided filename or generate LLM-friendly naming @@ -52,6 +52,14 @@ export class GCSStorageProvider extends StorageProvider { blobName = generateBlobName(requestId, `${shortId}${fileExtension}`); } + // If folderPath is provided, prepend it to create folder hierarchy (e.g., userId isolation for GCS) + if (folderPath) { + const normalizedFolder = folderPath.replace(/^\/+|\/+$/g, ''); + if (normalizedFolder) { + blobName = `${normalizedFolder}/${blobName}`; + } + } + if (typeof filePath === "string") { // Use bucket.upload for file-path uploads await bucket.upload(filePath, { @@ -74,12 +82,21 @@ export class GCSStorageProvider extends StorageProvider { }; } - async uploadStream(context, encodedFilename, stream, providedContentType = null) { + async uploadStream(context, encodedFilename, stream, providedContentType = null, retention = 'temporary', folderPath = null) { const bucket = this.storage.bucket(this.bucketName); - const blobName = sanitizeFilename(encodedFilename); + let blobName = sanitizeFilename(encodedFilename); + + // If folderPath is provided, prepend it to create folder hierarchy + if (folderPath) { + // Normalize folder path: remove leading/trailing slashes + const normalizedFolder = folderPath.replace(/^\/+|\/+$/g, ''); + if (normalizedFolder) { + blobName = `${normalizedFolder}/${blobName}`; + } + } let contentType = providedContentType || this.getContentType(encodedFilename) || "application/octet-stream"; - + // For text MIME types, ensure charset=utf-8 is included if not already present if (this.isTextMimeType(contentType)) { if (!contentType.includes('charset=')) { @@ -96,6 +113,7 @@ export class GCSStorageProvider extends StorageProvider { }); await new Promise((resolve, reject) => { + stream.on('error', reject); stream.pipe(writeStream) .on('finish', resolve) .on('error', reject); @@ -109,6 +127,122 @@ export class GCSStorageProvider extends StorageProvider { return isTextMimeTypeUtil(mimeType); } + /** + * List all files in a folder path + * @param {string} folderPath - The folder path to list (e.g., 'users/123/global') + * @returns {Promise} Array of file objects with name, filename, hash, lastModified + */ + async listFolder(folderPath) { + const bucket = this.storage.bucket(this.bucketName); + + // Ensure folder path ends with / for proper prefix matching. + // Empty string means "list everything" — use undefined prefix (no filter). + const prefix = folderPath === '' ? undefined : (folderPath.endsWith('/') ? folderPath : `${folderPath}/`); + const results = []; + + if (process.env.STORAGE_EMULATOR_HOST) { + // Use REST API for emulator + try { + const listResp = await axios.get( + `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${this.bucketName}/o`, + { + params: { prefix }, + validateStatus: (s) => s === 200 || s === 404, + }, + ); + + if (listResp.status === 200 && Array.isArray(listResp.data.items)) { + for (const item of listResp.data.items) { + const rawFilename = item.name.split('/').pop(); + let filename; + try { filename = decodeURIComponent(rawFilename); } catch { filename = rawFilename; } + const hashMatch = filename.match(/^([a-f0-9]+)_/i); + + // For emulator, construct a direct download URL (no signed URLs needed) + const url = `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${this.bucketName}/o/${encodeURIComponent(item.name)}?alt=media`; + + results.push({ + name: item.name, + filename: hashMatch ? filename.replace(/^[a-f0-9]+_/i, '') : filename, + hash: hashMatch ? hashMatch[1] : null, + lastModified: item.updated ? new Date(item.updated) : null, + contentType: item.contentType, + size: parseInt(item.size, 10) || 0, + url, + }); + } + } + } catch (error) { + console.error("Error listing folder from emulator:", error); + } + } else { + // Use GCS client library + try { + const [files] = await bucket.getFiles({ prefix }); + for (const file of files) { + const rawFilename = file.name.split('/').pop(); + let filename; + try { filename = decodeURIComponent(rawFilename); } catch { filename = rawFilename; } + const hashMatch = filename.match(/^([a-f0-9]+)_/i); + + const [metadata] = await file.getMetadata(); + + // Generate a signed URL for direct download (60 min) + const [signedUrl] = await file.getSignedUrl({ + action: 'read', + expires: Date.now() + 60 * 60 * 1000, + }); + + results.push({ + name: file.name, + filename: hashMatch ? filename.replace(/^[a-f0-9]+_/i, '') : filename, + hash: hashMatch ? hashMatch[1] : null, + lastModified: metadata.updated ? new Date(metadata.updated) : null, + contentType: metadata.contentType, + size: parseInt(metadata.size, 10) || 0, + url: signedUrl, + }); + } + } catch (error) { + console.error("Error listing folder from GCS:", error); + } + } + + return results; + } + + /** + * Rename a blob by copying to a new name and deleting the old one. + * @param {string} oldBlobName - The current blob name + * @param {string} newBlobName - The new blob name + * @returns {Promise<{url: string}>} + */ + async renameBlob(oldBlobName, newBlobName) { + const bucket = this.storage.bucket(this.bucketName); + + if (process.env.STORAGE_EMULATOR_HOST) { + // Emulator: use REST API to copy then delete + const copyUrl = `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${this.bucketName}/o/${encodeURIComponent(oldBlobName)}/copyTo/b/${this.bucketName}/o/${encodeURIComponent(newBlobName)}`; + await axios.post(copyUrl, null, { + validateStatus: (s) => s === 200, + }); + + await axios.delete( + `${process.env.STORAGE_EMULATOR_HOST}/storage/v1/b/${this.bucketName}/o/${encodeURIComponent(oldBlobName)}`, + { validateStatus: (s) => s === 200 || s === 204 }, + ); + } else { + // Real GCS: use client library + const file = bucket.file(oldBlobName); + await file.copy(bucket.file(newBlobName)); + await file.delete(); + } + + return { + url: `gs://${this.bucketName}/${newBlobName}`, + }; + } + async deleteFiles(requestId) { if (!requestId) throw new Error("Missing requestId parameter"); @@ -267,6 +401,9 @@ export class GCSStorageProvider extends StorageProvider { ); return response.status === 200; } catch (error) { + if (error.response?.status === 404) { + return false; + } console.error("Error checking emulator file:", error); return false; } @@ -277,6 +414,9 @@ export class GCSStorageProvider extends StorageProvider { const [exists] = await file.exists(); return exists; } catch (error) { + if (error.code === 404) { + return false; + } console.error("Error checking if GCS URL exists:", error); return false; } diff --git a/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js b/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js index f3687641..f9e3fa88 100644 --- a/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js +++ b/helper-apps/cortex-file-handler/src/services/storage/StorageFactory.js @@ -39,16 +39,16 @@ export class StorageFactory { return this.getLocalProvider(); } - async getAzureProvider() { - // Always use single container from env var - const containerName = getContainerName(); - + async getAzureProvider(containerName = null) { + // Use provided container name, or fall back to default from env var + const resolvedContainer = containerName || getContainerName(); + // Create unique key for caching - const key = `azure-${containerName}`; + const key = `azure-${resolvedContainer}`; if (!this.providers.has(key)) { const provider = new AzureStorageProvider( process.env.AZURE_STORAGE_CONNECTION_STRING, - containerName, + resolvedContainer, ); this.providers.set(key, provider); } diff --git a/helper-apps/cortex-file-handler/src/services/storage/StorageService.js b/helper-apps/cortex-file-handler/src/services/storage/StorageService.js index ddb4f56c..98b8ecc1 100644 --- a/helper-apps/cortex-file-handler/src/services/storage/StorageService.js +++ b/helper-apps/cortex-file-handler/src/services/storage/StorageService.js @@ -3,7 +3,9 @@ import path from "path"; import os from "os"; import fs from "fs"; import { v4 as uuidv4 } from "uuid"; -import { generateShortId } from "../../utils/filenameUtils.js"; +import { generateShortId, sanitizeFilename } from "../../utils/filenameUtils.js"; +import { sanitizeTargetBlobPath } from "../../utils/targetBlobPathUtils.js"; +import { AZURITE_ACCOUNT_NAME, getDefaultContainerName, getUserContainerName } from "../../constants.js"; export class StorageService { constructor(factory) { @@ -150,10 +152,13 @@ export class StorageService { async deleteFile(url) { await this._initialize(); - - // Always use primary provider - single container only - const provider = this.primaryProvider; - + + // Get the correct provider for the URL's container (may be per-user) + const containerName = this._extractContainerFromUrl(url); + const provider = containerName + ? await StorageFactory.getInstance().getAzureProvider(containerName) + : this.primaryProvider; + if (typeof provider.deleteFile === "function") { return await provider.deleteFile(url); } @@ -174,6 +179,82 @@ export class StorageService { return await this.backupProvider.deleteFiles([url]); } + /** + * Scan cloud storage for orphaned blobs matching a hash and delete them. + * Used when the Redis entry is gone but the blob may still exist. + * Blobs are named {hash}_{filename}, so we list the user's folder tree + * and match by hash prefix. + */ + async _deleteOrphanedBlobs(hash, contextId) { + const deleted = []; + + // Determine which Azure provider + folder prefix to scan. + // Per-user containers (getUserContainerName) store files at the container + // root (e.g., chats/{chatId}/{hash}_{file}), so we list everything (''). + // Legacy shared containers store files under users/{contextId}/. + const baseName = getDefaultContainerName(); + const perUserContainer = getUserContainerName(baseName, contextId); + const isPerUser = perUserContainer !== baseName; + + const azureProvider = isPerUser + ? await StorageFactory.getInstance().getAzureProvider(perUserContainer) + : this.primaryProvider; + const folderPrefix = isPerUser ? '' : `users/${contextId}`; + + // Scan Azure storage for orphaned blobs matching this hash + if (azureProvider && typeof azureProvider.listFolder === 'function') { + try { + const files = await azureProvider.listFolder(folderPrefix); + for (const file of files) { + if (file.hash === hash) { + try { + const result = await azureProvider.deleteFile(file.url); + if (result) { + console.log(`Deleted orphaned primary blob: ${file.name}`); + deleted.push({ provider: 'primary', result: file.name }); + } else { + console.warn(`Primary blob not found for orphan cleanup: ${file.name}`); + } + } catch (err) { + console.error(`Failed to delete orphaned primary blob ${file.name}: ${err.message}`); + } + } + } + } catch (err) { + console.error(`Error scanning primary storage for orphaned blobs: ${err.message}`); + } + } + + // Scan backup storage (GCS) — listFolder returns signed HTTPS URLs, + // but deleteFile expects gs:// URLs, so construct from file.name + if (this.backupProvider && typeof this.backupProvider.listFolder === 'function') { + try { + const files = await this.backupProvider.listFolder(folderPrefix); + for (const file of files) { + if (file.hash === hash) { + try { + // Construct gs:// URL from the blob name since listFolder returns signed HTTPS URLs + const gcsUrl = `gs://${this.backupProvider.bucketName}/${file.name}`; + const result = await this.backupProvider.deleteFile(gcsUrl); + if (result) { + console.log(`Deleted orphaned backup blob: ${file.name}`); + deleted.push({ provider: 'backup', result: file.name }); + } else { + console.warn(`Backup blob not found for orphan cleanup: ${file.name}`); + } + } catch (err) { + console.error(`Failed to delete orphaned backup blob ${file.name}: ${err.message}`); + } + } + } + } catch (err) { + console.error(`Error scanning backup storage for orphaned blobs: ${err.message}`); + } + } + + return deleted; + } + /** * Delete a single file by its hash from both primary and backup storage * @param {string} hash - The hash of the file to delete @@ -182,52 +263,69 @@ export class StorageService { */ async deleteFileByHash(hash, contextId = null) { await this._initialize(); - + if (!hash) { throw new Error("Missing hash parameter"); } const results = []; - // Get and remove file information from Redis map + // Get file information from Redis (skip lazy cleanup — we're deleting, not reading) const { getFileStoreMap, removeFromFileStoreMap } = await import("../../redis.js"); - const hashResult = await getFileStoreMap(hash, false, contextId); - - if (hashResult) { - // Remove from Redis - await removeFromFileStoreMap(hash, contextId); - } - + const hashResult = await getFileStoreMap(hash, true, contextId); + if (!hashResult) { - throw new Error(`File with hash ${hash} not found`); + // No URL info in Redis — but the blob may still exist in cloud storage + // (e.g., previous delete removed Redis entry but failed to delete the blob). + // Scan the user's folder for orphaned blobs matching this hash and clean them up. + await removeFromFileStoreMap(hash, contextId); + + if (contextId) { + const orphansDeleted = await this._deleteOrphanedBlobs(hash, contextId); + if (orphansDeleted.length > 0) { + console.log(`Cleaned up ${orphansDeleted.length} orphaned blob(s) for hash ${hash}`); + return { hash, deleted: orphansDeleted, orphanCleanup: true, results: orphansDeleted }; + } + } + + return { hash, alreadyDeleted: true, results }; } - // Delete from primary storage + // Delete from primary storage FIRST (before removing from Redis) + // This ensures we can retry if cloud deletion fails + let primaryDeleted = false; if (hashResult.url) { try { - // Log the URL being deleted for debugging (redact SAS token for security) const { redactSasToken } = await import('../../utils/logSecurity.js'); console.log(`Deleting file from primary storage - hash: ${hash}, url: ${redactSasToken(hashResult.url)}`); - - // Always use primary provider - single container only - const provider = this.primaryProvider; - + + // Get the correct provider for the URL's container (may be per-user) + const containerName = this._extractContainerFromUrl(hashResult.url); + const provider = containerName + ? await StorageFactory.getInstance().getAzureProvider(containerName) + : this.primaryProvider; const primaryResult = await provider.deleteFile(hashResult.url); if (primaryResult) { console.log(`Successfully deleted from primary storage - hash: ${hash}, result: ${primaryResult}`); results.push({ provider: 'primary', result: primaryResult }); + primaryDeleted = true; } else { - // deleteFile returned null, which means the URL was invalid or blob not found - console.warn(`Invalid or empty URL for hash ${hash}: ${redactSasToken(hashResult.url)}`); - results.push({ provider: 'primary', error: 'Invalid URL (container-only or empty blob name)' }); + // deleteFile returned null — blob not found, treat as already deleted + console.warn(`Primary blob not found for hash ${hash}: ${redactSasToken(hashResult.url)}`); + results.push({ provider: 'primary', result: 'not_found' }); + primaryDeleted = true; // Not found = already gone, safe to remove from Redis } } catch (error) { console.error(`Error deleting file from primary storage:`, error); results.push({ provider: 'primary', error: error.message }); + // primaryDeleted stays false — don't remove from Redis so delete can be retried } + } else { + primaryDeleted = true; // No URL to delete } // Delete from backup storage (GCS) + let backupDeleted = false; if (hashResult.gcs && this.backupProvider) { try { console.log(`Deleting file from backup storage - hash: ${hash}, gcs: ${hashResult.gcs}`); @@ -237,22 +335,24 @@ export class StorageService { results.push({ provider: 'backup', result: backupResult }); } else { console.warn(`Backup deletion returned null for hash ${hash}: ${hashResult.gcs}`); - results.push({ provider: 'backup', error: 'Deletion returned null' }); + results.push({ provider: 'backup', result: 'not_found' }); } + backupDeleted = true; } catch (error) { console.error(`Error deleting file from backup storage:`, error); results.push({ provider: 'backup', error: error.message }); } } else { - if (!hashResult.gcs) { - console.log(`No GCS URL found for hash ${hash}, skipping backup deletion`); - } else if (!this.backupProvider) { - console.log(`Backup provider not configured, skipping backup deletion for hash ${hash}`); - } + backupDeleted = true; // Nothing to delete } - // Note: Hash was already removed from Redis atomically at the beginning - // No need to remove again + // Only remove from Redis after cloud deletion succeeds (or blobs confirmed gone) + // This prevents orphaned blobs that can never be cleaned up via hash-based delete + if (primaryDeleted) { + await removeFromFileStoreMap(hash, contextId); + } else { + console.warn(`Keeping Redis entry for hash ${hash} — primary storage deletion failed, retry will be possible`); + } return { hash, @@ -299,8 +399,12 @@ export class StorageService { throw new Error(`File with hash ${hash} has no valid URL`); } - // Always use primary provider - single container only - const provider = this.primaryProvider; + // Use the provider for the URL's container so context-scoped uploads in + // per-user containers can extract blob names and generate SAS tokens. + const containerName = this._extractContainerFromUrl(hashResult.url); + const provider = containerName + ? await StorageFactory.getInstance().getAzureProvider(containerName) + : this.primaryProvider; // Check if provider supports blob tag operations (Azure only) const supportsBlobTags = typeof provider.extractBlobNameFromUrl === 'function' && @@ -330,8 +434,8 @@ export class StorageService { } // Generate new short-lived URL - const { containerClient } = await provider.getBlobClient(); - const shortLivedSasToken = provider.generateShortLivedSASToken(containerClient, blobName, 5); + await provider.ensureInitialized(); + const shortLivedSasToken = provider.generateShortLivedSASToken(blobName, 5); const urlObj = new URL(hashResult.url); const baseUrl = `${urlObj.protocol}//${urlObj.host}${urlObj.pathname}`; shortLivedUrl = `${baseUrl}?${shortLivedSasToken}`; @@ -345,7 +449,7 @@ export class StorageService { await provider.updateBlobTags(convertedBlobName, retention); const convertedUrlObj = new URL(hashResult.converted.url); const convertedBaseUrl = `${convertedUrlObj.protocol}//${convertedUrlObj.host}${convertedUrlObj.pathname}`; - const convertedShortLivedSasToken = provider.generateShortLivedSASToken(containerClient, convertedBlobName, 5); + const convertedShortLivedSasToken = provider.generateShortLivedSASToken(convertedBlobName, 5); const convertedShortLivedUrl = `${convertedBaseUrl}?${convertedShortLivedSasToken}`; convertedResult = { url: hashResult.converted.url, @@ -401,9 +505,154 @@ export class StorageService { }; } - async uploadFileWithProviders(context, filePath, requestId, hash = null, filename = null) { + /** + * Rename a file in cloud storage and update Redis. + * Copies the blob to a new name (preserving folder path and hash prefix), + * deletes the old blob, and updates the Redis entry. + * @param {string} hash - The file hash (Redis key) + * @param {string} newFilename - The new display filename + * @param {Object} context - Context object for logging + * @param {string|null} contextId - Optional context ID for scoped file storage + * @param {Object} options - Optional source/target blob path overrides + * @returns {Promise} Updated file info + */ + async renameFile(hash, newFilename, context = {}, contextId = null, options = {}) { await this._initialize(); - + + if (!hash) throw new Error("Missing hash parameter"); + if (!newFilename || !newFilename.trim()) throw new Error("Missing newFilename parameter"); + + const { + sourceBlobPath = "", + targetBlobPath = "", + } = typeof options === "string" + ? { targetBlobPath: options } + : options || {}; + const sanitizedTargetBlobPath = targetBlobPath + ? sanitizeTargetBlobPath(targetBlobPath) + : ""; + if (targetBlobPath && !sanitizedTargetBlobPath) { + throw new Error("Invalid targetBlobPath parameter"); + } + + const { getFileStoreMap, setFileStoreMap } = await import("../../redis.js"); + const hashResult = await getFileStoreMap(hash, false, contextId); + if (!hashResult) throw new Error(`File with hash ${hash} not found`); + + const trimmedName = newFilename.trim(); + const sanitized = sanitizeFilename(trimmedName); + + // --- Rename in primary (Azure) storage --- + let newUrl = hashResult.url; + let newShortLivedUrl = hashResult.shortLivedUrl || hashResult.url; + let newPrimaryBlobName = hashResult.blobPath || hashResult.blobName || ""; + + if (hashResult.url && hashResult.url.startsWith('http')) { + // Get the correct provider for the URL's container (may be per-user) + const containerName = this._extractContainerFromUrl(hashResult.url); + const provider = containerName + ? await StorageFactory.getInstance().getAzureProvider(containerName) + : this.primaryProvider; + + const oldBlobName = sourceBlobPath || provider.extractBlobNameFromUrl(hashResult.url); + if (oldBlobName) { + const newBlobName = sanitizedTargetBlobPath + || this._computeNewBlobName(oldBlobName, sanitized); + context.log?.(`Renaming blob: ${oldBlobName} → ${newBlobName}`); + const result = await provider.renameBlob(oldBlobName, newBlobName); + newUrl = result.url; + newShortLivedUrl = result.shortLivedUrl || result.url; + newPrimaryBlobName = newBlobName; + } + } + + // --- Rename in backup (GCS) storage --- + let newGcs = hashResult.gcs; + if (hashResult.gcs && this.backupProvider && typeof this.backupProvider.renameBlob === 'function') { + try { + const gcsUrl = this.backupProvider.ensureUnencodedGcsUrl + ? this.backupProvider.ensureUnencodedGcsUrl(hashResult.gcs) + : hashResult.gcs; + const oldGcsBlobName = gcsUrl.replace("gs://", "").split("/").slice(1).join("/"); + const newGcsBlobName = sanitizedTargetBlobPath + || this._computeNewBlobName(oldGcsBlobName, sanitized); + + context.log?.(`Renaming GCS blob: ${oldGcsBlobName} → ${newGcsBlobName}`); + const gcsResult = await this.backupProvider.renameBlob(oldGcsBlobName, newGcsBlobName); + newGcs = gcsResult.url; + } catch (err) { + context.log?.(`Warning: GCS rename failed: ${err.message}`); + } + } + + // --- Update Redis --- + const updatedInfo = { + ...hashResult, + url: newUrl, + gcs: newGcs, + filename: sanitized, + ...(newPrimaryBlobName ? { + blobPath: newPrimaryBlobName, + blobName: newPrimaryBlobName, + } : {}), + timestamp: new Date().toISOString(), + }; + // Remove displayFilename — the blob name is now the source of truth + delete updatedInfo.displayFilename; + delete updatedInfo.shortLivedUrl; + await setFileStoreMap(hash, updatedInfo, contextId); + + return { + hash, + filename: sanitized, + url: newUrl, + shortLivedUrl: newShortLivedUrl, + gcs: newGcs, + ...(newPrimaryBlobName ? { blobPath: newPrimaryBlobName } : {}), + message: `File renamed to "${trimmedName}"`, + }; + } + + /** + * Extract container name from an Azure blob URL. + * Handles both real Azure and Azurite URL formats. + */ + _extractContainerFromUrl(url) { + try { + const urlObj = new URL(url); + let pathParts = urlObj.pathname.split('/').filter(p => p.length > 0); + if (pathParts[0] === AZURITE_ACCOUNT_NAME) { + pathParts = pathParts.slice(1); + } + return pathParts[0] || null; + } catch { return null; } + } + + /** + * Compute a new blob name by replacing the filename portion while preserving + * the folder path and hash prefix. + * @param {string} oldBlobName - Current blob name (e.g., "chats/abc/hash_old.png") + * @param {string} sanitizedNewName - Sanitized new filename (e.g., "new name.png") + * @param {boolean} urlEncode - Whether to encodeURIComponent the filename (Azure yes, GCS no) + */ + _computeNewBlobName(oldBlobName, sanitizedNewName) { + const lastSlash = oldBlobName.lastIndexOf('/'); + const folderPath = lastSlash >= 0 ? oldBlobName.substring(0, lastSlash) : ''; + const oldFilePart = lastSlash >= 0 ? oldBlobName.substring(lastSlash + 1) : oldBlobName; + + // Inspect the hash prefix from the blob name + const hashMatch = oldFilePart.match(/^([a-f0-9]+)_/i); + const hashPrefix = hashMatch ? hashMatch[1] : null; + + // Blob names are stored unencoded; Azure SDK handles URL-encoding. + // Do NOT encodeURIComponent here — that would double-encode. + const newFileBase = hashPrefix ? `${hashPrefix}_${sanitizedNewName}` : sanitizedNewName; + return folderPath ? `${folderPath}/${newFileBase}` : newFileBase; + } + + async uploadFileWithProviders(context, filePath, requestId, hash = null, filename = null, gcsFolderPath = null) { + await this._initialize(); + // Use provided filename or generate one const finalFilename = filename || (() => { const fileExtension = path.extname(filePath); @@ -432,6 +681,7 @@ export class StorageService { requestId, hash, finalFilename, + gcsFolderPath, ); } @@ -441,10 +691,10 @@ export class StorageService { // Fallback: generate short-lived URL if not provided if (primaryProvider.generateShortLivedSASToken) { try { - const { containerClient } = await primaryProvider.getBlobClient(); + await primaryProvider.ensureInitialized(); const blobName = primaryResult.blobName || primaryProvider.extractBlobNameFromUrl(result.url); if (blobName) { - const shortLivedSasToken = primaryProvider.generateShortLivedSASToken(containerClient, blobName, 5); + const shortLivedSasToken = primaryProvider.generateShortLivedSASToken(blobName, 5); const urlObj = new URL(result.url); const baseUrl = `${urlObj.protocol}//${urlObj.host}${urlObj.pathname}`; result.shortLivedUrl = `${baseUrl}?${shortLivedSasToken}`; diff --git a/helper-apps/cortex-file-handler/src/start.js b/helper-apps/cortex-file-handler/src/start.js index e4383468..07b0b1e2 100644 --- a/helper-apps/cortex-file-handler/src/start.js +++ b/helper-apps/cortex-file-handler/src/start.js @@ -74,7 +74,7 @@ app.all("/api/CortexFileHandler", async (req, res) => { } catch (error) { const status = error.status || 500; const message = error.message || "Internal server error"; - res.status(status).send(message); + res.status(status).type("text/plain").send(message); } }); @@ -90,7 +90,7 @@ app.all("/api/MediaFileChunker", async (req, res) => { } catch (error) { const status = error.status || 500; const message = error.message || "Internal server error"; - res.status(status).send(message); + res.status(status).type("text/plain").send(message); } }); diff --git a/helper-apps/cortex-file-handler/src/utils/filenameUtils.js b/helper-apps/cortex-file-handler/src/utils/filenameUtils.js index bfe484d0..3df53b76 100644 --- a/helper-apps/cortex-file-handler/src/utils/filenameUtils.js +++ b/helper-apps/cortex-file-handler/src/utils/filenameUtils.js @@ -20,6 +20,10 @@ export function sanitizeFilename(filename) { // Get just the basename to prevent path traversal let basename = path.basename(decoded); + // Strip ASCII and C1 control characters. C1 controls can appear when UTF-8 + // filenames are misdecoded as Latin-1 and Azure rejects them as invalid URIs. + basename = basename.normalize("NFC").replace(/[\u0000-\u001F\u007F-\u009F]/g, "_"); + // Replace invalid characters with underscores basename = basename.replace(/[<>:"/\\|?*]/g, "_"); diff --git a/helper-apps/cortex-file-handler/src/utils/legacyWorkspacePrivateResolver.js b/helper-apps/cortex-file-handler/src/utils/legacyWorkspacePrivateResolver.js new file mode 100644 index 00000000..6f47c67b --- /dev/null +++ b/helper-apps/cortex-file-handler/src/utils/legacyWorkspacePrivateResolver.js @@ -0,0 +1,460 @@ +import path from "path"; +import { Readable } from "stream"; + +import { constructFolderPath, getMimeTypeFromUrl } from "../blobHandler.js"; +import { + getDefaultContainerName, + getUserContainerName, + getUserContainerNameCandidates, +} from "../constants.js"; +import { StorageFactory } from "../services/storage/StorageFactory.js"; +import { sanitizeFilename } from "./filenameUtils.js"; + +export function getLegacyWorkspacePrivateContextId({ + userId = null, + workspaceId = null, + fileScope = null, +} = {}) { + if (fileScope !== "workspace-user-legacy" || !userId || !workspaceId) { + return null; + } + return `${workspaceId}:${userId}`; +} + +export async function resolveHashRecordWithLegacyWorkspacePrivateFallback({ + hash, + resolvedContextId = null, + userId = null, + workspaceId = null, + fileScope = null, + getFileStoreMap, +}) { + if (!hash || !resolvedContextId || typeof getFileStoreMap !== "function") { + return null; + } + + const legacyContextId = getLegacyWorkspacePrivateContextId({ + userId, + workspaceId, + fileScope, + }); + + if (!legacyContextId || legacyContextId === resolvedContextId) { + return null; + } + + const hashResult = await getFileStoreMap(hash, true, legacyContextId); + if (!hashResult) { + return null; + } + + return { + hashResult, + sourceContextId: legacyContextId, + source: "legacy-workspace-private", + }; +} + +function getCanonicalFilename(hashResult, fallback = "file") { + const preferred = + hashResult?.displayFilename || + hashResult?.filename || + (() => { + try { + if (!hashResult?.url) return null; + const url = new URL(hashResult.url); + return decodeURIComponent(path.basename(url.pathname)); + } catch { + return null; + } + })() || + fallback; + + const sanitized = sanitizeFilename(preferred); + return sanitized || fallback; +} + +async function downloadLegacyBuffer(storageService, hashResult) { + const candidates = [hashResult?.url, hashResult?.gcs].filter(Boolean); + let lastError = null; + + for (const candidate of candidates) { + try { + return await storageService.downloadFile(candidate); + } catch (error) { + lastError = error; + } + } + + throw lastError || new Error("No legacy source available for migration"); +} + +async function persistResolvedRecord({ + hash, + hashResult, + sourceContextId = null, + resolvedContextId = null, + setFileStoreMap, + removeFromFileStoreMap, +}) { + const normalized = { + ...hashResult, + hash: hashResult?.hash || hash, + }; + delete normalized.shortLivedUrl; + + await setFileStoreMap(hash, normalized, resolvedContextId); + if ( + sourceContextId && + sourceContextId !== resolvedContextId && + typeof removeFromFileStoreMap === "function" + ) { + await removeFromFileStoreMap(hash, sourceContextId); + } + + return normalized; +} + +export async function migrateHashRecordToScopedStorage({ + context, + hash, + hashResult, + sourceContextId = null, + resolvedContextId = null, + userId = null, + chatId = null, + workspaceId = null, + appletId = null, + fileScope = null, + storageService, + setFileStoreMap, + removeFromFileStoreMap, +}) { + if ( + !hash || + !hashResult || + !resolvedContextId || + !userId || + sourceContextId === resolvedContextId || + typeof setFileStoreMap !== "function" + ) { + return hashResult; + } + + const folderPath = constructFolderPath({ + userId, + chatId, + workspaceId, + appletId, + contextId: resolvedContextId, + fileScope, + }); + + if (folderPath === null) { + return await persistResolvedRecord({ + hash, + hashResult, + sourceContextId, + resolvedContextId, + setFileStoreMap, + removeFromFileStoreMap, + }); + } + + const primaryProvider = await storageService.getPrimaryProvider(); + const isAzureProvider = + primaryProvider?.constructor?.name === "AzureStorageProvider"; + + if (!isAzureProvider) { + return await persistResolvedRecord({ + hash, + hashResult, + sourceContextId, + resolvedContextId, + setFileStoreMap, + removeFromFileStoreMap, + }); + } + + const buffer = await downloadLegacyBuffer(storageService, hashResult); + const uploadName = getCanonicalFilename(hashResult, `${hash}.bin`); + const contentType = + hashResult?.mimeType || getMimeTypeFromUrl(hashResult?.url || ""); + const retention = hashResult?.permanent ? "permanent" : "temporary"; + + const targetContainerName = getUserContainerName( + getDefaultContainerName(), + userId, + ); + const targetProvider = await StorageFactory.getInstance().getAzureProvider( + targetContainerName, + ); + const uploadResult = await targetProvider.uploadStream( + context || {}, + uploadName, + Readable.from([buffer]), + contentType, + retention, + folderPath, + ); + + const normalizedFolder = folderPath.replace(/^\/+|\/+$/g, ""); + const blobPath = normalizedFolder + ? `${normalizedFolder}/${uploadName}` + : uploadName; + + const migrated = { + ...hashResult, + url: uploadResult.url, + blobPath, + folderPath, + filename: uploadName, + hash: hashResult.hash || hash, + }; + + delete migrated.shortLivedUrl; + delete migrated.converted; + + return await persistResolvedRecord({ + hash, + hashResult: migrated, + sourceContextId, + resolvedContextId, + setFileStoreMap, + removeFromFileStoreMap, + }); +} + +function getTargetContainerOwnerId({ + resolvedContextId = null, + userId = null, + workspaceId = null, + fileScope = null, +} = {}) { + if (fileScope === "workspace-shared-legacy" && workspaceId) { + return workspaceId; + } + return userId || resolvedContextId || null; +} + +function getCanonicalTargetBlobPath(blobPath, { + contextId = null, + userId = null, + chatId = null, + workspaceId = null, + appletId = null, + fileScope = null, +} = {}) { + const normalizedName = sanitizeFilename(path.basename(blobPath || "")); + if (!normalizedName) { + return null; + } + + const folderPath = constructFolderPath({ + userId, + chatId, + workspaceId, + appletId, + contextId, + fileScope, + }); + + if (folderPath === null) { + return normalizedName; + } + + const normalizedFolder = folderPath.replace(/^\/+|\/+$/g, ""); + return normalizedFolder + ? `${normalizedFolder}/${normalizedName}` + : normalizedName; +} + +async function getAzureProviderForContainer(containerName) { + return await StorageFactory.getInstance().getAzureProvider(containerName); +} + +async function findExistingLegacyBlob(blobPath, providers = []) { + for (const entry of providers) { + if (!entry?.provider || !entry?.blobPath) { + continue; + } + + await entry.provider.ensureInitialized(); + const { containerClient } = await entry.provider.getBlobClient(); + const blockBlobClient = containerClient.getBlockBlobClient(entry.blobPath); + const exists = await blockBlobClient.exists(); + if (exists) { + return { + ...entry, + containerClient, + blockBlobClient, + }; + } + } + + return null; +} + +async function buildLegacyBlobCandidates({ + blobPath, + userId = null, + workspaceId = null, + fileScope = null, +} = {}) { + if (!blobPath) { + return []; + } + + const defaultContainerName = getDefaultContainerName(); + const candidates = [ + { + label: "default-root", + provider: await getAzureProviderForContainer(defaultContainerName), + blobPath, + }, + ]; + + const legacyContextId = getLegacyWorkspacePrivateContextId({ + userId, + workspaceId, + fileScope, + }); + + if (legacyContextId) { + const legacyContainerNames = getUserContainerNameCandidates( + defaultContainerName, + legacyContextId, + ).filter((containerName) => containerName !== defaultContainerName); + for (const legacyContainerName of legacyContainerNames) { + candidates.push({ + label: "legacy-compound-container", + provider: await getAzureProviderForContainer(legacyContainerName), + blobPath, + }); + } + } + + return candidates; +} + +export async function resolveBlobPathWithLegacyFallback({ + context, + hash = null, + blobPath, + resolvedContextId = null, + userId = null, + chatId = null, + workspaceId = null, + appletId = null, + fileScope = null, + storageService, + setFileStoreMap, +} = {}) { + if (!blobPath) { + return null; + } + + const source = await findExistingLegacyBlob( + blobPath, + await buildLegacyBlobCandidates({ + blobPath, + userId, + workspaceId, + fileScope, + }), + ); + + if (!source) { + return null; + } + + const targetOwnerId = getTargetContainerOwnerId({ + resolvedContextId, + userId, + workspaceId, + fileScope, + }); + const targetContainerName = targetOwnerId + ? getUserContainerName(getDefaultContainerName(), targetOwnerId) + : getDefaultContainerName(); + const targetProvider = await getAzureProviderForContainer(targetContainerName); + const targetBlobPath = getCanonicalTargetBlobPath(blobPath, { + contextId: resolvedContextId, + userId, + chatId, + workspaceId, + appletId, + fileScope, + }) || blobPath; + + await targetProvider.ensureInitialized(); + const { containerClient: targetContainerClient } = + await targetProvider.getBlobClient(); + const targetBlobClient = + targetContainerClient.getBlockBlobClient(targetBlobPath); + + if ( + source.provider.containerName !== targetProvider.containerName || + source.blobPath !== targetBlobPath + ) { + const sourceSas = source.provider.generateShortLivedSASToken( + source.blobPath, + 10, + ); + const sourceUrl = `${source.blockBlobClient.url}?${sourceSas}`; + const copyPoller = await targetBlobClient.beginCopyFromURL(sourceUrl); + await copyPoller.pollUntilDone(); + context?.log?.( + `Legacy blob self-healed: ${blobPath} -> ${targetBlobPath} (${source.label})`, + ); + } else { + context?.log?.(`Legacy blob found in-place: ${blobPath} (${source.label})`); + } + + const sasToken = targetProvider.generateSASToken(targetBlobPath); + const shortLivedSasToken = targetProvider.generateShortLivedSASToken( + targetBlobPath, + 5, + ); + const url = `${targetBlobClient.url}?${sasToken}`; + const shortLivedUrl = `${targetBlobClient.url}?${shortLivedSasToken}`; + const filename = sanitizeFilename(path.basename(targetBlobPath)); + + const result = { + url, + shortLivedUrl, + blobPath: targetBlobPath, + filename, + ...(hash ? { hash } : {}), + }; + + try { + const withBackup = await storageService?.ensureGCSUpload?.(context || {}, { + ...result, + blobName: targetBlobPath, + }); + if (withBackup?.gcs) { + result.gcs = withBackup.gcs; + } + } catch (error) { + context?.log?.( + `Warning: Could not ensure GCS backup for ${targetBlobPath}: ${error.message}`, + ); + } + + if (hash && resolvedContextId && typeof setFileStoreMap === "function") { + await setFileStoreMap( + hash, + { + url, + blobPath: targetBlobPath, + filename, + hash, + ...(result.gcs ? { gcs: result.gcs } : {}), + }, + resolvedContextId, + ); + } + + return result; +} diff --git a/helper-apps/cortex-file-handler/src/utils/targetBlobPathUtils.js b/helper-apps/cortex-file-handler/src/utils/targetBlobPathUtils.js new file mode 100644 index 00000000..901779a1 --- /dev/null +++ b/helper-apps/cortex-file-handler/src/utils/targetBlobPathUtils.js @@ -0,0 +1,32 @@ +import { sanitizeFilename } from "./filenameUtils.js"; + +export function sanitizeTargetBlobPath(targetBlobPath) { + const decoded = (() => { + try { + return decodeURIComponent(targetBlobPath); + } catch { + return targetBlobPath; + } + })(); + + const normalized = String(decoded || "") + .replace(/\\/g, "/") + .split("/") + .filter(Boolean) + .join("/"); + + if (!normalized) return ""; + + const sanitizedSegments = normalized.split("/").map((segment) => { + if (!segment || segment === "." || segment === "..") { + return ""; + } + return sanitizeFilename(segment); + }); + + if (sanitizedSegments.some((segment) => !segment)) { + return ""; + } + + return sanitizedSegments.join("/"); +} diff --git a/helper-apps/cortex-file-handler/tests/blobHandler.test.js b/helper-apps/cortex-file-handler/tests/blobHandler.test.js index 83110fb4..61432d0e 100644 --- a/helper-apps/cortex-file-handler/tests/blobHandler.test.js +++ b/helper-apps/cortex-file-handler/tests/blobHandler.test.js @@ -12,8 +12,17 @@ import { deleteGCS, getBlobClient, AZURE_STORAGE_CONTAINER_NAME, + constructFolderPath, getDefaultContainerName, + getScopedContainerOwnerId, + getScopedLogicalContextId, } from "../src/blobHandler.js"; +import { + getLegacyUserContainerName, + getUserContainerName, + getExtensionsForMimeType, + isAcceptedMimeType, +} from "../src/constants.js"; import { urlExists } from "../src/helper.js"; import CortexFileHandler from "../src/index.js"; import { setFileStoreMap } from "../src/redis.js"; @@ -29,6 +38,74 @@ function isGCSConfigured() { ); } +test("constructFolderPath supports scoped applet folders", (t) => { + t.is( + constructFolderPath({ + contextId: "applet-user:applet123:user456", + fileScope: "applet-user", + }), + "applets/applet123", + ); + t.is( + constructFolderPath({ + contextId: "applet-shared:applet123", + fileScope: "applet-shared", + }), + "applet-shared", + ); +}); + +test("accepts legacy WAV mime aliases for remote media", (t) => { + t.true(isAcceptedMimeType("audio/wav")); + t.true(isAcceptedMimeType("audio/x-wav")); + t.deepEqual(getExtensionsForMimeType("audio/x-wav"), [".wav"]); +}); + +test("applet-user routing keeps a logical scoped context but stores in the user container", (t) => { + const logicalContextId = getScopedLogicalContextId({ + userId: "user456", + appletId: "applet123", + fileScope: "applet-user", + }); + const containerOwnerId = getScopedContainerOwnerId({ + contextId: logicalContextId, + userId: "user456", + appletId: "applet123", + fileScope: "applet-user", + }); + const containerName = getUserContainerName( + getDefaultContainerName(), + containerOwnerId, + ); + + t.is(logicalContextId, "applet-user:applet123:user456"); + t.is(containerOwnerId, "user456"); + t.true(containerName.startsWith(`${getDefaultContainerName()}-`)); + t.false(containerName.includes("applet-user-applet123-user456")); +}); + +test("getUserContainerName preserves scope separators safely", (t) => { + const containerName = getUserContainerName( + getDefaultContainerName(), + "applet-user:applet123:user456", + ); + + t.true(containerName.startsWith(`${getDefaultContainerName()}-`)); + t.true(containerName.includes("applet-user-applet123-user456")); + t.false(containerName.includes(":")); +}); + +test("getLegacyUserContainerName preserves pre-scoped container mapping", (t) => { + const containerName = getLegacyUserContainerName( + getDefaultContainerName(), + "workspace-456:user-123", + ); + + t.true(containerName.startsWith(`${getDefaultContainerName()}-`)); + t.true(containerName.endsWith("workspace-456user-123")); + t.false(containerName.includes(":")); +}); + // Helper function to check file size in GCS async function getGCSFileSize(gcsUrl) { if (!isGCSConfigured()) return null; diff --git a/helper-apps/cortex-file-handler/tests/checkHashShortLived.test.js b/helper-apps/cortex-file-handler/tests/checkHashShortLived.test.js index f7c30678..ea7e9678 100644 --- a/helper-apps/cortex-file-handler/tests/checkHashShortLived.test.js +++ b/helper-apps/cortex-file-handler/tests/checkHashShortLived.test.js @@ -557,4 +557,83 @@ test.serial("checkHash with different file types should return shortLivedUrl", a } } } +}); + +test.serial("checkHash should copy blob to new chat folder when folder context differs", async (t) => { + if (!isUsingAzureStorage()) { + t.pass("Skipping test - requires Azure storage (including Azurite)"); + return; + } + + const testContent = "Content for cross-chat copy test"; + const filePath = await createTestFile(testContent); + const hash = `test-crosschat-${uuidv4()}`; + const userId = `testuser-${uuidv4().slice(0, 8)}`; + const chat1 = `chat1-${uuidv4().slice(0, 8)}`; + const chat2 = `chat2-${uuidv4().slice(0, 8)}`; + + try { + // Upload file with userId, chatId=chat1, fileScope=chat + const form = new FormData(); + form.append("hash", hash); + form.append("userId", userId); + form.append("chatId", chat1); + form.append("fileScope", "chat"); + form.append("file", fs.createReadStream(filePath)); + + const uploadResponse = await axios.post(baseUrl, form, { + headers: form.getHeaders(), + validateStatus: (status) => true, + timeout: 15000, + }); + t.is(uploadResponse.status, 200, "Upload should succeed"); + t.truthy(uploadResponse.data.url, "Upload should return URL"); + + // Verify the uploaded blob is in chats/chat1/ folder + const uploadedBlobPath = decodeURIComponent(new URL(uploadResponse.data.url).pathname); + t.true(uploadedBlobPath.includes(`chats/${chat1}/`), "Uploaded file should be in chat1 folder"); + + // checkHash with chatId=chat2 — should copy blob to chats/chat2/ + const checkResponse = await axios.get(baseUrl, { + params: { + hash, + checkHash: true, + userId, + chatId: chat2, + fileScope: "chat", + }, + validateStatus: (status) => true, + timeout: 15000, + }); + + t.is(checkResponse.status, 200, "checkHash should succeed"); + t.truthy(checkResponse.data.url, "checkHash should return URL"); + + // Verify the returned URL now points to chats/chat2/ + const returnedBlobPath = decodeURIComponent(new URL(checkResponse.data.url).pathname); + t.true(returnedBlobPath.includes(`chats/${chat2}/`), "Returned URL should be in chat2 folder"); + t.false(returnedBlobPath.includes(`chats/${chat1}/`), "Returned URL should NOT be in chat1 folder"); + + // listFolder for chat2 should show the file + const listResponse = await axios.get(baseUrl, { + params: { + listFolder: true, + userId, + chatId: chat2, + fileScope: "chat", + }, + validateStatus: (status) => true, + timeout: 10000, + }); + + t.is(listResponse.status, 200, "listFolder should succeed"); + t.true(listResponse.data.count >= 1, "chat2 folder should have at least 1 file"); + const fileNames = listResponse.data.files.map(f => f.name); + const hasFile = fileNames.some(n => n.includes(`chats/${chat2}/`)); + t.true(hasFile, "listFolder for chat2 should include the copied file"); + + } finally { + fs.unlinkSync(filePath); + await cleanupHashAndFile(hash, null, baseUrl); + } }); \ No newline at end of file diff --git a/helper-apps/cortex-file-handler/tests/cleanup.test.js b/helper-apps/cortex-file-handler/tests/cleanup.test.js index 7d8b8358..342876ba 100644 --- a/helper-apps/cortex-file-handler/tests/cleanup.test.js +++ b/helper-apps/cortex-file-handler/tests/cleanup.test.js @@ -80,13 +80,7 @@ function getRequestIdFromUploadResult(uploadResult) { // Ensure server is ready before tests test.before(async () => { // Start the server with Redis connection setup - await startTestServer({ - beforeReady: async () => { - // Ensure Redis is connected - const { connectClient } = await import("../src/redis.js"); - await connectClient(); - } - }); + await startTestServer(); }); test.after(async () => { diff --git a/helper-apps/cortex-file-handler/tests/containerConversionFlow.test.js b/helper-apps/cortex-file-handler/tests/containerConversionFlow.test.js index 9b26adc2..57c152b6 100644 --- a/helper-apps/cortex-file-handler/tests/containerConversionFlow.test.js +++ b/helper-apps/cortex-file-handler/tests/containerConversionFlow.test.js @@ -11,7 +11,6 @@ import { port } from "../src/start.js"; import { uploadBlob, AZURE_STORAGE_CONTAINER_NAME, - saveFileToBlob, } from "../src/blobHandler.js"; import { FileConversionService } from "../src/services/FileConversionService.js"; import CortexFileHandler from "../src/index.js"; diff --git a/helper-apps/cortex-file-handler/tests/fileChunker.test.js b/helper-apps/cortex-file-handler/tests/fileChunker.test.js index 8a2ed354..05a2c1bb 100644 --- a/helper-apps/cortex-file-handler/tests/fileChunker.test.js +++ b/helper-apps/cortex-file-handler/tests/fileChunker.test.js @@ -156,6 +156,21 @@ test("respects custom chunk duration", async (t) => { t.deepEqual(chunkOffsets, [0, 5], "Should have correct offset points"); }); +test("supports opt-in overlapped media chunks", async (t) => { + const { chunkPromises, chunkOffsets, uniqueOutputPath } = await splitMediaFile( + t.context.testFile10s, + 5, + undefined, + 1, + ); + + t.is(chunkPromises.length, 2, "Should create correct number of chunks"); + t.deepEqual(chunkOffsets, [0, 4], "Second chunk should include 1s overlap"); + + await Promise.all(chunkPromises); + await fs.rm(uniqueOutputPath, { recursive: true, force: true }); +}); + // Test URL-based file processing test("processes media file from URL", async (t) => { const url = "https://example.com/media/test.mp3"; diff --git a/helper-apps/cortex-file-handler/tests/fileUpload.test.js b/helper-apps/cortex-file-handler/tests/fileUpload.test.js index 2fc959af..8cd77b63 100644 --- a/helper-apps/cortex-file-handler/tests/fileUpload.test.js +++ b/helper-apps/cortex-file-handler/tests/fileUpload.test.js @@ -47,12 +47,9 @@ async function uploadFile(filePath, requestId = null, hash = null) { if (hash) form.append("hash", hash); const response = await axios.post(baseUrl, form, { - headers: { - ...form.getHeaders(), - "Content-Type": "multipart/form-data", - }, + headers: form.getHeaders(), validateStatus: (status) => true, - timeout: 30000, + timeout: 120000, maxContentLength: Infinity, maxBodyLength: Infinity, }); diff --git a/helper-apps/cortex-file-handler/tests/filenameUtils.test.js b/helper-apps/cortex-file-handler/tests/filenameUtils.test.js new file mode 100644 index 00000000..9ac4554b --- /dev/null +++ b/helper-apps/cortex-file-handler/tests/filenameUtils.test.js @@ -0,0 +1,16 @@ +import test from "ava"; + +import { sanitizeFilename } from "../src/utils/filenameUtils.js"; + +test("sanitizeFilename strips C1 control characters from mojibake names", (t) => { + const result = sanitizeFilename("اهل 😍.mp4"); + + t.false(/[\u0000-\u001F\u007F-\u009F]/.test(result)); + t.true(result.endsWith(".mp4")); +}); + +test("sanitizeFilename preserves valid Arabic filenames", (t) => { + const result = sanitizeFilename("أهل الجنوب.mp4"); + + t.is(result, "أهل الجنوب.mp4"); +}); diff --git a/helper-apps/cortex-file-handler/tests/folderStorage.test.js b/helper-apps/cortex-file-handler/tests/folderStorage.test.js new file mode 100644 index 00000000..dd470d1c --- /dev/null +++ b/helper-apps/cortex-file-handler/tests/folderStorage.test.js @@ -0,0 +1,698 @@ +import test from "ava"; +import fs from "fs"; +import path from "path"; +import { Readable } from "stream"; +import { fileURLToPath } from "url"; +import { constructFolderPath, sanitizeSubPath } from "../src/blobHandler.js"; +import { AzureStorageProvider } from "../src/services/storage/AzureStorageProvider.js"; +import { GCSStorageProvider } from "../src/services/storage/GCSStorageProvider.js"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); + +// ============================================================================ +// constructFolderPath tests +// ============================================================================ + +test("constructFolderPath › should return null when userId is missing and scope is not workspace-shared-legacy", (t) => { + t.is(constructFolderPath({}), null); + t.is(constructFolderPath({ chatId: "chat123" }), null); + t.is(constructFolderPath({ workspaceId: "ws123" }), null); + t.is(constructFolderPath({ fileScope: "global" }), null); + t.is(constructFolderPath({ fileScope: "chat", chatId: "chat123" }), null); +}); + +test("constructFolderPath › should return global path for userId only", (t) => { + const result = constructFolderPath({ userId: "user123" }); + t.is(result, "global"); +}); + +test("sanitizeSubPath › allows safe nested folder paths", (t) => { + t.is(sanitizeSubPath("versions/applet123"), "versions/applet123"); + t.is(sanitizeSubPath("/versions/applet123/"), "versions/applet123"); +}); + +test("sanitizeSubPath › rejects traversal and unsafe folder paths", (t) => { + t.is(sanitizeSubPath("../versions/applet123"), null); + t.is(sanitizeSubPath("versions/../../secret"), null); + t.is(sanitizeSubPath("versions/applet.123"), null); +}); + +test("constructFolderPath › should return global path for explicit global scope", (t) => { + const result = constructFolderPath({ userId: "user123", fileScope: "global" }); + t.is(result, "global"); +}); + +test("constructFolderPath › should return chat path when chatId is provided with chat scope", (t) => { + const result = constructFolderPath({ + userId: "user123", + chatId: "chat456", + fileScope: "chat" + }); + t.is(result, "chats/chat456"); +}); + +test("constructFolderPath › should fall back to global when chat scope but no chatId", (t) => { + const result = constructFolderPath({ + userId: "user123", + fileScope: "chat" + }); + t.is(result, "global"); +}); + +test("constructFolderPath › should return global for unknown scope with userId", (t) => { + const result = constructFolderPath({ + userId: "user123", + fileScope: "unknown-scope" + }); + t.is(result, "global"); +}); + +test("constructFolderPath › should return empty string for all scope", (t) => { + const result = constructFolderPath({ + userId: "user123", + fileScope: "all" + }); + t.is(result, ""); +}); + +test("constructFolderPath › should handle all parameters together", (t) => { + // Chat scope takes precedence when chatId is provided + const chatResult = constructFolderPath({ + userId: "user123", + chatId: "chat456", + fileScope: "chat" + }); + t.is(chatResult, "chats/chat456"); +}); + +test("constructFolderPath › should return workspace-user-legacy path with workspaceId", (t) => { + const result = constructFolderPath({ + userId: "user123", + workspaceId: "ws789", + fileScope: "workspace-user-legacy" + }); + t.is(result, "applets/ws789"); +}); + +test("constructFolderPath › should fall back to global for workspace-user-legacy scope without workspaceId", (t) => { + const result = constructFolderPath({ + userId: "user123", + fileScope: "workspace-user-legacy" + }); + t.is(result, "global"); +}); + +test("constructFolderPath › should return profile path", (t) => { + const result = constructFolderPath({ + userId: "user123", + fileScope: "profile" + }); + t.is(result, "profile"); +}); + +test("constructFolderPath › should return media path", (t) => { + const result = constructFolderPath({ + userId: "user123", + fileScope: "media" + }); + t.is(result, "media"); +}); + +test("constructFolderPath › media path is suitable for remote-file uploads", (t) => { + const folderPath = constructFolderPath({ + userId: "user123", + fileScope: "media" + }); + const blobName = folderPath ? `${folderPath}/mmlhat5b-shz.webp` : "mmlhat5b-shz.webp"; + t.is(blobName, "media/mmlhat5b-shz.webp"); +}); + +test("constructFolderPath › should return articles path", (t) => { + const result = constructFolderPath({ + userId: "user123", + fileScope: "articles" + }); + t.is(result, "articles"); +}); + +test("constructFolderPath › should return applets path for user-scoped applets scope", (t) => { + const result = constructFolderPath({ + userId: "user123", + fileScope: "applets" + }); + t.is(result, "applets"); +}); + +test("constructFolderPath › should route applet-user scope into an applet-specific folder", (t) => { + const result = constructFolderPath({ + userId: "user123", + appletId: "applet456", + fileScope: "applet-user" + }); + t.is(result, "applets/applet456"); +}); + +test("constructFolderPath › should derive the applet-user folder from scoped contextId", (t) => { + const result = constructFolderPath({ + contextId: "applet-user:applet456:user123", + fileScope: "applet-user" + }); + t.is(result, "applets/applet456"); +}); + +test("constructFolderPath › workspace-shared-legacy scope should return empty string (root)", (t) => { + const result = constructFolderPath({ + workspaceId: "ws789", + fileScope: "workspace-shared-legacy" + }); + t.is(result, ""); +}); + +test("constructFolderPath › workspace-shared-legacy scope should return null without workspaceId", (t) => { + const result = constructFolderPath({ + fileScope: "workspace-shared-legacy" + }); + t.is(result, null); +}); + +test("constructFolderPath › workspace-shared-legacy scope should not require userId", (t) => { + // workspace-shared-legacy scopes the container by workspaceId, not userId + const result = constructFolderPath({ + workspaceId: "ws789", + fileScope: "workspace-shared-legacy" + }); + t.is(result, ""); +}); + +test("constructFolderPath › should reject invalid workspaceId for workspace-shared-legacy", (t) => { + const result = constructFolderPath({ + workspaceId: "../escape", + fileScope: "workspace-shared-legacy" + }); + t.is(result, null); +}); + +test("constructFolderPath › should reject invalid workspaceId for workspace-user-legacy scope", (t) => { + const result = constructFolderPath({ + userId: "user123", + workspaceId: "bad/path", + fileScope: "workspace-user-legacy" + }); + t.is(result, null); +}); + +// ============================================================================ +// Azure listFolder tests +// ============================================================================ + +test("Azure listFolder › should list files in a folder", async (t) => { + if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { + t.pass("Skipping test - Azure not configured"); + return; + } + + const provider = new AzureStorageProvider( + process.env.AZURE_STORAGE_CONNECTION_STRING, + process.env.AZURE_STORAGE_CONTAINER_NAME || "test-container" + ); + + const testFolder = `test-folder-${Date.now()}`; + const testFiles = []; + + try { + // Upload some test files to a folder + for (let i = 0; i < 3; i++) { + const content = `Test content ${i}`; + const stream = Readable.from([content]); + const filename = `file${i}.txt`; + const folderPath = testFolder; + + await provider.uploadStream( + { log: () => {} }, + filename, + stream, + "text/plain", + "temporary", + folderPath + ); + testFiles.push(`${testFolder}/${filename}`); + } + + // List the folder + const files = await provider.listFolder(testFolder); + + t.is(files.length, 3); + for (const file of files) { + t.true(file.name.startsWith(testFolder + "/")); + t.truthy(file.filename); + t.truthy(file.lastModified); + t.is(file.contentType, "text/plain; charset=utf-8"); + t.truthy(file.size); + } + } finally { + // Cleanup: delete the test folder contents + const { containerClient } = await provider.getBlobClient(); + for await (const blob of containerClient.listBlobsFlat({ prefix: testFolder })) { + const blockBlobClient = containerClient.getBlockBlobClient(blob.name); + await blockBlobClient.delete().catch(() => {}); + } + } +}); + +test("Azure listFolder › should return empty array for non-existent folder", async (t) => { + if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { + t.pass("Skipping test - Azure not configured"); + return; + } + + const provider = new AzureStorageProvider( + process.env.AZURE_STORAGE_CONNECTION_STRING, + process.env.AZURE_STORAGE_CONTAINER_NAME || "test-container" + ); + + const files = await provider.listFolder("non-existent-folder-12345"); + t.deepEqual(files, []); +}); + +test("Azure listFolder › should extract hash from filename with hash prefix", async (t) => { + if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { + t.pass("Skipping test - Azure not configured"); + return; + } + + const provider = new AzureStorageProvider( + process.env.AZURE_STORAGE_CONNECTION_STRING, + process.env.AZURE_STORAGE_CONTAINER_NAME || "test-container" + ); + + const testFolder = `test-hash-folder-${Date.now()}`; + + try { + // Upload a file with hash prefix format: {hash}_{filename} + const content = "Test content with hash"; + const stream = Readable.from([content]); + const hashPrefix = "abc123def456"; + const originalFilename = "document.pdf"; + const filename = `${hashPrefix}_${originalFilename}`; + + await provider.uploadStream( + { log: () => {} }, + filename, + stream, + "application/pdf", + "temporary", + testFolder + ); + + // List the folder and verify hash extraction + const files = await provider.listFolder(testFolder); + + t.is(files.length, 1); + t.is(files[0].hash, hashPrefix); + t.is(files[0].filename, originalFilename); + } finally { + // Cleanup + const { containerClient } = await provider.getBlobClient(); + for await (const blob of containerClient.listBlobsFlat({ prefix: testFolder })) { + const blockBlobClient = containerClient.getBlockBlobClient(blob.name); + await blockBlobClient.delete().catch(() => {}); + } + } +}); + +test("Azure listFolder › should handle files without hash prefix", async (t) => { + if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { + t.pass("Skipping test - Azure not configured"); + return; + } + + const provider = new AzureStorageProvider( + process.env.AZURE_STORAGE_CONNECTION_STRING, + process.env.AZURE_STORAGE_CONTAINER_NAME || "test-container" + ); + + const testFolder = `test-no-hash-folder-${Date.now()}`; + + try { + // Upload a file without hash prefix + const content = "Test content no hash"; + const stream = Readable.from([content]); + const filename = "simple-file.txt"; + + await provider.uploadStream( + { log: () => {} }, + filename, + stream, + "text/plain", + "temporary", + testFolder + ); + + // List the folder + const files = await provider.listFolder(testFolder); + + t.is(files.length, 1); + t.is(files[0].hash, null); + t.is(files[0].filename, filename); + } finally { + // Cleanup + const { containerClient } = await provider.getBlobClient(); + for await (const blob of containerClient.listBlobsFlat({ prefix: testFolder })) { + const blockBlobClient = containerClient.getBlockBlobClient(blob.name); + await blockBlobClient.delete().catch(() => {}); + } + } +}); + +test("Azure uploadStream › should create folder hierarchy with folderPath", async (t) => { + if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { + t.pass("Skipping test - Azure not configured"); + return; + } + + const provider = new AzureStorageProvider( + process.env.AZURE_STORAGE_CONNECTION_STRING, + process.env.AZURE_STORAGE_CONTAINER_NAME || "test-container" + ); + + const chatId = "chat456"; + const folderPath = `chats/${chatId}`; + const testFolder = `chats`; + + try { + const content = "Test file in folder hierarchy"; + const stream = Readable.from([content]); + const filename = "nested-file.txt"; + + const result = await provider.uploadStream( + { log: () => {} }, + filename, + stream, + "text/plain", + "temporary", + folderPath + ); + + t.truthy(result.url); + // URL should contain the folder path components (may be URL-encoded) + t.true(result.url.includes("chats")); + + // Verify file is in the correct folder + const files = await provider.listFolder(folderPath); + t.true(files.length >= 1); + t.true(files.some(f => f.filename === filename)); + } finally { + // Cleanup + const { containerClient } = await provider.getBlobClient(); + for await (const blob of containerClient.listBlobsFlat({ prefix: testFolder })) { + const blockBlobClient = containerClient.getBlockBlobClient(blob.name); + await blockBlobClient.delete().catch(() => {}); + } + } +}); + +// ============================================================================ +// GCS listFolder tests +// ============================================================================ + +test("GCS listFolder › should list files in a folder", async (t) => { + if ( + !process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 && + !process.env.GCP_SERVICE_ACCOUNT_KEY + ) { + t.pass("Skipping test - GCS not configured"); + return; + } + + const credentials = JSON.parse( + process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 + ? Buffer.from(process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64, "base64").toString() + : process.env.GCP_SERVICE_ACCOUNT_KEY + ); + + const provider = new GCSStorageProvider( + credentials, + process.env.GCS_BUCKETNAME || "cortextempfiles" + ); + + const testFolder = `test-folder-${Date.now()}`; + + try { + // Upload some test files to a folder + for (let i = 0; i < 3; i++) { + const content = `Test content ${i}`; + const stream = Readable.from([content]); + const filename = `file${i}.txt`; + + await provider.uploadStream( + { log: () => {} }, + filename, + stream, + "text/plain", + "temporary", + testFolder + ); + } + + // List the folder + const files = await provider.listFolder(testFolder); + + t.is(files.length, 3); + for (const file of files) { + t.true(file.name.startsWith(testFolder + "/")); + t.truthy(file.filename); + t.true(file.contentType.startsWith("text/plain")); + } + } finally { + // Cleanup + await provider.deleteFiles(testFolder); + } +}); + +test("GCS listFolder › should return empty array for non-existent folder", async (t) => { + if ( + !process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 && + !process.env.GCP_SERVICE_ACCOUNT_KEY + ) { + t.pass("Skipping test - GCS not configured"); + return; + } + + const credentials = JSON.parse( + process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 + ? Buffer.from(process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64, "base64").toString() + : process.env.GCP_SERVICE_ACCOUNT_KEY + ); + + const provider = new GCSStorageProvider( + credentials, + process.env.GCS_BUCKETNAME || "cortextempfiles" + ); + + const files = await provider.listFolder("non-existent-folder-12345"); + t.deepEqual(files, []); +}); + +test("GCS listFolder › should extract hash from filename with hash prefix", async (t) => { + if ( + !process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 && + !process.env.GCP_SERVICE_ACCOUNT_KEY + ) { + t.pass("Skipping test - GCS not configured"); + return; + } + + const credentials = JSON.parse( + process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 + ? Buffer.from(process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64, "base64").toString() + : process.env.GCP_SERVICE_ACCOUNT_KEY + ); + + const provider = new GCSStorageProvider( + credentials, + process.env.GCS_BUCKETNAME || "cortextempfiles" + ); + + const testFolder = `test-hash-folder-${Date.now()}`; + + try { + // Upload a file with hash prefix format: {hash}_{filename} + const content = "Test content with hash"; + const stream = Readable.from([content]); + const hashPrefix = "abc123def456"; + const originalFilename = "document.pdf"; + const filename = `${hashPrefix}_${originalFilename}`; + + await provider.uploadStream( + { log: () => {} }, + filename, + stream, + "application/pdf", + "temporary", + testFolder + ); + + // List the folder and verify hash extraction + const files = await provider.listFolder(testFolder); + + t.is(files.length, 1); + t.is(files[0].hash, hashPrefix); + t.is(files[0].filename, originalFilename); + } finally { + // Cleanup + await provider.deleteFiles(testFolder); + } +}); + +test("GCS uploadStream › should create folder hierarchy with folderPath", async (t) => { + if ( + !process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 && + !process.env.GCP_SERVICE_ACCOUNT_KEY + ) { + t.pass("Skipping test - GCS not configured"); + return; + } + + const credentials = JSON.parse( + process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 + ? Buffer.from(process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64, "base64").toString() + : process.env.GCP_SERVICE_ACCOUNT_KEY + ); + + const provider = new GCSStorageProvider( + credentials, + process.env.GCS_BUCKETNAME || "cortextempfiles" + ); + + const chatId = "chat456"; + const folderPath = `chats/${chatId}`; + const testFolder = `chats`; + + try { + const content = "Test file in folder hierarchy"; + const stream = Readable.from([content]); + const filename = "nested-file.txt"; + + const result = await provider.uploadStream( + { log: () => {} }, + filename, + stream, + "text/plain", + "temporary", + folderPath + ); + + t.truthy(result); + t.true(result.includes("gs://")); + t.true(result.includes(folderPath)); + + // Verify file is in the correct folder + const files = await provider.listFolder(folderPath); + t.true(files.length >= 1); + t.true(files.some(f => f.filename === filename)); + } finally { + // Cleanup + await provider.deleteFiles(testFolder); + } +}); + +// ============================================================================ +// Edge case tests +// ============================================================================ + +test("Azure listFolder › should handle nested folder paths", async (t) => { + if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { + t.pass("Skipping test - Azure not configured"); + return; + } + + const provider = new AzureStorageProvider( + process.env.AZURE_STORAGE_CONNECTION_STRING, + process.env.AZURE_STORAGE_CONTAINER_NAME || "test-container" + ); + + const baseFolder = `test-nested-${Date.now()}`; + const nestedFolder = `${baseFolder}/level1/level2/level3`; + + try { + // Upload file to deeply nested folder + const content = "Deeply nested content"; + const stream = Readable.from([content]); + const filename = "deep-file.txt"; + + await provider.uploadStream( + { log: () => {} }, + filename, + stream, + "text/plain", + "temporary", + nestedFolder + ); + + // List should find file at exact path + const exactFiles = await provider.listFolder(nestedFolder); + t.is(exactFiles.length, 1); + t.is(exactFiles[0].filename, filename); + + // Blob storage prefix matching finds all files with that prefix (including nested) + // This is expected behavior - listFolder returns all files under the folder path + const baseFiles = await provider.listFolder(baseFolder); + t.is(baseFiles.length, 1); // File is found via prefix match + + // Verify the full blob name shows the nested path + t.true(baseFiles[0].name.includes("level1/level2/level3")); + } finally { + // Cleanup + const { containerClient } = await provider.getBlobClient(); + for await (const blob of containerClient.listBlobsFlat({ prefix: baseFolder })) { + const blockBlobClient = containerClient.getBlockBlobClient(blob.name); + await blockBlobClient.delete().catch(() => {}); + } + } +}); + +test("Azure listFolder › should normalize folder paths with leading/trailing slashes", async (t) => { + if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { + t.pass("Skipping test - Azure not configured"); + return; + } + + const provider = new AzureStorageProvider( + process.env.AZURE_STORAGE_CONNECTION_STRING, + process.env.AZURE_STORAGE_CONTAINER_NAME || "test-container" + ); + + const testFolder = `test-normalize-${Date.now()}`; + + try { + // Upload a file + const content = "Normalization test"; + const stream = Readable.from([content]); + const filename = "norm-file.txt"; + + await provider.uploadStream( + { log: () => {} }, + filename, + stream, + "text/plain", + "temporary", + testFolder + ); + + // List with trailing slash + const filesWithSlash = await provider.listFolder(`${testFolder}/`); + t.is(filesWithSlash.length, 1); + + // List without trailing slash + const filesWithoutSlash = await provider.listFolder(testFolder); + t.is(filesWithoutSlash.length, 1); + } finally { + // Cleanup + const { containerClient } = await provider.getBlobClient(); + for await (const blob of containerClient.listBlobsFlat({ prefix: testFolder })) { + const blockBlobClient = containerClient.getBlockBlobClient(blob.name); + await blockBlobClient.delete().catch(() => {}); + } + } +}); diff --git a/helper-apps/cortex-file-handler/tests/getOperations.test.js b/helper-apps/cortex-file-handler/tests/getOperations.test.js index e7af9e96..226e2497 100644 --- a/helper-apps/cortex-file-handler/tests/getOperations.test.js +++ b/helper-apps/cortex-file-handler/tests/getOperations.test.js @@ -8,12 +8,21 @@ import FormData from "form-data"; import XLSX from "xlsx"; import nock from "nock"; import { port } from "../src/start.js"; +import { gcsUrlExists } from "../src/blobHandler.js"; +import { getDefaultContainerName, getUserContainerName } from "../src/constants.js"; import { cleanupHashAndFile, createTestMediaFile, startTestServer, stopTestServer, setupTestDirectory } from "./testUtils.helper.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); const baseUrl = `http://localhost:${port}/api/CortexFileHandler`; +function isGCSConfigured() { + return ( + process.env.GCP_SERVICE_ACCOUNT_KEY_BASE64 || + process.env.GCP_SERVICE_ACCOUNT_KEY + ); +} + // Helper function to create test files async function createTestFile(content, extension) { const testDir = path.join(__dirname, "test-files"); @@ -50,6 +59,35 @@ async function uploadFile(filePath, requestId = null, hash = null) { return response; } +async function uploadScopedFile(filePath, { + contextId = null, + userId = null, + workspaceId = null, + appletId = null, + fileScope = null, + hash = null, +} = {}) { + const form = new FormData(); + if (contextId) form.append("contextId", contextId); + if (userId) form.append("userId", userId); + if (workspaceId) form.append("workspaceId", workspaceId); + if (appletId) form.append("appletId", appletId); + if (fileScope) form.append("fileScope", fileScope); + if (hash) form.append("hash", hash); + form.append("file", fs.createReadStream(filePath)); + + return await axios.post(baseUrl, form, { + headers: { + ...form.getHeaders(), + "Content-Type": "multipart/form-data", + }, + validateStatus: (status) => true, + timeout: 30000, + maxContentLength: Infinity, + maxBodyLength: Infinity, + }); +} + // Setup: Create test directory and start server test.before(async (t) => { await startTestServer(); @@ -260,6 +298,109 @@ test.serial("should fetch remote file", async (t) => { t.true(scope.isDone(), "All external requests should be mocked and used"); }); +test.serial("applet-user uploads use the user container and stay isolated by applet folder", async (t) => { + const fileA = await createTestFile("alpha applet content", "txt"); + const fileB = await createTestFile("beta applet content", "txt"); + const userId = `user-${uuidv4().slice(0, 8)}`; + const appletA = `applet-${uuidv4().slice(0, 6)}`; + const appletB = `applet-${uuidv4().slice(0, 6)}`; + const hashA = "aaaaaaaaaaaaaaaa"; + const hashB = "bbbbbbbbbbbbbbbb"; + const contextIdA = `applet-user:${appletA}:${userId}`; + const contextIdB = `applet-user:${appletB}:${userId}`; + + let responseA; + let responseB; + + try { + responseA = await uploadScopedFile(fileA, { + contextId: contextIdA, + userId, + appletId: appletA, + fileScope: "applet-user", + hash: hashA, + }); + responseB = await uploadScopedFile(fileB, { + contextId: contextIdB, + userId, + appletId: appletB, + fileScope: "applet-user", + hash: hashB, + }); + + t.is(responseA.status, 200); + t.is(responseB.status, 200); + + const userContainerName = getUserContainerName( + getDefaultContainerName(), + userId, + ); + + if (!responseA.data.url.startsWith("http://localhost:7071/files/")) { + t.true(responseA.data.url.includes(`/${userContainerName}/`)); + t.true(responseB.data.url.includes(`/${userContainerName}/`)); + t.false(responseA.data.url.includes(`applet-user-${appletA}-${userId}`)); + t.false(responseB.data.url.includes(`applet-user-${appletB}-${userId}`)); + } + t.is(responseA.data.folderPath, `applets/${appletA}`); + t.is(responseB.data.folderPath, `applets/${appletB}`); + t.is(responseA.data.contextId, contextIdA); + t.is(responseB.data.contextId, contextIdB); + + const listA = await axios.get(baseUrl, { + params: { + listFolder: true, + userId, + appletId: appletA, + fileScope: "applet-user", + }, + validateStatus: () => true, + timeout: 30000, + }); + const listB = await axios.get(baseUrl, { + params: { + listFolder: true, + userId, + appletId: appletB, + fileScope: "applet-user", + }, + validateStatus: () => true, + timeout: 30000, + }); + + if (listA.status === 200 && listB.status === 200) { + t.is(listA.data.folderPath, `applets/${appletA}`); + t.is(listB.data.folderPath, `applets/${appletB}`); + t.is(listA.data.count, 1); + t.is(listB.data.count, 1); + t.is(listA.data.files[0].hash, hashA); + t.is(listB.data.files[0].hash, hashB); + } else { + t.is(listA.status, 500); + t.is(listB.status, 500); + t.is(listA.data, "Storage provider does not support folder listing"); + t.is(listB.data, "Storage provider does not support folder listing"); + } + } finally { + if (responseA?.data?.hash) { + await axios.delete(baseUrl, { + params: { hash: responseA.data.hash, contextId: contextIdA }, + validateStatus: () => true, + timeout: 10000, + }); + } + if (responseB?.data?.hash) { + await axios.delete(baseUrl, { + params: { hash: responseB.data.hash, contextId: contextIdB }, + validateStatus: () => true, + timeout: 10000, + }); + } + fs.unlinkSync(fileA); + fs.unlinkSync(fileB); + } +}); + // Test: Redis caching behavior for remote files test.serial("should cache remote files in Redis", async (t) => { const requestId = uuidv4(); @@ -387,3 +528,82 @@ test.serial("should handle long filenames", async (t) => { } } }); + +// Test: blobPath lookup should return a short-lived URL +test.serial("should return short-lived URL for blobPath lookup", async (t) => { + const fileContent = "blobPath test content"; + const filePath = await createTestFile(fileContent, "txt"); + const requestId = uuidv4(); + let response; + + try { + // Upload a file first + response = await uploadFile(filePath, requestId); + t.is(response.status, 200, "Upload should succeed"); + + // Extract the blob path from the upload URL + const uploadUrl = response.data.url; + let extractedBlobPath; + + if (uploadUrl.includes("127.0.0.1:10000")) { + // Azurite URL: http://127.0.0.1:10000/devstoreaccount1/container/blobpath + const urlObj = new URL(uploadUrl); + const parts = urlObj.pathname.split("/").filter(Boolean); + // Skip account name and container name + extractedBlobPath = parts.slice(2).join("/"); + } else if (uploadUrl.includes("blob.core.windows.net")) { + // Azure URL: https://account.blob.core.windows.net/container/blobpath + const urlObj = new URL(uploadUrl.split("?")[0]); + const parts = urlObj.pathname.split("/").filter(Boolean); + // Skip container name + extractedBlobPath = parts.slice(1).join("/"); + } else { + // Local URL: http://localhost:port/files/requestId/filename + const urlObj = new URL(uploadUrl); + const parts = urlObj.pathname.split("/").filter(Boolean); + // Skip "files" prefix + extractedBlobPath = parts.slice(1).join("/"); + } + + t.truthy(extractedBlobPath, "Should extract blob path from URL"); + + // Look up by blobPath (no hash) + const lookupResponse = await axios.get(baseUrl, { + params: { blobPath: extractedBlobPath }, + validateStatus: (status) => true, + }); + + t.is(lookupResponse.status, 200, "blobPath lookup should succeed"); + t.truthy(lookupResponse.data.shortLivedUrl, "Should return shortLivedUrl"); + t.truthy(lookupResponse.data.url, "Should return url"); + if (isGCSConfigured()) { + t.truthy(lookupResponse.data.gcs, "Should return a GCS backup URL"); + t.true( + await gcsUrlExists(lookupResponse.data.gcs), + "Returned GCS backup URL should exist", + ); + } + + // Verify the short-lived URL is accessible and returns correct content + const fileResponse = await axios.get(lookupResponse.data.shortLivedUrl, { + validateStatus: (status) => true, + }); + t.is(fileResponse.status, 200, "Short-lived URL should be accessible"); + t.true(fileResponse.data.includes(fileContent), "Should return correct file content"); + } finally { + fs.unlinkSync(filePath); + if (response?.data?.url) { + await cleanupHashAndFile(null, response.data.url, baseUrl); + } + } +}); + +// Test: blobPath lookup should return 404 for non-existent blob +test.serial("should return 404 for non-existent blobPath", async (t) => { + const response = await axios.get(baseUrl, { + params: { blobPath: `non-existent-${uuidv4()}/fake-file.txt` }, + validateStatus: (status) => true, + }); + + t.is(response.status, 404, "Should return 404 for non-existent blobPath"); +}); diff --git a/helper-apps/cortex-file-handler/tests/legacyWorkspacePrivateResolver.test.js b/helper-apps/cortex-file-handler/tests/legacyWorkspacePrivateResolver.test.js new file mode 100644 index 00000000..00f48329 --- /dev/null +++ b/helper-apps/cortex-file-handler/tests/legacyWorkspacePrivateResolver.test.js @@ -0,0 +1,357 @@ +import test from "ava"; + +import { + getLegacyWorkspacePrivateContextId, + migrateHashRecordToScopedStorage, + resolveBlobPathWithLegacyFallback, + resolveHashRecordWithLegacyWorkspacePrivateFallback, +} from "../src/utils/legacyWorkspacePrivateResolver.js"; +import { + listLegacyScopedFolderFiles, + resolveLegacyScopedBlobClient, +} from "../src/index.js"; +import { StorageFactory } from "../src/services/storage/StorageFactory.js"; +import { + getDefaultContainerName, + getLegacyUserContainerName, + getUserContainerName, +} from "../src/constants.js"; + +const originalGetInstance = StorageFactory.getInstance; + +test.afterEach.always(() => { + StorageFactory.getInstance = originalGetInstance; + StorageFactory.resetInstance(); +}); + +test("getLegacyWorkspacePrivateContextId returns the legacy compound context for workspace-user-legacy scope", (t) => { + t.is( + getLegacyWorkspacePrivateContextId({ + userId: "user-123", + workspaceId: "workspace-456", + fileScope: "workspace-user-legacy", + }), + "workspace-456:user-123", + ); + + t.is( + getLegacyWorkspacePrivateContextId({ + userId: "user-123", + workspaceId: "workspace-456", + fileScope: "chat", + }), + null, + ); +}); + +test("resolveHashRecordWithLegacyWorkspacePrivateFallback probes the legacy compound context", async (t) => { + const calls = []; + const result = await resolveHashRecordWithLegacyWorkspacePrivateFallback({ + hash: "hash-123", + resolvedContextId: "user-123", + userId: "user-123", + workspaceId: "workspace-456", + fileScope: "workspace-user-legacy", + getFileStoreMap: async (...args) => { + calls.push(args); + return { + url: "https://legacy.example/file.pdf", + filename: "file.pdf", + }; + }, + }); + + t.deepEqual(calls, [["hash-123", true, "workspace-456:user-123"]]); + t.deepEqual(result, { + hashResult: { + url: "https://legacy.example/file.pdf", + filename: "file.pdf", + }, + sourceContextId: "workspace-456:user-123", + source: "legacy-workspace-private", + }); +}); + +test("migrateHashRecordToScopedStorage rewrites a legacy workspace-user-private record into the new scoped container", async (t) => { + const uploaded = []; + StorageFactory.getInstance = () => ({ + getAzureProvider: async (containerName) => ({ + containerName, + uploadStream: async ( + context, + filename, + stream, + contentType, + retention, + folderPath, + ) => { + const chunks = []; + for await (const chunk of stream) { + chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); + } + uploaded.push({ + context, + filename, + contentType, + retention, + folderPath, + body: Buffer.concat(chunks).toString("utf8"), + }); + return { + url: `https://example.blob.core.windows.net/${containerName}/${folderPath}/${filename}`, + }; + }, + }), + }); + + const setCalls = []; + const removeCalls = []; + const hash = "hash-123"; + const resolvedContextId = "user-123"; + const sourceContextId = "workspace-456:user-123"; + const context = { log: () => {} }; + + const result = await migrateHashRecordToScopedStorage({ + context, + hash, + hashResult: { + url: "https://legacy.example/file.pdf", + filename: "file.pdf", + displayFilename: "file.pdf", + mimeType: "application/pdf", + permanent: true, + }, + sourceContextId, + resolvedContextId, + userId: "user-123", + workspaceId: "workspace-456", + fileScope: "workspace-user-legacy", + storageService: { + getPrimaryProvider: async () => ({ + constructor: { name: "AzureStorageProvider" }, + }), + downloadFile: async () => Buffer.from("legacy pdf bytes"), + }, + setFileStoreMap: async (...args) => { + setCalls.push(args); + }, + removeFromFileStoreMap: async (...args) => { + removeCalls.push(args); + }, + }); + + const targetContainerName = getUserContainerName( + getDefaultContainerName(), + resolvedContextId, + ); + + t.is(uploaded.length, 1); + t.deepEqual(uploaded[0], { + context, + filename: "file.pdf", + contentType: "application/pdf", + retention: "permanent", + folderPath: "applets/workspace-456", + body: "legacy pdf bytes", + }); + + t.is(result.hash, hash); + t.is(result.filename, "file.pdf"); + t.is(result.blobPath, "applets/workspace-456/file.pdf"); + t.is( + result.url, + `https://example.blob.core.windows.net/${targetContainerName}/applets/workspace-456/file.pdf`, + ); + + t.is(setCalls.length, 1); + t.is(setCalls[0][0], hash); + t.is(setCalls[0][2], resolvedContextId); + t.is(setCalls[0][1].blobPath, "applets/workspace-456/file.pdf"); + t.is(setCalls[0][1].url, result.url); + + t.deepEqual(removeCalls, [[hash, sourceContextId]]); +}); + +test("resolveBlobPathWithLegacyFallback ensures a fresh GCS backup and persists it", async (t) => { + const defaultContainerName = getDefaultContainerName(); + const legacyContextId = "workspace-456:user-123"; + const targetContextId = "user-123"; + const legacyContainerName = getLegacyUserContainerName( + defaultContainerName, + legacyContextId, + ); + const targetContainerName = getUserContainerName( + defaultContainerName, + targetContextId, + ); + const blobPath = "applets/workspace-456/file.pdf"; + + const sourceBlobClient = { + url: `https://example.blob.core.windows.net/${legacyContainerName}/${blobPath}`, + exists: async () => true, + }; + const targetBlobClient = { + url: `https://example.blob.core.windows.net/${targetContainerName}/${blobPath}`, + beginCopyFromURL: async () => ({ + pollUntilDone: async () => {}, + }), + }; + + const providers = new Map([ + [ + defaultContainerName, + { + containerName: defaultContainerName, + ensureInitialized: async () => {}, + getBlobClient: async () => ({ + containerClient: { + getBlockBlobClient: () => ({ + exists: async () => false, + }), + }, + }), + }, + ], + [ + legacyContainerName, + { + containerName: legacyContainerName, + ensureInitialized: async () => {}, + getBlobClient: async () => ({ + containerClient: { + getBlockBlobClient: () => sourceBlobClient, + }, + }), + generateShortLivedSASToken: () => "legacy-sas", + }, + ], + [ + targetContainerName, + { + containerName: targetContainerName, + ensureInitialized: async () => {}, + getBlobClient: async () => ({ + containerClient: { + getBlockBlobClient: () => targetBlobClient, + }, + }), + generateSASToken: () => "long-sas", + generateShortLivedSASToken: () => "short-sas", + }, + ], + ]); + + StorageFactory.getInstance = () => ({ + getAzureProvider: async (containerName) => providers.get(containerName), + }); + + const setCalls = []; + const result = await resolveBlobPathWithLegacyFallback({ + context: { log: () => {} }, + hash: "hash-123", + blobPath, + resolvedContextId: targetContextId, + userId: "user-123", + workspaceId: "workspace-456", + fileScope: "workspace-user-legacy", + storageService: { + ensureGCSUpload: async (context, existingFile) => ({ + ...existingFile, + gcs: "gs://cortextempfiles/fresh-file.pdf", + }), + }, + setFileStoreMap: async (...args) => { + setCalls.push(args); + }, + }); + + t.is( + result.url, + `https://example.blob.core.windows.net/${targetContainerName}/${blobPath}?long-sas`, + ); + t.is( + result.shortLivedUrl, + `https://example.blob.core.windows.net/${targetContainerName}/${blobPath}?short-sas`, + ); + t.is(result.gcs, "gs://cortextempfiles/fresh-file.pdf"); + t.is(setCalls.length, 1); + t.is(setCalls[0][0], "hash-123"); + t.is(setCalls[0][2], targetContextId); + t.is(setCalls[0][1].gcs, "gs://cortextempfiles/fresh-file.pdf"); +}); + +test("listLegacyScopedFolderFiles returns files from the legacy scoped container name", async (t) => { + const containerOwnerId = "shared:user_123"; + const defaultContainerName = getDefaultContainerName(); + const legacyContainerName = getLegacyUserContainerName( + defaultContainerName, + containerOwnerId, + ); + const blobName = "global/abc123_shared-file.txt"; + + StorageFactory.getInstance = () => ({ + getAzureProvider: async (containerName) => ({ + ensureInitialized: async () => {}, + _containerClient: { + listBlobsFlat: async function* ({ prefix }) { + t.is(containerName, legacyContainerName); + t.is(prefix, "global/"); + yield { + name: blobName, + properties: { + lastModified: new Date("2024-01-01T00:00:00Z"), + contentType: "text/plain", + contentLength: 42, + }, + }; + }, + getBlockBlobClient: (currentBlobName) => ({ + url: `https://example.blob.core.windows.net/${containerName}/${currentBlobName}`, + }), + }, + generateShortLivedSASToken: () => "legacy-short-sas", + }), + }); + + const files = await listLegacyScopedFolderFiles(containerOwnerId, "global"); + + t.is(files.length, 1); + t.is(files[0].hash, "abc123"); + t.is(files[0].filename, "shared-file.txt"); + t.true(files[0].url.includes(legacyContainerName)); +}); + +test("resolveLegacyScopedBlobClient finds blobs in the legacy scoped container name", async (t) => { + const containerOwnerId = "shared:user_123"; + const defaultContainerName = getDefaultContainerName(); + const legacyContainerName = getLegacyUserContainerName( + defaultContainerName, + containerOwnerId, + ); + const blobPath = "global/abc123_shared-file.txt"; + + StorageFactory.getInstance = () => ({ + getAzureProvider: async (containerName) => ({ + ensureInitialized: async () => {}, + _containerClient: { + getBlockBlobClient: (currentBlobPath) => ({ + url: `https://example.blob.core.windows.net/${containerName}/${currentBlobPath}`, + exists: async () => { + t.is(currentBlobPath, blobPath); + return true; + }, + }), + }, + }), + }); + + const result = await resolveLegacyScopedBlobClient( + containerOwnerId, + blobPath, + ); + + t.truthy(result); + t.is(result.containerName, legacyContainerName); + t.truthy(result.blockBlobClient); + t.true(result.blockBlobClient.url.includes(legacyContainerName)); +}); diff --git a/helper-apps/cortex-file-handler/tests/postOperations.test.js b/helper-apps/cortex-file-handler/tests/postOperations.test.js index ce0024a3..c611ebc5 100644 --- a/helper-apps/cortex-file-handler/tests/postOperations.test.js +++ b/helper-apps/cortex-file-handler/tests/postOperations.test.js @@ -45,12 +45,9 @@ async function uploadFile(filePath, requestId = null, hash = null) { if (hash) form.append("hash", hash); const response = await axios.post(baseUrl, form, { - headers: { - ...form.getHeaders(), - "Content-Type": "multipart/form-data", - }, + headers: form.getHeaders(), validateStatus: (status) => true, - timeout: 30000, + timeout: 120000, maxContentLength: Infinity, maxBodyLength: Infinity, }); diff --git a/helper-apps/cortex-file-handler/tests/renameTargetBlobPath.test.js b/helper-apps/cortex-file-handler/tests/renameTargetBlobPath.test.js new file mode 100644 index 00000000..0f6500b5 --- /dev/null +++ b/helper-apps/cortex-file-handler/tests/renameTargetBlobPath.test.js @@ -0,0 +1,74 @@ +import test from "ava"; + +import { setFileStoreMap, getFileStoreMap, removeFromFileStoreMap } from "../src/redis.js"; +import { StorageService } from "../src/services/storage/StorageService.js"; +import { sanitizeTargetBlobPath } from "../src/utils/targetBlobPathUtils.js"; + +test("sanitizeTargetBlobPath preserves safe folder segments", (t) => { + t.is( + sanitizeTargetBlobPath( + "users/user-context-1/media/Jarvis Article Demo/demo.png", + ), + "users/user-context-1/media/Jarvis Article Demo/demo.png", + ); +}); + +test("sanitizeTargetBlobPath rejects traversal segments", (t) => { + t.is(sanitizeTargetBlobPath("users/user-context-1/../demo.png"), ""); +}); + +test("sanitizeTargetBlobPath sanitizes each segment without dropping folders", (t) => { + t.is( + sanitizeTargetBlobPath("media/folder:name/file:name.png"), + "media/folder_name/file_name.png", + ); +}); + +test("sanitizeTargetBlobPath collapses repeated separators", (t) => { + t.is( + sanitizeTargetBlobPath("//media///folder/file.png//"), + "media/folder/file.png", + ); +}); + +test("hash rename honors targetBlobPath and persists the moved blob path", async (t) => { + const testHash = `test-rename-target-${Date.now()}`; + const targetBlobPath = "users/user-context-1/media/Jarvis Article Demo/new-name.png"; + const realStorageService = new StorageService(); + const uploadedFile = await realStorageService.uploadFile( + Buffer.from("target path rename test"), + "old-name.png", + ); + let renamedUrl = null; + + await setFileStoreMap(testHash, { + url: uploadedFile.url, + filename: "old-name.png", + hash: testHash, + blobPath: uploadedFile.blobName, + blobName: uploadedFile.blobName, + timestamp: new Date().toISOString(), + }); + + try { + const result = await realStorageService.renameFile( + testHash, + "new-name.png", + { log: () => {} }, + null, + { targetBlobPath }, + ); + renamedUrl = result.url; + + t.is(result.blobPath, targetBlobPath); + t.is(result.filename, "new-name.png"); + + const storedInfo = await getFileStoreMap(testHash, true); + t.is(storedInfo.blobPath, targetBlobPath); + t.is(storedInfo.blobName, targetBlobPath); + t.is(storedInfo.filename, "new-name.png"); + } finally { + await removeFromFileStoreMap(testHash); + await realStorageService.deleteFile(renamedUrl || uploadedFile.url); + } +}); diff --git a/helper-apps/cortex-file-handler/tests/setRetention.test.js b/helper-apps/cortex-file-handler/tests/setRetention.test.js index e513f1b5..f915b0ba 100644 --- a/helper-apps/cortex-file-handler/tests/setRetention.test.js +++ b/helper-apps/cortex-file-handler/tests/setRetention.test.js @@ -38,10 +38,10 @@ async function createTestFile(content, extension) { // Helper function to upload file with hash and container async function uploadFile(filePath, hash = null, containerName = null, contextId = null) { const form = new FormData(); - form.append("file", fs.createReadStream(filePath)); if (hash) form.append("hash", hash); if (containerName) form.append("container", containerName); if (contextId) form.append("contextId", contextId); + form.append("file", fs.createReadStream(filePath)); return await axios.post(baseUrl, form, { headers: form.getHeaders(), diff --git a/helper-apps/cortex-file-handler/tests/start.test.js b/helper-apps/cortex-file-handler/tests/start.test.js index b7342a65..4067ae36 100644 --- a/helper-apps/cortex-file-handler/tests/start.test.js +++ b/helper-apps/cortex-file-handler/tests/start.test.js @@ -361,7 +361,7 @@ test.serial("should validate requestId for delete operation", async (t) => { t.is(response.status, 400, "Should return 400 for missing requestId"); t.is( response.data, - "Please pass either a requestId or hash in the query string or request body", + "Please pass either a requestId, hash, or blobPath in the query string or request body", "Should return proper error message", ); }); diff --git a/helper-apps/cortex-file-handler/tests/storage/AzureStorageProvider.test.js b/helper-apps/cortex-file-handler/tests/storage/AzureStorageProvider.test.js index 15372b68..b14e6a65 100644 --- a/helper-apps/cortex-file-handler/tests/storage/AzureStorageProvider.test.js +++ b/helper-apps/cortex-file-handler/tests/storage/AzureStorageProvider.test.js @@ -2,31 +2,36 @@ import test from "ava"; import fs from "fs"; import path from "path"; import { fileURLToPath } from "url"; +import { + StorageSharedKeyCredential, +} from "@azure/storage-blob"; import { AzureStorageProvider } from "../../src/services/storage/AzureStorageProvider.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); +// Azurite well-known connection string — works without a running emulator +// for tests that only exercise constructor / SAS generation (no network calls). +const AZURITE_CONN_STRING = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;"; + test.before(() => { // Ensure we have the required environment variables if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { console.warn( - "Skipping Azure tests - AZURE_STORAGE_CONNECTION_STRING not set", + "Skipping Azure integration tests - AZURE_STORAGE_CONNECTION_STRING not set", ); } }); -test("should create provider with valid credentials", (t) => { - if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { - t.pass("Skipping test - Azure not configured"); - return; - } +// ── Constructor tests ────────────────────────────────────────────── +test("should create provider with valid credentials", (t) => { const provider = new AzureStorageProvider( - process.env.AZURE_STORAGE_CONNECTION_STRING, + AZURITE_CONN_STRING, "test-container", ); t.truthy(provider); + t.is(provider.containerName, "test-container"); }); test("should throw error with missing credentials", (t) => { @@ -38,6 +43,465 @@ test("should throw error with missing credentials", (t) => { ); }); +test("constructor should initialize cached fields to null", (t) => { + const provider = new AzureStorageProvider( + AZURITE_CONN_STRING, + "test-container", + ); + t.is(provider._blobServiceClient, null); + t.is(provider._containerClient, null); + t.is(provider._sharedKeyCredential, null); + t.is(provider._initPromise, null); +}); + +// ── ensureInitialized / _doInitialize tests ──────────────────────── + +test("ensureInitialized should populate cached fields", async (t) => { + const provider = new AzureStorageProvider( + AZURITE_CONN_STRING, + "test-container", + ); + + // Stub _doInitialize to avoid real network calls + provider._doInitialize = async () => { + provider._blobServiceClient = { fake: true }; + provider._containerClient = { fake: true, containerName: "test-container" }; + provider._sharedKeyCredential = new StorageSharedKeyCredential( + "devstoreaccount1", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + ); + }; + + await provider.ensureInitialized(); + + t.truthy(provider._blobServiceClient); + t.truthy(provider._containerClient); + t.truthy(provider._sharedKeyCredential); +}); + +test("ensureInitialized should only run _doInitialize once", async (t) => { + const provider = new AzureStorageProvider( + AZURITE_CONN_STRING, + "test-container", + ); + + let callCount = 0; + provider._doInitialize = async () => { + callCount++; + provider._blobServiceClient = { fake: true }; + provider._containerClient = { fake: true, containerName: "test-container" }; + provider._sharedKeyCredential = new StorageSharedKeyCredential( + "devstoreaccount1", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + ); + }; + + // Call multiple times — should only init once + await Promise.all([ + provider.ensureInitialized(), + provider.ensureInitialized(), + provider.ensureInitialized(), + ]); + + t.is(callCount, 1, "_doInitialize should be called exactly once"); +}); + +test("ensureInitialized is a no-op after successful init", async (t) => { + const provider = new AzureStorageProvider( + AZURITE_CONN_STRING, + "test-container", + ); + + let callCount = 0; + provider._doInitialize = async () => { + callCount++; + provider._blobServiceClient = { fake: true }; + provider._containerClient = { fake: true, containerName: "test-container" }; + provider._sharedKeyCredential = new StorageSharedKeyCredential( + "devstoreaccount1", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + ); + }; + + await provider.ensureInitialized(); + t.is(callCount, 1); + + // Second call should short-circuit via the _sharedKeyCredential check + await provider.ensureInitialized(); + t.is(callCount, 1, "should not call _doInitialize again"); +}); + +// ── getBlobClient caching tests ──────────────────────────────────── + +test("getBlobClient should return cached clients", async (t) => { + const provider = new AzureStorageProvider( + AZURITE_CONN_STRING, + "test-container", + ); + + const fakeServiceClient = { fake: "service" }; + const fakeContainerClient = { + fake: "container", + containerName: "test-container", + createIfNotExists: async () => {}, + }; + + provider._doInitialize = async () => { + provider._blobServiceClient = fakeServiceClient; + provider._containerClient = fakeContainerClient; + provider._sharedKeyCredential = new StorageSharedKeyCredential( + "devstoreaccount1", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + ); + }; + + const result1 = await provider.getBlobClient(); + const result2 = await provider.getBlobClient(); + + t.is(result1.blobServiceClient, fakeServiceClient); + t.is(result1.containerClient, fakeContainerClient); + // Same references on second call + t.is(result1.blobServiceClient, result2.blobServiceClient); + t.is(result1.containerClient, result2.containerClient); +}); + +test("getBlobClient retries createIfNotExists after a transient failure", async (t) => { + const provider = new AzureStorageProvider( + AZURITE_CONN_STRING, + "test-container", + ); + + let createCalls = 0; + const fakeContainerClient = { + containerName: "test-container", + createIfNotExists: async () => { + createCalls++; + if (createCalls === 1) { + const err = new Error("transient"); + err.statusCode = 500; + throw err; + } + // succeed on retry + }, + }; + + provider._doInitialize = async () => { + provider._blobServiceClient = { fake: "service" }; + provider._containerClient = fakeContainerClient; + provider._sharedKeyCredential = new StorageSharedKeyCredential( + "devstoreaccount1", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + ); + }; + + await t.throwsAsync( + () => provider.getBlobClient(), + { message: "transient" }, + ); + t.false(provider._containerEnsured, "should not cache failed create"); + t.is(createCalls, 1); + + await provider.getBlobClient(); + t.true(provider._containerEnsured, "successful retry should set the cache"); + t.is(createCalls, 2, "second call retries createIfNotExists"); + + await provider.getBlobClient(); + t.is(createCalls, 2, "subsequent calls skip createIfNotExists"); +}); + +test("getBlobClient caches success when create returns 409 (already exists)", async (t) => { + const provider = new AzureStorageProvider( + AZURITE_CONN_STRING, + "test-container", + ); + + let createCalls = 0; + const fakeContainerClient = { + containerName: "test-container", + createIfNotExists: async () => { + createCalls++; + const err = new Error("already exists"); + err.statusCode = 409; + throw err; + }, + }; + + provider._doInitialize = async () => { + provider._blobServiceClient = { fake: "service" }; + provider._containerClient = fakeContainerClient; + provider._sharedKeyCredential = new StorageSharedKeyCredential( + "devstoreaccount1", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + ); + }; + + await provider.getBlobClient(); + await provider.getBlobClient(); + t.true(provider._containerEnsured); + t.is(createCalls, 1, "409 is treated as success"); +}); + +// ── listFolder defensive tests ───────────────────────────────────── + +test("listFolder returns empty array when container does not exist", async (t) => { + const provider = new AzureStorageProvider( + AZURITE_CONN_STRING, + "test-container", + ); + + const fakeContainerClient = { + containerName: "test-container", + createIfNotExists: async () => {}, + listBlobsFlat: () => ({ + [Symbol.asyncIterator]: () => ({ + next: async () => { + const err = new Error("The specified container does not exist."); + err.statusCode = 404; + err.code = "ContainerNotFound"; + throw err; + }, + }), + }), + }; + + provider._doInitialize = async () => { + provider._blobServiceClient = { fake: "service" }; + provider._containerClient = fakeContainerClient; + provider._sharedKeyCredential = new StorageSharedKeyCredential( + "devstoreaccount1", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + ); + }; + + const result = await provider.listFolder("global"); + t.deepEqual(result, []); +}); + +test("listFolder propagates non-404 errors", async (t) => { + const provider = new AzureStorageProvider( + AZURITE_CONN_STRING, + "test-container", + ); + + const fakeContainerClient = { + containerName: "test-container", + createIfNotExists: async () => {}, + listBlobsFlat: () => ({ + [Symbol.asyncIterator]: () => ({ + next: async () => { + const err = new Error("boom"); + err.statusCode = 500; + throw err; + }, + }), + }), + }; + + provider._doInitialize = async () => { + provider._blobServiceClient = { fake: "service" }; + provider._containerClient = fakeContainerClient; + provider._sharedKeyCredential = new StorageSharedKeyCredential( + "devstoreaccount1", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + ); + }; + + await t.throwsAsync(() => provider.listFolder("global"), { message: "boom" }); +}); + +test("uploadStream recreates missing container and retries once", async (t) => { + const provider = new AzureStorageProvider( + AZURITE_CONN_STRING, + "test-container", + ); + + const tempFile = path.join(__dirname, `retry-${Date.now()}.txt`); + fs.writeFileSync(tempFile, "retry me"); + t.teardown(() => { + if (fs.existsSync(tempFile)) { + fs.unlinkSync(tempFile); + } + }); + + let createCalls = 0; + let uploadCalls = 0; + const fakeContainerClient = { + containerName: "test-container", + createIfNotExists: async () => { + createCalls++; + }, + getBlockBlobClient: (blobName) => ({ + url: `http://127.0.0.1:10000/devstoreaccount1/test-container/${blobName}`, + uploadStream: async (body) => { + await new Promise((resolve, reject) => { + body.on("error", reject); + body.on("end", resolve); + body.resume(); + }); + uploadCalls++; + if (uploadCalls === 1) { + const err = new Error("The specified container does not exist."); + err.statusCode = 404; + err.code = "ContainerNotFound"; + throw err; + } + }, + }), + }; + + provider._doInitialize = async () => { + provider._blobServiceClient = { fake: "service" }; + provider._containerClient = fakeContainerClient; + provider._sharedKeyCredential = new StorageSharedKeyCredential( + "devstoreaccount1", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + ); + }; + + const result = await provider.uploadStream( + {}, + "retry.txt", + fs.createReadStream(tempFile), + "text/plain", + ); + + t.true(result.url.includes("/test-container/retry.txt?")); + t.is(createCalls, 2, "container should be re-ensured before retry"); + t.is(uploadCalls, 2, "upload should be retried once"); +}); + +// ── generateSASToken dual-signature tests ────────────────────────── + +// Helper: create an initialized provider with the Azurite credential cached +function createInitializedProvider() { + const provider = new AzureStorageProvider( + AZURITE_CONN_STRING, + "test-container", + ); + provider._sharedKeyCredential = new StorageSharedKeyCredential( + "devstoreaccount1", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + ); + return provider; +} + +test("generateSASToken new signature: (blobName)", (t) => { + const provider = createInitializedProvider(); + const token = provider.generateSASToken("folder/file.txt"); + t.truthy(token, "should return a SAS token string"); + t.true(token.includes("sig="), "token should contain a signature"); + t.true(token.includes("se="), "token should contain an expiry"); +}); + +test("generateSASToken new signature: (blobName, options)", (t) => { + const provider = createInitializedProvider(); + const token = provider.generateSASToken("file.txt", { minutes: 10 }); + t.truthy(token); + t.true(token.includes("sig=")); +}); + +test("generateSASToken old signature: (containerClient, blobName)", (t) => { + const provider = createInitializedProvider(); + const fakeContainerClient = { containerName: "test-container" }; + const token = provider.generateSASToken(fakeContainerClient, "file.txt"); + t.truthy(token); + t.true(token.includes("sig=")); +}); + +test("generateSASToken old signature: (containerClient, blobName, options)", (t) => { + const provider = createInitializedProvider(); + const fakeContainerClient = { containerName: "test-container" }; + const token = provider.generateSASToken(fakeContainerClient, "file.txt", { minutes: 10 }); + t.truthy(token); + t.true(token.includes("sig=")); +}); + +test("generateSASToken new and old signatures produce equivalent tokens", (t) => { + const provider = createInitializedProvider(); + const fakeContainerClient = { containerName: "test-container" }; + + // Both should produce tokens for the same blob with the same options + // We can't compare exact strings (timestamp varies) but can verify both are valid + const newToken = provider.generateSASToken("file.txt", { hours: 1 }); + const oldToken = provider.generateSASToken(fakeContainerClient, "file.txt", { hours: 1 }); + + t.truthy(newToken); + t.truthy(oldToken); + // Both should have the same permissions and container + t.true(newToken.includes("sp=r"), "new signature should default to read permission"); + t.true(oldToken.includes("sp=r"), "old signature should default to read permission"); +}); + +test("generateSASToken throws if not initialized", (t) => { + const provider = new AzureStorageProvider( + AZURITE_CONN_STRING, + "test-container", + ); + // _sharedKeyCredential is null — should throw + t.throws( + () => provider.generateSASToken("file.txt"), + { message: /not initialized/ }, + ); +}); + +test("generateSASToken uses this.containerName (not containerClient.containerName)", (t) => { + const provider = createInitializedProvider(); + // Pass a containerClient with a DIFFERENT containerName — should be ignored + const wrongContainerClient = { containerName: "wrong-container" }; + const token = provider.generateSASToken(wrongContainerClient, "file.txt"); + // The token is generated for this.containerName ("test-container"), not "wrong-container" + t.truthy(token); + t.true(token.includes("sig=")); +}); + +test("generateSASToken respects custom permissions", (t) => { + const provider = createInitializedProvider(); + const token = provider.generateSASToken("file.txt", { permissions: "rw" }); + t.true(token.includes("sp=rw")); +}); + +test("generateSASToken respects days option", (t) => { + const provider = createInitializedProvider(); + const token = provider.generateSASToken("file.txt", { days: 7 }); + t.truthy(token); + t.true(token.includes("se="), "should have an expiry"); +}); + +// ── generateShortLivedSASToken dual-signature tests ──────────────── + +test("generateShortLivedSASToken new signature: (blobName)", (t) => { + const provider = createInitializedProvider(); + const token = provider.generateShortLivedSASToken("file.txt"); + t.truthy(token); + t.true(token.includes("sig=")); +}); + +test("generateShortLivedSASToken new signature: (blobName, minutes)", (t) => { + const provider = createInitializedProvider(); + const token = provider.generateShortLivedSASToken("file.txt", 15); + t.truthy(token); + t.true(token.includes("sig=")); +}); + +test("generateShortLivedSASToken old signature: (containerClient, blobName, minutes)", (t) => { + const provider = createInitializedProvider(); + const fakeContainerClient = { containerName: "test-container" }; + const token = provider.generateShortLivedSASToken(fakeContainerClient, "file.txt", 10); + t.truthy(token); + t.true(token.includes("sig=")); +}); + +test("generateShortLivedSASToken defaults to 5 minutes", (t) => { + const provider = createInitializedProvider(); + // Both signatures should default to 5 min + const newToken = provider.generateShortLivedSASToken("file.txt"); + const oldToken = provider.generateShortLivedSASToken({}, "file.txt"); + t.truthy(newToken); + t.truthy(oldToken); +}); + +// ── Integration tests (require live Azure/Azurite) ───────────────── + test("should upload and delete file", async (t) => { if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { t.pass("Skipping test - Azure not configured"); diff --git a/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js b/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js index 8228d5e6..88af2bab 100644 --- a/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js +++ b/helper-apps/cortex-file-handler/tests/storage/StorageFactory.test.js @@ -117,52 +117,54 @@ test("should get azure provider with default container when no container specifi t.truthy(provider.containerName); }); -test("should get azure provider (container parameter ignored)", async (t) => { +test("should get azure provider with custom container name", async (t) => { if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { t.pass("Skipping test - Azure not configured"); return; } const factory = new StorageFactory(); - - // Container parameter is ignored - always uses default container from env - const provider = await factory.getAzureProvider("any-container-name"); + + // Custom container names are used for per-user containers + const provider = await factory.getAzureProvider("custom-container"); t.truthy(provider); - // Should use the default container from env, not the parameter - const { getContainerName } = await import("../../src/constants.js"); - t.is(provider.containerName, getContainerName()); + t.is(provider.containerName, "custom-container"); }); -test("should ignore container parameter and use default container", async (t) => { +test("should return different providers for different container names", async (t) => { if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { t.pass("Skipping test - Azure not configured"); return; } const factory = new StorageFactory(); - - // Container parameter is ignored - always uses default container - const provider1 = await factory.getAzureProvider("invalid-container"); + + const provider1 = await factory.getAzureProvider("container-a"); const provider2 = await factory.getAzureProvider(); - - // Both should return the same provider instance (same default container) - t.is(provider1, provider2); + + // Different containers should yield different provider instances + t.not(provider1, provider2); + t.is(provider1.containerName, "container-a"); }); -test("should cache provider instance (single container)", async (t) => { +test("should cache provider instance per container name", async (t) => { if (!process.env.AZURE_STORAGE_CONNECTION_STRING) { t.pass("Skipping test - Azure not configured"); return; } const factory = new StorageFactory(); - - // All calls should return the same provider instance (single container) + + // Same container name should return the same cached instance const provider1 = await factory.getAzureProvider(); const provider2 = await factory.getAzureProvider(); - const provider3 = await factory.getAzureProvider("ignored-container"); - - // All should return the same instance t.is(provider1, provider2); - t.is(provider1, provider3); + + // Different container name returns a different instance + const provider3 = await factory.getAzureProvider("other-container"); + t.not(provider1, provider3); + + // But asking for that same name again returns the cached one + const provider4 = await factory.getAzureProvider("other-container"); + t.is(provider3, provider4); }); diff --git a/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js b/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js index 6f1e9e4a..c2d455a5 100644 --- a/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js +++ b/helper-apps/cortex-file-handler/tests/storage/StorageService.test.js @@ -187,12 +187,9 @@ test("should handle delete file by hash when file not found", async (t) => { const service = new StorageService(factory); const nonExistentHash = "non-existent-hash-456"; - try { - await service.deleteFileByHash(nonExistentHash); - t.fail("Should have thrown an error for non-existent hash"); - } catch (error) { - t.true(error.message.includes("not found")); - } + const result = await service.deleteFileByHash(nonExistentHash); + t.true(result.alreadyDeleted, "Should indicate file was already deleted"); + t.is(result.hash, nonExistentHash); }); test("should handle delete file by hash with missing hash parameter", async (t) => { diff --git a/helper-apps/cortex-file-handler/tests/testUtils.helper.js b/helper-apps/cortex-file-handler/tests/testUtils.helper.js index 5277e75f..2d29146f 100644 --- a/helper-apps/cortex-file-handler/tests/testUtils.helper.js +++ b/helper-apps/cortex-file-handler/tests/testUtils.helper.js @@ -4,6 +4,7 @@ import fs from "fs/promises"; import path from "path"; import { app, port } from "../src/start.js"; +import { client as redisClient } from "../src/redis.js"; export async function cleanupHashAndFile(hash, uploadedUrl, baseUrl) { // Only perform hash operations if hash is provided @@ -105,11 +106,16 @@ export async function startTestServer(options = {}) { throw new Error("Test server is already running"); } + // Clear mock Redis state from previous test files to prevent leakage + if (typeof redisClient._reset === 'function') { + redisClient._reset(); + } + // Start the server for tests server = app.listen(port, () => { console.log(`Test server started on port ${port}`); }); - + // Wait for server to be ready await new Promise((resolve) => setTimeout(resolve, 1000)); @@ -140,9 +146,15 @@ export async function stopTestServer(beforeClose) { if (beforeClose) { await beforeClose(); } - + if (server) { - server.close(); + // Force-close keep-alive connections so the port is released immediately. + // Without this, server.close() waits for idle keep-alive connections to + // drain, causing EADDRINUSE when the next test file starts. + if (typeof server.closeAllConnections === 'function') { + server.closeAllConnections(); + } + await new Promise((resolve) => server.close(resolve)); server = null; } } diff --git a/helper-apps/cortex-workspace/.npmrc b/helper-apps/cortex-workspace/.npmrc new file mode 100644 index 00000000..7253a5ce --- /dev/null +++ b/helper-apps/cortex-workspace/.npmrc @@ -0,0 +1 @@ +min-release-age=7 diff --git a/helper-apps/cortex-workspace/Dockerfile b/helper-apps/cortex-workspace/Dockerfile new file mode 100644 index 00000000..b9f7f3d4 --- /dev/null +++ b/helper-apps/cortex-workspace/Dockerfile @@ -0,0 +1,41 @@ +FROM node:22-slim + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git curl wget jq vim nano htop less tree \ + unzip zip zstd pigz \ + build-essential python3 python-is-python3 python3-pip python3-venv \ + python3-numpy python3-pandas python3-pil python3-matplotlib \ + python3-scipy python3-sklearn python3-openpyxl \ + python3-requests python3-bs4 python3-lxml \ + procps net-tools fuse3 gnupg \ + && rm -rf /var/lib/apt/lists/* + +RUN curl -fsSL https://registry.npmjs.org/npm/-/npm-11.14.1.tgz -o /tmp/npm-11.14.1.tgz \ + && mkdir -p /opt/npm-11.14.1 \ + && tar -xzf /tmp/npm-11.14.1.tgz -C /opt/npm-11.14.1 --strip-components=1 \ + && node /opt/npm-11.14.1/bin/npm-cli.js --version \ + && printf '%s\n' '#!/bin/sh' 'exec node /opt/npm-11.14.1/bin/npm-cli.js "$@"' > /usr/local/bin/npm \ + && printf '%s\n' '#!/bin/sh' 'exec node /opt/npm-11.14.1/bin/npx-cli.js "$@"' > /usr/local/bin/npx \ + && chmod +x /usr/local/bin/npm /usr/local/bin/npx \ + && rm /tmp/npm-11.14.1.tgz +# Install blobfuse2 for Azure Blob Storage FUSE mount +RUN curl -fsSL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor -o /usr/share/keyrings/microsoft.gpg \ + && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft.gpg] https://packages.microsoft.com/repos/microsoft-debian-bookworm-prod bookworm main" \ + > /etc/apt/sources.list.d/microsoft.list \ + && apt-get update && apt-get install -y --no-install-recommends blobfuse2 \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app +COPY .npmrc package*.json ./ +RUN npm ci --omit=dev +COPY . . + +RUN mkdir -p /workspace /persist /blob-files /tmp/blobfuse2 + +EXPOSE 3100 + +HEALTHCHECK --interval=30s --timeout=5s --retries=3 \ + CMD curl -sf http://localhost:3100/health || exit 1 + +ENTRYPOINT ["/app/entrypoint.sh"] +CMD ["node", "server.js"] diff --git a/helper-apps/cortex-workspace/entrypoint.sh b/helper-apps/cortex-workspace/entrypoint.sh new file mode 100755 index 00000000..4e28dc89 --- /dev/null +++ b/helper-apps/cortex-workspace/entrypoint.sh @@ -0,0 +1,136 @@ +#!/bin/bash +set -e + +# Mount Azure Blob Storage at /blob-files if credentials are provided, then +# expose it at /workspace/files for compatibility. +# The blob container name (e.g. cortexfiles-local, cortexfiles-local-abc123) +# comes from the AZURE_BLOB_CONTAINER env var, set per environment. +# +# Auth modes (checked in order): +# 1. SAS token — AZURE_BLOB_SAS_TOKEN (preferred, container-scoped) +# 2. Account key — AZURE_STORAGE_ACCOUNT_KEY (legacy fallback) + +WORKSPACE_DIR="${WORKSPACE_DIR:-/workspace}" +PERSIST_DIR="${WORKSPACE_PERSIST_DIR:-/persist}" +BLOB_FILES_DIR="${WORKSPACE_BLOB_FILES_DIR:-/blob-files}" +CHECKPOINT_PATH="${WORKSPACE_CHECKPOINT_PATH:-$PERSIST_DIR/workspace.tar.gz}" + +mkdir -p "$WORKSPACE_DIR" "$PERSIST_DIR" "$BLOB_FILES_DIR" + +expose_blob_files() { + local target="$WORKSPACE_DIR/files" + + umount "$target" >/dev/null 2>&1 || true + umount -l "$target" >/dev/null 2>&1 || true + if ! rm -rf "$target" >/dev/null 2>&1; then + echo "WARNING: ${target} is busy; leaving existing blob-files exposure in place" + return 0 + fi + mkdir -p "$target" + + if mount --bind "$BLOB_FILES_DIR" "$target" >/dev/null 2>&1; then + echo "Exposed ${BLOB_FILES_DIR} at ${target} with bind mount" + else + if rmdir "$target" >/dev/null 2>&1 || rm -rf "$target" >/dev/null 2>&1; then + ln -s "$BLOB_FILES_DIR" "$target" + echo "WARNING: bind mount failed; ${target} is a symlink to ${BLOB_FILES_DIR}" + else + echo "WARNING: bind mount failed and ${target} is busy; leaving existing exposure in place" + fi + fi +} + +if [ -f "$CHECKPOINT_PATH" ]; then + echo "Restoring workspace checkpoint from ${CHECKPOINT_PATH}" + tar xzf "$CHECKPOINT_PATH" -C "$WORKSPACE_DIR" +elif find "$PERSIST_DIR" -mindepth 1 \ + ! -name 'workspace.tar.gz' \ + ! -name 'workspace.tar.gz.tmp' \ + ! -name 'workspace.tar.gz.*.tmp' \ + ! -name 'workspace.prev.tar.gz' \ + -print -quit | grep -q .; then + echo "Importing legacy workspace files from ${PERSIST_DIR}" + tar \ + --exclude='./workspace.tar.gz' \ + --exclude='./workspace.tar.gz.tmp' \ + --exclude='./workspace.tar.gz.*.tmp' \ + --exclude='./workspace.prev.tar.gz' \ + -cf - -C "$PERSIST_DIR" . | tar xf - -C "$WORKSPACE_DIR" +fi + +if [ -n "$AZURE_STORAGE_ACCOUNT_NAME" ] && [ -n "$AZURE_BLOB_CONTAINER" ]; then + # Determine auth mode + AUTH_MODE="" + if [ -n "$AZURE_BLOB_SAS_TOKEN" ]; then + AUTH_MODE="sas" + elif [ -n "$AZURE_STORAGE_ACCOUNT_KEY" ]; then + AUTH_MODE="key" + fi + + if [ -n "$AUTH_MODE" ]; then + MOUNT_DIR="$BLOB_FILES_DIR" + CACHE_DIR="/tmp/blobfuse2" + + mkdir -p "$MOUNT_DIR" "$CACHE_DIR" + + echo "Mounting blob storage ($AUTH_MODE): ${AZURE_BLOB_CONTAINER} -> ${MOUNT_DIR}" + + # Generate blobfuse2 config — auth section varies by mode + cat > /tmp/blobfuse2-config.yaml <> /tmp/blobfuse2-config.yaml <> /tmp/blobfuse2-config.yaml <&1 || echo "WARNING: blobfuse2 mount failed (container may lack SYS_ADMIN capability)" + else + echo "No blob storage credentials — ${BLOB_FILES_DIR}/ is a regular directory" + fi +else + echo "No blob storage credentials — ${BLOB_FILES_DIR}/ is a regular directory" +fi + +expose_blob_files + +# Run the CMD (default: node server.js) +exec "$@" diff --git a/helper-apps/cortex-workspace/lib/auth.js b/helper-apps/cortex-workspace/lib/auth.js new file mode 100644 index 00000000..e0ce955c --- /dev/null +++ b/helper-apps/cortex-workspace/lib/auth.js @@ -0,0 +1,42 @@ +import crypto from 'node:crypto'; + +let _secret = process.env.WORKSPACE_SECRET; + +/** + * Get the current workspace secret. + */ +export function getSecret() { + return _secret; +} + +/** + * Replace the workspace secret at runtime. + * Used by /reconfigure to rotate the secret after a warm-pool container is claimed. + */ +export function setSecret(s) { + _secret = s; +} + +/** + * Express middleware for shared secret authentication. + * Validates x-workspace-secret header using timing-safe comparison. + */ +export function requireAuth(req, res, next) { + if (!_secret) { + return res.status(500).json({ error: 'Server misconfigured: no secret set' }); + } + + const provided = req.headers['x-workspace-secret']; + if (!provided || typeof provided !== 'string') { + return res.status(401).json({ error: 'Missing x-workspace-secret header' }); + } + + const expected = Buffer.from(_secret, 'utf8'); + const actual = Buffer.from(provided, 'utf8'); + + if (expected.length !== actual.length || !crypto.timingSafeEqual(expected, actual)) { + return res.status(401).json({ error: 'Invalid secret' }); + } + + next(); +} diff --git a/helper-apps/cortex-workspace/lib/files.js b/helper-apps/cortex-workspace/lib/files.js new file mode 100644 index 00000000..14812bfa --- /dev/null +++ b/helper-apps/cortex-workspace/lib/files.js @@ -0,0 +1,201 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; + +const MAX_READ_BYTES = 100 * 1024; // 100KB +const MAX_LINES = 1000; + +/** + * Read file contents with optional line range. + */ +export async function readFile(filePath, options = {}) { + const { startLine, endLine, encoding } = options; + + try { + const stat = await fs.stat(filePath); + if (!stat.isFile()) { + return { error: `Not a file: ${filePath}` }; + } + + if (encoding === 'base64') { + const buf = await fs.readFile(filePath); + return { + content: buf.toString('base64'), + totalBytes: stat.size, + encoding: 'base64', + truncated: false, + }; + } + + const raw = await fs.readFile(filePath, 'utf8'); + const lines = raw.split(/\r?\n/); + const totalLines = lines.length; + const totalBytes = stat.size; + + let selected = lines; + let actualStart = 1; + let actualEnd = totalLines; + let truncated = false; + + if (startLine !== undefined || endLine !== undefined) { + const start = Math.max(1, startLine || 1) - 1; // to 0-indexed + const end = endLine !== undefined ? Math.min(totalLines, endLine) : Math.min(totalLines, start + MAX_LINES); + selected = lines.slice(start, end); + actualStart = start + 1; + actualEnd = Math.min(end, totalLines); + truncated = actualEnd < totalLines || actualStart > 1; + } else if (lines.length > MAX_LINES) { + selected = lines.slice(0, MAX_LINES); + actualEnd = MAX_LINES; + truncated = true; + } + + let content = selected.join('\n'); + if (content.length > MAX_READ_BYTES) { + content = content.slice(0, MAX_READ_BYTES); + truncated = true; + } + + return { + content, + totalLines, + totalBytes, + startLine: actualStart, + endLine: actualEnd, + returnedLines: selected.length, + truncated, + }; + } catch (e) { + if (e.code === 'ENOENT') return { error: `File not found: ${filePath}` }; + if (e.code === 'EACCES') return { error: `Permission denied: ${filePath}` }; + return { error: e.message }; + } +} + +/** + * Write content to a file. Creates parent directories by default. + */ +export async function writeFile(filePath, content, options = {}) { + const { encoding, createDirs = true } = options; + + try { + if (createDirs) { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + } + + let buf; + if (encoding === 'base64') { + buf = Buffer.from(content, 'base64'); + } else { + buf = Buffer.from(content, 'utf8'); + } + + await fs.writeFile(filePath, buf); + + return { + path: filePath, + bytesWritten: buf.length, + }; + } catch (e) { + if (e.code === 'EACCES') return { error: `Permission denied: ${filePath}` }; + return { error: e.message }; + } +} + +/** + * Search-and-replace in a file (exact string match). + */ +export async function editFile(filePath, oldString, newString, options = {}) { + const { replaceAll = false } = options; + + try { + const content = await fs.readFile(filePath, 'utf8'); + + if (!content.includes(oldString)) { + return { error: `String not found in ${filePath}` }; + } + + let updated; + let replacements; + if (replaceAll) { + replacements = content.split(oldString).length - 1; + updated = content.split(oldString).join(newString); + } else { + replacements = 1; + updated = content.replace(oldString, newString); + } + + await fs.writeFile(filePath, updated, 'utf8'); + + return { + path: filePath, + replacements, + }; + } catch (e) { + if (e.code === 'ENOENT') return { error: `File not found: ${filePath}` }; + if (e.code === 'EACCES') return { error: `Permission denied: ${filePath}` }; + return { error: e.message }; + } +} + +/** + * Browse directory contents. + */ +export async function browseDir(dirPath, options = {}) { + const { recursive = false, maxDepth = 3 } = options; + + async function listEntries(dir, depth) { + const entries = []; + try { + const items = await fs.readdir(dir, { withFileTypes: true }); + for (const item of items) { + const fullPath = path.join(dir, item.name); + const entry = { name: item.name }; + + if (item.isDirectory()) { + entry.type = 'directory'; + try { + const stat = await fs.stat(fullPath); + entry.modified = stat.mtime.toISOString(); + } catch { /* ignore stat errors */ } + if (recursive && depth < maxDepth) { + entry.children = await listEntries(fullPath, depth + 1); + } + } else if (item.isFile()) { + entry.type = 'file'; + try { + const stat = await fs.stat(fullPath); + entry.size = stat.size; + entry.modified = stat.mtime.toISOString(); + } catch { /* ignore stat errors */ } + } else if (item.isSymbolicLink()) { + entry.type = 'symlink'; + } else { + entry.type = 'other'; + } + + entries.push(entry); + } + } catch (e) { + if (e.code === 'EACCES') return [{ error: `Permission denied: ${dir}` }]; + throw e; + } + return entries; + } + + try { + const stat = await fs.stat(dirPath); + if (!stat.isDirectory()) { + return { error: `Not a directory: ${dirPath}` }; + } + + const entries = await listEntries(dirPath, 0); + return { + path: dirPath, + entries, + }; + } catch (e) { + if (e.code === 'ENOENT') return { error: `Directory not found: ${dirPath}` }; + if (e.code === 'EACCES') return { error: `Permission denied: ${dirPath}` }; + return { error: e.message }; + } +} diff --git a/helper-apps/cortex-workspace/lib/shell.js b/helper-apps/cortex-workspace/lib/shell.js new file mode 100644 index 00000000..d6f732f8 --- /dev/null +++ b/helper-apps/cortex-workspace/lib/shell.js @@ -0,0 +1,214 @@ +import { spawn } from 'node:child_process'; +import crypto from 'node:crypto'; + +const MAX_OUTPUT = 100 * 1024; // 100KB per stream +const MAX_BACKGROUND = 20; +const RESULT_TTL_MS = 24 * 60 * 60 * 1000; // 24 hours +const DEFAULT_TIMEOUT_MS = 110_000; +const MAX_TIMEOUT_MS = 600_000; +const DEFAULT_CWD = '/workspace'; + +// Background process store: processId -> { proc, stdout, stderr, exitCode, startedAt, completedAt } +const backgroundProcesses = new Map(); + +// Periodic cleanup of expired results +const cleanupTimer = setInterval(() => { + const now = Date.now(); + for (const [id, entry] of backgroundProcesses) { + if (entry.completedAt && (now - entry.completedAt) > RESULT_TTL_MS) { + backgroundProcesses.delete(id); + } + } +}, 60_000); +cleanupTimer.unref(); + +function truncate(str, max) { + if (str.length <= max) return { text: str, truncated: false }; + return { text: str.slice(0, max), truncated: true }; +} + +function normalizeTimeout(timeout) { + return Math.min(timeout || DEFAULT_TIMEOUT_MS, MAX_TIMEOUT_MS); +} + +function execCommand(command, options = {}) { + const cwd = options.cwd || DEFAULT_CWD; + const timeoutMs = normalizeTimeout(options.timeout); + + return new Promise((resolve) => { + const startTime = Date.now(); + let stdout = ''; + let stderr = ''; + let killed = false; + + // Source /workspace/.env (injected by /reconfigure) so env vars + // are available in all shell commands, not just interactive shells. + const wrappedCommand = '[ -f /workspace/.env ] && . /workspace/.env; ' + command; + const proc = spawn('/bin/bash', ['-c', wrappedCommand], { + cwd, + env: { ...process.env, HOME: '/root' }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + const timer = setTimeout(() => { + killed = true; + proc.kill('SIGKILL'); + }, timeoutMs); + + proc.stdout.on('data', (chunk) => { + if (stdout.length < MAX_OUTPUT) { + stdout += chunk.toString(); + } + }); + + proc.stderr.on('data', (chunk) => { + if (stderr.length < MAX_OUTPUT) { + stderr += chunk.toString(); + } + }); + + proc.on('close', (code) => { + clearTimeout(timer); + const durationMs = Date.now() - startTime; + const out = truncate(stdout, MAX_OUTPUT); + const err = truncate(stderr, MAX_OUTPUT); + + resolve({ + success: code === 0 && !killed, + stdout: out.text, + stderr: err.text, + exitCode: killed ? -1 : (code ?? -1), + durationMs, + killed, + truncated: out.truncated || err.truncated, + }); + }); + + proc.on('error', (e) => { + clearTimeout(timer); + resolve({ + success: false, + stdout: '', + stderr: e.message, + exitCode: -1, + durationMs: Date.now() - startTime, + killed: false, + truncated: false, + }); + }); + }); +} + +/** + * Execute a shell command synchronously (blocks until done or timeout). + */ +export function execSync(command, options = {}) { + return execCommand(command, options); +} + +/** + * Start a background process. Returns processId immediately. + */ +export function execBackground(command, options = {}) { + if (backgroundProcesses.size >= MAX_BACKGROUND) { + // Count running only + let running = 0; + for (const entry of backgroundProcesses.values()) { + if (!entry.completedAt) running++; + } + if (running >= MAX_BACKGROUND) { + return { error: `Maximum concurrent background processes (${MAX_BACKGROUND}) reached` }; + } + } + + const processId = crypto.randomBytes(8).toString('hex'); + const cwd = options.cwd || DEFAULT_CWD; + + const entry = { + stdout: '', + stderr: '', + exitCode: null, + startedAt: Date.now(), + completedAt: null, + command: command.slice(0, 200), // store truncated command for display + }; + + const proc = spawn('/bin/bash', ['-c', command], { + cwd, + env: { ...process.env, HOME: '/root' }, + stdio: ['ignore', 'pipe', 'pipe'], + }); + + proc.stdout.on('data', (chunk) => { + if (entry.stdout.length < MAX_OUTPUT) { + entry.stdout += chunk.toString(); + } + }); + + proc.stderr.on('data', (chunk) => { + if (entry.stderr.length < MAX_OUTPUT) { + entry.stderr += chunk.toString(); + } + }); + + proc.on('close', (code) => { + if (entry.exitCode === null) { + entry.exitCode = code ?? -1; + } + entry.completedAt = Date.now(); + }); + + proc.on('error', (e) => { + entry.stderr += e.message; + entry.exitCode = -1; + entry.completedAt = Date.now(); + }); + + entry.proc = proc; + backgroundProcesses.set(processId, entry); + + return { processId }; +} + +/** + * Get the result/status of a background process. + */ +export function getResult(processId) { + const entry = backgroundProcesses.get(processId); + if (!entry) { + return { error: `Process ${processId} not found` }; + } + + const out = truncate(entry.stdout, MAX_OUTPUT); + const err = truncate(entry.stderr, MAX_OUTPUT); + const status = entry.completedAt ? (entry.exitCode === 0 ? 'completed' : 'failed') : 'running'; + + return { + processId, + status, + stdout: out.text, + stderr: err.text, + exitCode: entry.exitCode, + durationMs: (entry.completedAt || Date.now()) - entry.startedAt, + truncated: out.truncated || err.truncated, + }; +} + +/** + * List all background processes with summary info. + */ +export function listBackgroundJobs() { + const jobs = []; + for (const [id, entry] of backgroundProcesses) { + const status = entry.completedAt ? (entry.exitCode === 0 ? 'completed' : 'failed') : 'running'; + jobs.push({ + processId: id, + status, + command: entry.command, + startedAt: new Date(entry.startedAt).toISOString(), + durationMs: (entry.completedAt || Date.now()) - entry.startedAt, + exitCode: entry.exitCode, + }); + } + return jobs; +} diff --git a/helper-apps/cortex-workspace/lib/system.js b/helper-apps/cortex-workspace/lib/system.js new file mode 100644 index 00000000..08fe5344 --- /dev/null +++ b/helper-apps/cortex-workspace/lib/system.js @@ -0,0 +1,881 @@ +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync, execSync, spawn } from 'node:child_process'; +import { createWriteStream } from 'node:fs'; +import { Readable, Writable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import crypto from 'node:crypto'; +import { listBackgroundJobs } from './shell.js'; + +const startedAt = Date.now(); +const WORKSPACE_DIR = process.env.WORKSPACE_DIR || '/workspace'; +const PERSIST_DIR = process.env.WORKSPACE_PERSIST_DIR || '/persist'; +const BLOB_FILES_DIR = process.env.WORKSPACE_BLOB_FILES_DIR || '/blob-files'; +const CHECKPOINT_NAME = 'workspace.tar.gz'; +const CHECKPOINT_PATH = process.env.WORKSPACE_CHECKPOINT_PATH || path.join(PERSIST_DIR, CHECKPOINT_NAME); +const CHECKPOINT_PREV_PATH = path.join(path.dirname(CHECKPOINT_PATH), 'workspace.prev.tar.gz'); +const CHECKPOINT_TIMEOUT_MS = parseInt(process.env.WORKSPACE_CHECKPOINT_TIMEOUT_MS || '900000', 10); +const CHECKPOINT_BLOCK_SIZE = Math.max( + 1024 * 1024, + parseInt(process.env.WORKSPACE_CHECKPOINT_BLOCK_SIZE_BYTES || String(8 * 1024 * 1024), 10), +); +const CHECKPOINT_GZIP_LEVEL = Math.min( + 9, + Math.max(1, parseInt(process.env.WORKSPACE_CHECKPOINT_GZIP_LEVEL || '1', 10) || 1), +); +const CHECKPOINT_COMPRESSION = (process.env.WORKSPACE_CHECKPOINT_COMPRESSION || 'auto').toLowerCase(); +const WORKSPACE_FILES_DIR = path.join(WORKSPACE_DIR, 'files'); +const CHECKPOINT_EXCLUDES = [ + './files', + './.env', + './.env.*', + './node_modules', + './*/node_modules', + './*/*/node_modules', + './*/*/*/node_modules', + './.npm', + './*/.npm', + './*/*/.npm', + './.pnpm-store', + './*/.pnpm-store', + './*/*/.pnpm-store', + './.yarn/cache', + './*/.yarn/cache', + './*/*/.yarn/cache', + './.bun/install/cache', + './*/.bun/install/cache', + './*/*/.bun/install/cache', + './__pycache__', + './*/__pycache__', + './*/*/__pycache__', + './*/*/*/__pycache__', + './.pytest_cache', + './*/.pytest_cache', + './*/*/.pytest_cache', + './.mypy_cache', + './*/.mypy_cache', + './*/*/.mypy_cache', + './.ruff_cache', + './*/.ruff_cache', + './*/*/.ruff_cache', + './.tox', + './*/.tox', + './*/*/.tox', + './.venv', + './*/.venv', + './*/*/.venv', + './venv', + './*/venv', + './*/*/venv', + './.next', + './*/.next', + './*/*/.next', + './dist', + './*/dist', + './*/*/dist', + './build', + './*/build', + './*/*/build', + './out', + './*/out', + './*/*/out', + './.turbo', + './*/.turbo', + './*/*/.turbo', + './.vite', + './*/.vite', + './*/*/.vite', + './.parcel-cache', + './*/.parcel-cache', + './*/*/.parcel-cache', + './coverage', + './*/coverage', + './*/*/coverage', + './.expo', + './*/.expo', + './*/*/.expo', + './.metro', + './*/.metro', + './*/*/.metro', +]; +let _curlUploadRunnerOverride = null; +let _blockUploadRunnerOverride = null; +const _commandAvailability = new Map(); + +async function exposeBlobFiles() { + try { + execSync(`umount "${WORKSPACE_FILES_DIR}"`, { stdio: 'ignore', timeout: 5000 }); + } catch { + try { + execSync(`umount -l "${WORKSPACE_FILES_DIR}"`, { stdio: 'ignore', timeout: 5000 }); + } catch { + // Not already mounted, or the platform does not support lazy unmount. + } + } + + try { + await fs.rm(WORKSPACE_FILES_DIR, { recursive: true, force: true }); + } catch (e) { + if (e.code !== 'EBUSY') throw e; + return { mode: 'existing', warning: `${WORKSPACE_FILES_DIR} is busy; leaving existing exposure in place` }; + } + await fs.mkdir(WORKSPACE_FILES_DIR, { recursive: true }); + + try { + execSync(`mount --bind "${BLOB_FILES_DIR}" "${WORKSPACE_FILES_DIR}"`, { stdio: 'ignore', timeout: 5000 }); + return { mode: 'bind' }; + } catch (e) { + try { + await fs.rm(WORKSPACE_FILES_DIR, { recursive: true, force: true }); + } catch (rmErr) { + if (rmErr.code !== 'EBUSY') throw rmErr; + return { mode: 'existing', warning: `${WORKSPACE_FILES_DIR} is busy after bind mount failure: ${e.message}` }; + } + await fs.symlink(BLOB_FILES_DIR, WORKSPACE_FILES_DIR, 'dir'); + return { mode: 'symlink', warning: e.message }; + } +} + +/** + * Get system status: disk, memory, CPU, uptime, processes, background jobs. + */ +export async function getStatus() { + const uptime = Math.floor((Date.now() - startedAt) / 1000); + + // Memory + const totalMem = os.totalmem(); + const freeMem = os.freemem(); + const memory = { + totalMB: Math.round(totalMem / 1024 / 1024), + freeMB: Math.round(freeMem / 1024 / 1024), + usedMB: Math.round((totalMem - freeMem) / 1024 / 1024), + usedPercent: Math.round(((totalMem - freeMem) / totalMem) * 100), + }; + + // CPU load + const loadAvg = os.loadavg(); + const cpu = { + cores: os.cpus().length, + loadAvg1m: loadAvg[0], + loadAvg5m: loadAvg[1], + loadAvg15m: loadAvg[2], + }; + + // Disk usage for /workspace + let disk = {}; + try { + const dfOutput = execSync(`df -B1 "${WORKSPACE_DIR}" 2>/dev/null | tail -1`, { encoding: 'utf8', timeout: 5000 }); + const parts = dfOutput.trim().split(/\s+/); + if (parts.length >= 4) { + const total = parseInt(parts[1], 10); + const used = parseInt(parts[2], 10); + const available = parseInt(parts[3], 10); + disk = { + totalMB: Math.round(total / 1024 / 1024), + usedMB: Math.round(used / 1024 / 1024), + availableMB: Math.round(available / 1024 / 1024), + usedPercent: total > 0 ? Math.round((used / total) * 100) : 0, + }; + } + } catch { + disk = { error: 'Unable to read disk info' }; + } + + // Running processes + let processes = []; + try { + const psOutput = execSync('ps aux --sort=-%mem 2>/dev/null | head -11', { encoding: 'utf8', timeout: 5000 }); + const lines = psOutput.trim().split('\n'); + // Skip header, parse top 10 + for (let i = 1; i < lines.length; i++) { + const parts = lines[i].trim().split(/\s+/); + if (parts.length >= 11) { + processes.push({ + user: parts[0], + pid: parseInt(parts[1], 10), + cpu: parseFloat(parts[2]), + mem: parseFloat(parts[3]), + command: parts.slice(10).join(' ').slice(0, 100), + }); + } + } + } catch { + // ps not available + } + + return { + uptime, + memory, + cpu, + disk, + processes, + backgroundJobs: listBackgroundJobs(), + }; +} + +/** + * Create a durable tarball checkpoint of /workspace. + * Returns the path and size of the created archive. + */ +export async function createBackup() { + const timestamp = new Date().toISOString(); + const started = Date.now(); + const tmpPath = buildCheckpointTmpPath(CHECKPOINT_PATH, process.pid, started); + + try { + await fs.mkdir(path.dirname(CHECKPOINT_PATH), { recursive: true }); + await fs.rm(tmpPath, { force: true }); + + const compression = resolveCheckpointCompression(); + execFileSync('tar', buildCheckpointTarArgs(tmpPath, WORKSPACE_DIR, compression), { + encoding: 'utf8', + timeout: CHECKPOINT_TIMEOUT_MS, + }); + + let previousPath = null; + try { + await fs.copyFile(CHECKPOINT_PATH, CHECKPOINT_PREV_PATH); + previousPath = CHECKPOINT_PREV_PATH; + } catch (e) { + if (e.code !== 'ENOENT') throw e; + } + + await fs.rename(tmpPath, CHECKPOINT_PATH); + + const stat = await fs.stat(CHECKPOINT_PATH); + return { + path: CHECKPOINT_PATH, + previousPath, + sizeBytes: stat.size, + sizeMB: Math.round(stat.size / 1024 / 1024 * 100) / 100, + timestamp, + durationMs: Date.now() - started, + compression: compression.id, + }; + } catch (e) { + try { + await fs.rm(tmpPath, { force: true }); + } catch { + // Best effort cleanup only. + } + return { error: `Backup failed: ${e.message}` }; + } +} + +function normalizeCheckpointCompressionId(value) { + const id = String(value || '').toLowerCase(); + if (id === 'zstd' || id === 'zst') return 'zstd'; + if (id === 'pigz') return 'pigz'; + if (id === 'gzip' || id === 'gz') return 'gzip'; + if (id === 'auto' || id === '') return 'auto'; + throw new Error(`Unsupported checkpoint compression: ${value}`); +} + +function commandAvailable(command) { + if (_commandAvailability.has(command)) return _commandAvailability.get(command); + let available = false; + try { + execFileSync('sh', ['-lc', `command -v ${command}`], { + stdio: 'ignore', + timeout: 1000, + }); + available = true; + } catch { + available = false; + } + _commandAvailability.set(command, available); + return available; +} + +function checkpointCompressionDefinition(id) { + if (id === 'zstd') { + return { + id: 'zstd', + createProgram: 'zstd -T0 -1', + extractProgram: 'zstd', + }; + } + if (id === 'pigz') { + return { + id: 'pigz', + createProgram: `pigz -${CHECKPOINT_GZIP_LEVEL}`, + extractProgram: 'pigz', + }; + } + return { + id: 'gzip', + createProgram: `gzip -${CHECKPOINT_GZIP_LEVEL}`, + extractProgram: 'gzip', + }; +} + +function resolveCheckpointCompression(requested = CHECKPOINT_COMPRESSION, isCommandAvailable = commandAvailable) { + const id = normalizeCheckpointCompressionId(requested); + if (id === 'auto') { + if (isCommandAvailable('zstd')) return checkpointCompressionDefinition('zstd'); + if (isCommandAvailable('pigz')) return checkpointCompressionDefinition('pigz'); + return checkpointCompressionDefinition('gzip'); + } + if ((id === 'zstd' || id === 'pigz') && !isCommandAvailable(id)) { + throw new Error(`Checkpoint compression ${id} requested but ${id} is not installed`); + } + return checkpointCompressionDefinition(id); +} + +async function detectCheckpointCompressionFromFile(archivePath) { + let handle; + try { + handle = await fs.open(archivePath, 'r'); + const buffer = Buffer.alloc(4); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + if (bytesRead >= 4 && buffer[0] === 0x28 && buffer[1] === 0xb5 && buffer[2] === 0x2f && buffer[3] === 0xfd) { + return 'zstd'; + } + if (bytesRead >= 2 && buffer[0] === 0x1f && buffer[1] === 0x8b) { + return 'gzip'; + } + } finally { + if (handle) await handle.close(); + } + return 'gzip'; +} + +function buildCheckpointTarArgs(targetPath, workspaceDir, compression = resolveCheckpointCompression()) { + const resolved = typeof compression === 'string' + ? resolveCheckpointCompression(compression) + : compression; + return [ + `--use-compress-program=${resolved.createProgram}`, + '-cf', + targetPath, + ...CHECKPOINT_EXCLUDES.map(exclude => `--exclude=${exclude}`), + '-C', + workspaceDir, + '.', + ]; +} + +function buildCheckpointExtractArgs(sourcePath, workspaceDir, compression = 'gzip') { + const resolved = typeof compression === 'string' + ? checkpointCompressionDefinition(normalizeCheckpointCompressionId(compression)) + : compression; + return [ + `--use-compress-program=${resolved.extractProgram}`, + '-xf', + sourcePath, + '--no-same-owner', + '-C', + workspaceDir, + ]; +} + +function buildCheckpointTmpPath(checkpointPath, pid = process.pid, started = Date.now()) { + return `${checkpointPath}.${pid}.${started}.tmp`; +} + +/** + * Restore workspace from a tarball at the given path. + */ +export async function restoreBackup(archivePath) { + try { + const stat = await fs.stat(archivePath); + if (!stat.isFile()) { + return { error: `Not a file: ${archivePath}` }; + } + + const compression = await detectCheckpointCompressionFromFile(archivePath); + // Extract to /workspace (overwrites existing files) + execFileSync( + 'tar', + buildCheckpointExtractArgs(archivePath, WORKSPACE_DIR, compression), + { encoding: 'utf8', timeout: CHECKPOINT_TIMEOUT_MS } + ); + + const exposeResult = await exposeBlobFiles(); + + return { + message: 'Workspace restored from backup', + archivePath, + sizeBytes: stat.size, + compression, + filesPathMode: exposeResult.mode, + warning: exposeResult.warning, + }; + } catch (e) { + if (e.code === 'ENOENT') return { error: `Archive not found: ${archivePath}` }; + return { error: `Restore failed: ${e.message}` }; + } +} + +export async function restoreBackupFromUrl(archiveUrl, archivePath = CHECKPOINT_PATH) { + try { + if (!archiveUrl || typeof archiveUrl !== 'string') { + return { error: 'archiveUrl is required' }; + } + if (!archiveUrl.startsWith('https://')) { + return { error: 'archiveUrl must be an HTTPS URL' }; + } + + const targetPath = archivePath || CHECKPOINT_PATH; + const tempPath = `${targetPath}.download`; + await fs.mkdir(path.dirname(targetPath), { recursive: true }); + await fs.rm(tempPath, { force: true }); + + const response = await fetch(archiveUrl); + if (!response.ok || !response.body) { + return { error: `Download failed: ${response.status} ${response.statusText}` }; + } + + await pipeline(Readable.fromWeb(response.body), createWriteStream(tempPath)); + await fs.rename(tempPath, targetPath); + return await restoreBackup(targetPath); + } catch (e) { + try { + const targetPath = archivePath || CHECKPOINT_PATH; + await fs.rm(`${targetPath}.download`, { force: true }); + } catch { + // Best effort cleanup only. + } + return { error: `Restore from URL failed: ${e.message}` }; + } +} + +function parseCheckpointEncryption(encryption) { + if (!encryption) return null; + if (encryption.algorithm !== 'aes-256-gcm') { + throw new Error(`Unsupported checkpoint encryption algorithm: ${encryption.algorithm || 'unknown'}`); + } + if (!encryption.keyBase64 || typeof encryption.keyBase64 !== 'string') { + throw new Error('checkpoint encryption key is required'); + } + const key = Buffer.from(encryption.keyBase64, 'base64'); + if (key.length !== 32) { + throw new Error('checkpoint encryption key must be 32 bytes'); + } + const iv = encryption.ivBase64 + ? Buffer.from(encryption.ivBase64, 'base64') + : crypto.randomBytes(12); + if (iv.length !== 12) { + throw new Error('checkpoint encryption iv must be 12 bytes'); + } + const tag = encryption.tagBase64 ? Buffer.from(encryption.tagBase64, 'base64') : null; + if (tag && tag.length !== 16) { + throw new Error('checkpoint encryption tag must be 16 bytes'); + } + return { + algorithm: 'aes-256-gcm', + key, + keyId: encryption.keyId || null, + iv, + tag, + compression: normalizeCheckpointCompressionId(encryption.compression || encryption.checkpointCompression || 'gzip'), + }; +} + +async function restoreEncryptedBackupFromUrl(archiveUrl, encryption, timeoutMs = CHECKPOINT_TIMEOUT_MS) { + const parsed = parseCheckpointEncryption(encryption); + if (!parsed.tag) { + return { error: 'checkpoint encryption tag is required' }; + } + + const response = await fetch(archiveUrl); + if (!response.ok || !response.body) { + return { error: `Download failed: ${response.status} ${response.statusText}` }; + } + + const decipher = crypto.createDecipheriv(parsed.algorithm, parsed.key, parsed.iv); + decipher.setAuthTag(parsed.tag); + const tar = spawn('tar', buildCheckpointExtractArgs('-', WORKSPACE_DIR, parsed.compression), { + stdio: ['pipe', 'ignore', 'pipe'], + }); + let stderr = ''; + tar.stderr.setEncoding('utf8'); + tar.stderr.on('data', chunk => { stderr = `${stderr}${chunk}`.slice(-8192); }); + const timeout = setTimeout(() => { + tar.kill('SIGTERM'); + }, timeoutMs); + if (timeout.unref) timeout.unref(); + const tarExit = waitForChildClose(tar); + + try { + await pipeline(Readable.fromWeb(response.body), decipher, tar.stdin); + const exitCode = await tarExit; + if (exitCode !== 0) { + return { error: `tar restore failed: ${stderr || `exit ${exitCode}`}` }; + } + return { + message: 'Workspace restored from encrypted backup', + encrypted: true, + compression: parsed.compression, + }; + } catch (e) { + tar.kill('SIGTERM'); + return { error: `Encrypted restore failed: ${e.message}` }; + } finally { + clearTimeout(timeout); + } +} + +export async function restoreBackupFromUrlEncrypted(archiveUrl, encryption) { + try { + if (!archiveUrl || typeof archiveUrl !== 'string') { + return { error: 'archiveUrl is required' }; + } + if (!archiveUrl.startsWith('https://')) { + return { error: 'archiveUrl must be an HTTPS URL' }; + } + return await restoreEncryptedBackupFromUrl(archiveUrl, encryption); + } catch (e) { + return { error: `Restore from encrypted URL failed: ${e.message}` }; + } +} + +function curlConfigValue(value) { + return `"${String(value) + .replace(/[\r\n]/g, ' ') + .replace(/\\/g, '\\\\') + .replace(/"/g, '\\"')}"`; +} + +function buildCurlUploadConfig(archiveUrl, sourcePath, headers) { + return [ + 'fail', + 'silent', + 'show-error', + 'request = "PUT"', + `url = ${curlConfigValue(archiveUrl)}`, + `upload-file = ${curlConfigValue(sourcePath)}`, + 'output = "/dev/null"', + 'write-out = "http_code=%{http_code} time_total=%{time_total} size_upload=%{size_upload} speed_upload=%{speed_upload}\\n"', + ...Object.entries(headers).map(([key, value]) => ( + `header = ${curlConfigValue(`${key}: ${value}`)}` + )), + ].join('\n') + '\n'; +} + +function runCurlUpload(configText, timeoutMs = CHECKPOINT_TIMEOUT_MS) { + if (_curlUploadRunnerOverride) { + return _curlUploadRunnerOverride(configText, timeoutMs); + } + + return new Promise((resolve) => { + const child = spawn('curl', ['--config', '-'], { + stdio: ['pipe', 'pipe', 'pipe'], + }); + + let stdout = ''; + let stderr = ''; + const limitOutput = (current, chunk) => `${current}${chunk}`.slice(-8192); + const timeout = setTimeout(() => { + child.kill('SIGTERM'); + }, timeoutMs); + if (timeout.unref) timeout.unref(); + + child.stdout.setEncoding('utf8'); + child.stderr.setEncoding('utf8'); + child.stdout.on('data', chunk => { stdout = limitOutput(stdout, chunk); }); + child.stderr.on('data', chunk => { stderr = limitOutput(stderr, chunk); }); + child.on('error', error => { + clearTimeout(timeout); + resolve({ success: false, error: error.message, stdout, stderr }); + }); + child.on('close', (code, signal) => { + clearTimeout(timeout); + if (code === 0) { + resolve({ success: true, stdout, stderr }); + return; + } + const status = signal ? `signal ${signal}` : `exit ${code}`; + resolve({ success: false, error: `curl ${status}`, stdout, stderr }); + }); + + child.stdin.end(configText); + }); +} + +export async function uploadBackupToUrl(archiveUrl, archivePath = CHECKPOINT_PATH, metadata = {}) { + try { + if (!archiveUrl || typeof archiveUrl !== 'string') { + return { error: 'archiveUrl is required' }; + } + if (!archiveUrl.startsWith('https://')) { + return { error: 'archiveUrl must be an HTTPS URL' }; + } + + const sourcePath = archivePath || CHECKPOINT_PATH; + const stat = await fs.stat(sourcePath); + if (!stat.isFile()) { + return { error: `Not a file: ${sourcePath}` }; + } + + const headers = { + 'x-ms-blob-type': 'BlockBlob', + 'Content-Type': 'application/gzip', + 'Content-Length': String(stat.size), + Expect: '', + }; + for (const [key, value] of Object.entries(metadata || {})) { + if (value == null) continue; + const safeKey = String(key).replace(/[^A-Za-z0-9_]/g, ''); + if (!safeKey) continue; + headers[`x-ms-meta-${safeKey}`] = String(value); + } + + const started = Date.now(); + const upload = await runCurlUpload(buildCurlUploadConfig(archiveUrl, sourcePath, headers)); + if (!upload.success) { + const detail = [upload.error, upload.stderr, upload.stdout] + .filter(Boolean) + .join(': '); + return { error: `Upload failed: ${detail || 'curl failed'}` }; + } + + return { + message: 'Workspace backup uploaded', + archivePath: sourcePath, + sizeBytes: stat.size, + durationMs: Date.now() - started, + uploadMethod: 'curl', + uploadStats: upload.stdout.trim() || undefined, + }; + } catch (e) { + if (e.code === 'ENOENT') return { error: `Archive not found: ${archivePath || CHECKPOINT_PATH}` }; + return { error: `Upload to URL failed: ${e.message}` }; + } +} + +function appendSasQuery(url, params) { + const separator = url.includes('?') ? '&' : '?'; + return `${url}${separator}${params}`; +} + +function safeMetadataHeaders(metadata = {}) { + const headers = {}; + for (const [key, value] of Object.entries(metadata || {})) { + if (value == null) continue; + const safeKey = String(key).replace(/[^A-Za-z0-9_]/g, ''); + if (!safeKey) continue; + headers[`x-ms-meta-${safeKey}`] = String(value); + } + return headers; +} + +async function uploadBlock(archiveUrl, blockId, chunk) { + if (_blockUploadRunnerOverride) { + return _blockUploadRunnerOverride({ type: 'block', archiveUrl, blockId, chunk }); + } + const response = await fetch(appendSasQuery(archiveUrl, `comp=block&blockid=${encodeURIComponent(blockId)}`), { + method: 'PUT', + headers: { + 'Content-Length': String(chunk.length), + }, + body: chunk, + }); + if (!response.ok) { + throw new Error(`block upload failed: ${response.status} ${response.statusText}`); + } +} + +async function commitBlockList(archiveUrl, blockIds, metadata) { + const body = [ + '', + '', + ...blockIds.map(blockId => `${blockId}`), + '', + ].join(''); + if (_blockUploadRunnerOverride) { + return _blockUploadRunnerOverride({ type: 'commit', archiveUrl, blockIds, body, metadata }); + } + const response = await fetch(appendSasQuery(archiveUrl, 'comp=blocklist'), { + method: 'PUT', + headers: { + 'Content-Length': String(Buffer.byteLength(body)), + 'Content-Type': 'application/xml', + 'x-ms-blob-content-type': 'application/octet-stream', + ...safeMetadataHeaders(metadata), + }, + body, + }); + if (!response.ok) { + throw new Error(`block list commit failed: ${response.status} ${response.statusText}`); + } +} + +function waitForChildClose(child) { + return new Promise((resolve, reject) => { + child.once('error', reject); + child.once('close', resolve); + }); +} + +function createAzureBlockUploadWritable(archiveUrl, options = {}) { + const blockSize = options.blockSize || CHECKPOINT_BLOCK_SIZE; + let pending = Buffer.alloc(0); + let index = 0; + let sizeBytes = 0; + const blockIds = []; + + const uploadPendingBlocks = async (force = false) => { + while (pending.length >= blockSize || (force && pending.length > 0)) { + const chunk = pending.subarray(0, Math.min(blockSize, pending.length)); + pending = pending.subarray(chunk.length); + const blockId = Buffer.from(String(index).padStart(8, '0')).toString('base64'); + index += 1; + blockIds.push(blockId); + sizeBytes += chunk.length; + await uploadBlock(archiveUrl, blockId, chunk); + } + }; + + const writable = new Writable({ + async write(chunk, _encoding, callback) { + try { + pending = pending.length ? Buffer.concat([pending, chunk]) : Buffer.from(chunk); + await uploadPendingBlocks(false); + callback(); + } catch (e) { + callback(e); + } + }, + async final(callback) { + try { + await uploadPendingBlocks(true); + callback(); + } catch (e) { + callback(e); + } + }, + }); + + writable.getUploadState = () => ({ blockIds, sizeBytes }); + return writable; +} + +export async function uploadStreamingBackupToUrl(archiveUrl, metadata = {}, encryption = {}) { + const started = Date.now(); + try { + if (!archiveUrl || typeof archiveUrl !== 'string') { + return { error: 'archiveUrl is required' }; + } + if (!archiveUrl.startsWith('https://')) { + return { error: 'archiveUrl must be an HTTPS URL' }; + } + + const parsed = parseCheckpointEncryption(encryption); + const cipher = crypto.createCipheriv(parsed.algorithm, parsed.key, parsed.iv); + const uploader = createAzureBlockUploadWritable(archiveUrl); + const compression = resolveCheckpointCompression(); + const tar = spawn('tar', buildCheckpointTarArgs('-', WORKSPACE_DIR, compression), { + stdio: ['ignore', 'pipe', 'pipe'], + }); + const tarExit = waitForChildClose(tar); + let stderr = ''; + tar.stderr.setEncoding('utf8'); + tar.stderr.on('data', chunk => { stderr = `${stderr}${chunk}`.slice(-8192); }); + const timeout = setTimeout(() => { + tar.kill('SIGTERM'); + }, CHECKPOINT_TIMEOUT_MS); + if (timeout.unref) timeout.unref(); + + try { + await pipeline(tar.stdout, cipher, uploader); + const exitCode = await tarExit; + if (exitCode !== 0) { + return { error: `tar backup failed: ${stderr || `exit ${exitCode}`}` }; + } + } catch (e) { + tar.kill('SIGTERM'); + return { error: `Encrypted backup stream failed: ${e.message}` }; + } finally { + clearTimeout(timeout); + } + + const tag = cipher.getAuthTag(); + const uploadState = uploader.getUploadState(); + const encryptionMetadata = { + checkpointEncryptionAlgorithm: parsed.algorithm, + checkpointEncryptionKeyId: parsed.keyId || '', + checkpointEncryptionIv: parsed.iv.toString('base64'), + checkpointEncryptionTag: tag.toString('base64'), + checkpointCompression: compression.id, + }; + await commitBlockList(archiveUrl, uploadState.blockIds, { + ...metadata, + ...encryptionMetadata, + }); + + return { + message: 'Encrypted workspace backup uploaded', + encrypted: true, + sizeBytes: uploadState.sizeBytes, + durationMs: Date.now() - started, + uploadMethod: 'azure-block-stream', + compression: compression.id, + encryption: { + algorithm: parsed.algorithm, + keyId: parsed.keyId, + ivBase64: parsed.iv.toString('base64'), + tagBase64: tag.toString('base64'), + compression: compression.id, + }, + }; + } catch (e) { + return { error: `Encrypted upload to URL failed: ${e.message}` }; + } +} + +export const __testables = { + buildCheckpointTarArgs, + buildCheckpointExtractArgs, + buildCheckpointTmpPath, + buildCurlUploadConfig, + createAzureBlockUploadWritable, + detectCheckpointCompressionFromFile, + parseCheckpointEncryption, + resolveCheckpointCompression, + waitForChildClose, + CHECKPOINT_EXCLUDES, + setBlockUploadRunnerForTest(fn) { + _blockUploadRunnerOverride = fn; + }, + setCurlUploadRunnerForTest(fn) { + _curlUploadRunnerOverride = fn; + }, +}; + +/** + * Reset workspace: wipe /workspace except preserved paths. + */ +export async function resetWorkspace(preservePaths = []) { + const workspaceDir = WORKSPACE_DIR; + + const preserveSet = new Set(preservePaths.map((p) => { + const normalized = p.startsWith(WORKSPACE_DIR) + ? p.slice(WORKSPACE_DIR.length) + : p; + return normalized.replace(/^\//, '').replace(/\/$/, ''); + })); + + // Always preserve /workspace/files — it may be a FUSE mount (blobfuse2) + preserveSet.add('files'); + + try { + const entries = await fs.readdir(workspaceDir); + let removed = 0; + + for (const entry of entries) { + if (preserveSet.has(entry)) continue; + const fullPath = `${workspaceDir}/${entry}`; + await fs.rm(fullPath, { recursive: true, force: true }); + removed++; + } + + return { + message: `Workspace reset. Removed ${removed} items.`, + preservedPaths: [...preserveSet], + }; + } catch (e) { + return { error: `Failed to reset workspace: ${e.message}` }; + } +} diff --git a/helper-apps/cortex-workspace/package-lock.json b/helper-apps/cortex-workspace/package-lock.json new file mode 100644 index 00000000..ee83e5a1 --- /dev/null +++ b/helper-apps/cortex-workspace/package-lock.json @@ -0,0 +1,831 @@ +{ + "name": "cortex-workspace", + "version": "1.0.12", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "cortex-workspace", + "version": "1.0.12", + "dependencies": { + "express": "^4.21.0" + }, + "devDependencies": {}, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/accepts": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", + "integrity": "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw==", + "license": "MIT", + "dependencies": { + "mime-types": "~2.1.34", + "negotiator": "0.6.3" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/array-flatten": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz", + "integrity": "sha512-PCVAQswWemu6UdxsDFFX/+gVeYqKAod3D3UVm91jHwynguOwAvYPhx8nNlM++NqRcK6CxxpUafjmhIdKiHibqg==", + "license": "MIT" + }, + "node_modules/body-parser": { + "version": "1.20.4", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.4.tgz", + "integrity": "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.14.0", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/content-disposition": { + "version": "0.5.4", + "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.4.tgz", + "integrity": "sha512-FveZTNuGw04cxlAiWbzi6zTAL/lhehaWbTtgluJh4/E95DqMwTmha3KZN1aAWA8cFIhHzMZUvLevkw5Rqk+tSQ==", + "license": "MIT", + "dependencies": { + "safe-buffer": "5.2.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie": { + "version": "0.7.2", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.7.2.tgz", + "integrity": "sha512-yki5XnKuf750l50uGTllt6kKILY4nQ1eNIQatoXEByZ5dWgnKqbnqmTrBE5B4N7lrMJKQ2ytWMiTO2o0v6Ew/w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/cookie-signature": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.7.tgz", + "integrity": "sha512-NXdYc3dLr47pBkpUCHtKSwIOQXLVn8dZEuywboCOJY/osA0wFSLlSawr3KN8qXJEyX66FcONTH8EIlVuK0yyFA==", + "license": "MIT" + }, + "node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "license": "MIT", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==", + "license": "MIT", + "engines": { + "node": ">= 0.8", + "npm": "1.2.8000 || >= 1.4.16" + } + }, + "node_modules/dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==", + "license": "MIT" + }, + "node_modules/encodeurl": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-2.0.0.tgz", + "integrity": "sha512-Q0n9HRi4m6JuGIV1eFlmvJB7ZEVxu93IrMyiMsGC0lrMJMWzRgx6WGquyfQgZVb31vhGgXnfmPNNXmxnOkRBrg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-object-atoms": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", + "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/escape-html": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz", + "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==", + "license": "MIT" + }, + "node_modules/etag": { + "version": "1.8.1", + "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz", + "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/express": { + "version": "4.22.1", + "resolved": "https://registry.npmjs.org/express/-/express-4.22.1.tgz", + "integrity": "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g==", + "license": "MIT", + "dependencies": { + "accepts": "~1.3.8", + "array-flatten": "1.1.1", + "body-parser": "~1.20.3", + "content-disposition": "~0.5.4", + "content-type": "~1.0.4", + "cookie": "~0.7.1", + "cookie-signature": "~1.0.6", + "debug": "2.6.9", + "depd": "2.0.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "finalhandler": "~1.3.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.0", + "merge-descriptors": "1.0.3", + "methods": "~1.1.2", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "path-to-regexp": "~0.1.12", + "proxy-addr": "~2.0.7", + "qs": "~6.14.0", + "range-parser": "~1.2.1", + "safe-buffer": "5.2.1", + "send": "~0.19.0", + "serve-static": "~1.16.2", + "setprototypeof": "1.2.0", + "statuses": "~2.0.1", + "type-is": "~1.6.18", + "utils-merge": "1.0.1", + "vary": "~1.1.2" + }, + "engines": { + "node": ">= 0.10.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/finalhandler": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.3.2.tgz", + "integrity": "sha512-aA4RyPcd3badbdABGDuTXCMTtOneUCAYH/gxoYRTZlIJdF0YPWuGqiAsIrhNnnqdXGswYk6dGujem4w80UJFhg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "on-finished": "~2.4.1", + "parseurl": "~1.3.3", + "statuses": "~2.0.2", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/forwarded": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz", + "integrity": "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/fresh": { + "version": "0.5.2", + "resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz", + "integrity": "sha512-zJ2mQYM18rEFOudeV4GShTGIQ7RbzA7ozbU9I/XBpm7kqgMywgmylMwXHxZJmkVoYkna9d2pVXVXPdYTP9ej8Q==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "license": "MIT", + "dependencies": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "license": "MIT", + "dependencies": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "license": "MIT", + "dependencies": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + }, + "engines": { + "node": ">= 0.8" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/express" + } + }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "license": "MIT", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", + "license": "ISC" + }, + "node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/merge-descriptors": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", + "integrity": "sha512-gaNvAS7TZ897/rVaZ0nMtAyxNyi/pdbjbAwUpFQpN70GqnVfOiXpeUUMKRBmzXaSQ8DdTX4/0ms62r2K+hE6mQ==", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/methods": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz", + "integrity": "sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/mime/-/mime-1.6.0.tgz", + "integrity": "sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==", + "license": "MIT", + "bin": { + "mime": "cli.js" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/mime-db": { + "version": "1.52.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", + "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/mime-types": { + "version": "2.1.35", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", + "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", + "license": "MIT", + "dependencies": { + "mime-db": "1.52.0" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==", + "license": "MIT" + }, + "node_modules/negotiator": { + "version": "0.6.3", + "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.3.tgz", + "integrity": "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==", + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/on-finished": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", + "integrity": "sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==", + "license": "MIT", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/parseurl": { + "version": "1.3.3", + "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", + "integrity": "sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/path-to-regexp": { + "version": "0.1.12", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.12.tgz", + "integrity": "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ==", + "license": "MIT" + }, + "node_modules/proxy-addr": { + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz", + "integrity": "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg==", + "license": "MIT", + "dependencies": { + "forwarded": "0.2.0", + "ipaddr.js": "1.9.1" + }, + "engines": { + "node": ">= 0.10" + } + }, + "node_modules/qs": { + "version": "6.14.1", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.14.1.tgz", + "integrity": "sha512-4EK3+xJl8Ts67nLYNwqw/dsFVnCf+qR7RgXSK9jEEm9unao3njwMDdmsdvoKBKHzxd7tCYz5e5M+SnMjdtXGQQ==", + "license": "BSD-3-Clause", + "dependencies": { + "side-channel": "^1.1.0" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/range-parser": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.1.tgz", + "integrity": "sha512-Hrgsx+orqoygnmhFbKaHE6c296J+HTAQXoxEF6gNupROmmGJRoyzfG3ccAveqCBrwr/2yxQ5BVd/GTl5agOwSg==", + "license": "MIT", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "license": "MIT", + "dependencies": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", + "license": "MIT" + }, + "node_modules/send": { + "version": "0.19.2", + "resolved": "https://registry.npmjs.org/send/-/send-0.19.2.tgz", + "integrity": "sha512-VMbMxbDeehAxpOtWJXlcUS5E8iXh6QmN+BkRX1GARS3wRaXEEgzCcB10gTQazO42tpNIya8xIyNx8fll1OFPrg==", + "license": "MIT", + "dependencies": { + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "1.2.0", + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "etag": "~1.8.1", + "fresh": "~0.5.2", + "http-errors": "~2.0.1", + "mime": "1.6.0", + "ms": "2.1.3", + "on-finished": "~2.4.1", + "range-parser": "~1.2.1", + "statuses": "~2.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/send/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/serve-static": { + "version": "1.16.3", + "resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.16.3.tgz", + "integrity": "sha512-x0RTqQel6g5SY7Lg6ZreMmsOzncHFU7nhnRWkKgWuMTu5NN0DR5oruckMqRvacAN9d5w6ARnRBXl9xhDCgfMeA==", + "license": "MIT", + "dependencies": { + "encodeurl": "~2.0.0", + "escape-html": "~1.0.3", + "parseurl": "~1.3.3", + "send": "~0.19.1" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==", + "license": "ISC" + }, + "node_modules/side-channel": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.0.tgz", + "integrity": "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3", + "side-channel-list": "^1.0.0", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-list": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.0.tgz", + "integrity": "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA==", + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "license": "MIT", + "dependencies": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==", + "license": "MIT", + "engines": { + "node": ">=0.6" + } + }, + "node_modules/type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "license": "MIT", + "dependencies": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/utils-merge": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz", + "integrity": "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA==", + "license": "MIT", + "engines": { + "node": ">= 0.4.0" + } + }, + "node_modules/vary": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", + "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==", + "license": "MIT", + "engines": { + "node": ">= 0.8" + } + } + } +} diff --git a/helper-apps/cortex-workspace/package.json b/helper-apps/cortex-workspace/package.json new file mode 100644 index 00000000..cb807af6 --- /dev/null +++ b/helper-apps/cortex-workspace/package.json @@ -0,0 +1,18 @@ +{ + "name": "cortex-workspace", + "version": "1.0.12", + "description": "Lightweight workspace client for entity sandboxed Docker containers", + "main": "server.js", + "type": "module", + "scripts": { + "start": "node server.js", + "test": "node --test tests/" + }, + "dependencies": { + "express": "^4.21.0" + }, + "devDependencies": {}, + "engines": { + "node": ">=22.0.0" + } +} diff --git a/helper-apps/cortex-workspace/server.js b/helper-apps/cortex-workspace/server.js new file mode 100644 index 00000000..ab5f7579 --- /dev/null +++ b/helper-apps/cortex-workspace/server.js @@ -0,0 +1,343 @@ +import express from 'express'; +import fs from 'node:fs'; +import path from 'node:path'; +import { execSync as shellExecSync } from 'node:child_process'; +import { pipeline } from 'node:stream/promises'; +import { requireAuth, setSecret } from './lib/auth.js'; +import { execSync, execBackground, getResult, listBackgroundJobs } from './lib/shell.js'; +import { readFile, writeFile, editFile, browseDir } from './lib/files.js'; +import { + getStatus, + resetWorkspace, + createBackup, + restoreBackup, + restoreBackupFromUrl, + restoreBackupFromUrlEncrypted, + uploadBackupToUrl, + uploadStreamingBackupToUrl, +} from './lib/system.js'; + +const { version } = JSON.parse(fs.readFileSync(new URL('./package.json', import.meta.url), 'utf8')); + +const app = express(); +const PORT = parseInt(process.env.PORT || '3100', 10); +const WORKSPACE_DIR = process.env.WORKSPACE_DIR || '/workspace'; +const BLOB_FILES_DIR = process.env.WORKSPACE_BLOB_FILES_DIR || '/blob-files'; +const WORKSPACE_FILES_DIR = path.join(WORKSPACE_DIR, 'files'); + +// Wrap async route handlers so Express 4 catches rejections +const wrap = (fn) => (req, res, next) => fn(req, res, next).catch(next); + +app.use(express.json({ limit: '10mb' })); + +function unmountPath(mountPath) { + try { + shellExecSync(`umount "${mountPath}"`, { stdio: 'ignore', timeout: 5000 }); + return; + } catch { + // Fall through to lazy unmount. + } + + try { + shellExecSync(`umount -l "${mountPath}"`, { stdio: 'ignore', timeout: 5000 }); + } catch { + // Not already mounted, or the platform does not support lazy unmount. + } +} + +function exposeBlobFiles() { + unmountPath(WORKSPACE_FILES_DIR); + + try { + fs.rmSync(WORKSPACE_FILES_DIR, { recursive: true, force: true }); + } catch (e) { + if (e.code !== 'EBUSY') throw e; + return { mode: 'existing', warning: `${WORKSPACE_FILES_DIR} is busy; leaving existing exposure in place` }; + } + fs.mkdirSync(WORKSPACE_FILES_DIR, { recursive: true }); + + try { + shellExecSync(`mount --bind "${BLOB_FILES_DIR}" "${WORKSPACE_FILES_DIR}"`, { stdio: 'ignore', timeout: 5000 }); + return { mode: 'bind' }; + } catch (e) { + try { + fs.rmSync(WORKSPACE_FILES_DIR, { recursive: true, force: true }); + } catch (rmErr) { + if (rmErr.code !== 'EBUSY') throw rmErr; + return { mode: 'existing', warning: `${WORKSPACE_FILES_DIR} is busy after bind mount failure: ${e.message}` }; + } + fs.symlinkSync(BLOB_FILES_DIR, WORKSPACE_FILES_DIR, 'dir'); + return { mode: 'symlink', warning: e.message }; + } +} + +// --- Unauthenticated --- + +app.get('/health', (_req, res) => { + res.json({ status: 'ok', version }); +}); + +// --- Authenticated routes --- + +app.use(requireAuth); + +// Shell execution +app.post('/shell', wrap(async (req, res) => { + const { command, cwd, timeout, background, processId } = req.body; + + if (processId) { + return res.json(getResult(processId)); + } + + if (!command || typeof command !== 'string') { + return res.status(400).json({ error: 'command is required' }); + } + + if (background) { + return res.json(execBackground(command, { cwd, timeout })); + } + + const result = await execSync(command, { cwd, timeout }); + res.json(result); +})); + +// Poll background process result +app.get('/shell/result/:processId', (req, res) => { + res.json(getResult(req.params.processId)); +}); + +// List all background processes +app.get('/shell/jobs', (_req, res) => { + res.json(listBackgroundJobs()); +}); + +// Read file +app.post('/read', wrap(async (req, res) => { + const { path, startLine, endLine, encoding } = req.body; + if (!path || typeof path !== 'string') { + return res.status(400).json({ error: 'path is required' }); + } + res.json(await readFile(path, { startLine, endLine, encoding })); +})); + +// Write file +app.post('/write', wrap(async (req, res) => { + const { path, content, encoding, createDirs } = req.body; + if (!path || typeof path !== 'string') { + return res.status(400).json({ error: 'path is required' }); + } + if (content === undefined || content === null) { + return res.status(400).json({ error: 'content is required' }); + } + res.json(await writeFile(path, content, { encoding, createDirs })); +})); + +// Edit file (search-and-replace) +app.post('/edit', wrap(async (req, res) => { + const { path, oldString, newString, replaceAll } = req.body; + if (!path || typeof path !== 'string') { + return res.status(400).json({ error: 'path is required' }); + } + if (!oldString || typeof oldString !== 'string') { + return res.status(400).json({ error: 'oldString is required' }); + } + if (newString === undefined || newString === null) { + return res.status(400).json({ error: 'newString is required' }); + } + res.json(await editFile(path, oldString, newString, { replaceAll })); +})); + +// Browse directory +app.post('/browse', wrap(async (req, res) => { + const { path: dirPath, recursive, maxDepth } = req.body; + if (!dirPath || typeof dirPath !== 'string') { + return res.status(400).json({ error: 'path is required' }); + } + res.json(await browseDir(dirPath, { recursive, maxDepth })); +})); + +// System status +app.get('/status', wrap(async (_req, res) => { + res.json(await getStatus()); +})); + +// Create backup tarball of /workspace +app.post('/backup', wrap(async (_req, res) => { + res.json(await createBackup()); +})); + +// Restore workspace from a tarball +app.post('/restore', wrap(async (req, res) => { + const { archivePath } = req.body; + if (!archivePath || typeof archivePath !== 'string') { + return res.status(400).json({ error: 'archivePath is required' }); + } + res.json(await restoreBackup(archivePath)); +})); + +// Download and restore a workspace tarball directly from Blob/SAS URL. +app.post('/restore-url', wrap(async (req, res) => { + const { archiveUrl, archivePath, encryption } = req.body; + if (encryption) { + res.json(await restoreBackupFromUrlEncrypted(archiveUrl, encryption)); + return; + } + res.json(await restoreBackupFromUrl(archiveUrl, archivePath)); +})); + +// Upload a workspace tarball directly to Blob/SAS URL. +app.post('/upload-url', wrap(async (req, res) => { + const { archiveUrl, archivePath, metadata } = req.body; + res.json(await uploadBackupToUrl(archiveUrl, archivePath, metadata)); +})); + +// Stream tar/gzip output through encryption directly to Blob blocks. +app.post('/backup-upload-url', wrap(async (req, res) => { + const { archiveUrl, metadata, encryption } = req.body; + res.json(await uploadStreamingBackupToUrl(archiveUrl, metadata, encryption)); +})); + +// Reset workspace +app.post('/reset', wrap(async (req, res) => { + res.json(await resetWorkspace(req.body.preservePaths)); +})); + +// Stream-download a file from the container +app.get('/download', wrap(async (req, res) => { + const filePath = req.query.path; + if (!filePath || typeof filePath !== 'string') { + return res.status(400).json({ error: 'path query parameter is required' }); + } + + let stat; + try { + stat = await fs.promises.stat(filePath); + } catch (err) { + if (err.code === 'ENOENT') return res.status(404).json({ error: `File not found: ${filePath}` }); + if (err.code === 'EACCES') return res.status(403).json({ error: `Permission denied: ${filePath}` }); + throw err; + } + + if (!stat.isFile()) { + return res.status(400).json({ error: `Not a file: ${filePath}` }); + } + + res.setHeader('Content-Length', stat.size); + res.setHeader('Content-Type', 'application/octet-stream'); + const stream = fs.createReadStream(filePath); + await pipeline(stream, res); +})); + +// Stream-upload a file into the container +app.post('/upload', wrap(async (req, res) => { + const filePath = req.query.path; + if (!filePath || typeof filePath !== 'string') { + return res.status(400).json({ error: 'path query parameter is required' }); + } + + // Ensure parent directory exists + await fs.promises.mkdir(path.dirname(filePath), { recursive: true }); + + const ws = fs.createWriteStream(filePath); + await pipeline(req, ws); + + const stat = await fs.promises.stat(filePath); + res.json({ path: filePath, bytesWritten: stat.size }); +})); + +// Reconfigure — rotates secret, injects env vars, mounts blob storage at runtime. +// Used by the warm pool: a pool container starts "clean" and gets reconfigured +// when claimed by a specific entity. +app.post('/reconfigure', wrap(async (req, res) => { + const { secret, blobMount, env } = req.body; + + // 1. Mount blob storage via blobfuse2 (optional) + if (blobMount) { + const { accountName, sasToken, containerName } = blobMount; + if (!accountName || !sasToken || !containerName) { + return res.status(400).json({ error: 'blobMount requires accountName, sasToken, and containerName' }); + } + + const configYaml = [ + 'logging:', + ' type: syslog', + ' level: log_err', + 'azstorage:', + ' type: block', + ` account-name: ${accountName}`, + ` sas: ${sasToken}`, + ` container: ${containerName}`, + ' endpoint: https://' + accountName + '.blob.core.windows.net', + 'file_cache:', + ' path: /tmp/blobfuse2-cache', + ' timeout-sec: 120', + ' max-size-mb: 512', + ].join('\n') + '\n'; + + fs.writeFileSync('/tmp/blobfuse2-reconfig.yaml', configYaml); + fs.mkdirSync('/tmp/blobfuse2-cache', { recursive: true }); + fs.mkdirSync(BLOB_FILES_DIR, { recursive: true }); + + try { + // The image starts without blob credentials for warm-pool/fresh + // containers, so entrypoint may have bind-mounted the plain + // /blob-files directory at /workspace/files. Detach that + // compatibility mount before mounting blobfuse on /blob-files. + unmountPath(WORKSPACE_FILES_DIR); + unmountPath(BLOB_FILES_DIR); + shellExecSync( + `blobfuse2 mount "${BLOB_FILES_DIR}" --config-file=/tmp/blobfuse2-reconfig.yaml --allow-other --set-content-type=true -o nonempty`, + { stdio: 'pipe', timeout: 30000 }, + ); + const exposeResult = exposeBlobFiles(); + if (exposeResult.warning) { + console.warn(`WARNING: bind mount failed; ${WORKSPACE_FILES_DIR} is a symlink to ${BLOB_FILES_DIR}: ${exposeResult.warning}`); + } + } catch (e) { + return res.status(500).json({ error: `blobfuse2 mount failed: ${e.stderr?.toString() || e.message}` }); + } + } + + // 2. Inject environment variables (optional) + if (env && typeof env === 'object') { + const SAFE_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/; + const envContent = Object.entries(env) + .filter(([k]) => SAFE_KEY.test(k)) + .map(([k, v]) => { + const escaped = String(v).replace(/'/g, "'\\''"); + return `export ${k}='${escaped}'`; + }) + .join('\n') + '\n'; + + fs.writeFileSync('/workspace/.env', envContent); + + // Ensure .bashrc sources .env + const sourceLine = '[ -f /workspace/.env ] && . /workspace/.env'; + const bashrcPath = path.join(process.env.HOME || '/root', '.bashrc'); + try { + const bashrc = fs.existsSync(bashrcPath) ? fs.readFileSync(bashrcPath, 'utf8') : ''; + if (!bashrc.includes(sourceLine)) { + fs.appendFileSync(bashrcPath, '\n' + sourceLine + '\n'); + } + } catch { + // Best-effort + } + } + + // 3. Rotate secret (done LAST so caller can retry with old secret if 1-2 fail) + if (secret && typeof secret === 'string') { + setSecret(secret); + } + + res.json({ success: true }); +})); + +// Global error handler — async rejections now route here via wrap() +app.use((err, _req, res, _next) => { + console.error('Unhandled error:', err.message); + res.status(500).json({ error: err.message }); +}); + +app.listen(PORT, '0.0.0.0', () => { + console.log(`Workspace client listening on port ${PORT}`); +}); diff --git a/helper-apps/cortex-workspace/tests/files.test.js b/helper-apps/cortex-workspace/tests/files.test.js new file mode 100644 index 00000000..e53fd4dc --- /dev/null +++ b/helper-apps/cortex-workspace/tests/files.test.js @@ -0,0 +1,138 @@ +import { describe, it, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { readFile, writeFile, editFile, browseDir } from '../lib/files.js'; + +const TEST_DIR = '/tmp/workspace-test-files'; + +describe('files', () => { + before(async () => { + await fs.mkdir(TEST_DIR, { recursive: true }); + }); + + after(async () => { + await fs.rm(TEST_DIR, { recursive: true, force: true }); + }); + + describe('writeFile', () => { + it('writes a text file', async () => { + const result = await writeFile(path.join(TEST_DIR, 'test.txt'), 'hello world'); + assert.ok(!result.error); + assert.equal(result.bytesWritten, 11); + const content = await fs.readFile(path.join(TEST_DIR, 'test.txt'), 'utf8'); + assert.equal(content, 'hello world'); + }); + + it('creates parent directories', async () => { + const result = await writeFile(path.join(TEST_DIR, 'sub/dir/file.txt'), 'nested'); + assert.ok(!result.error); + const content = await fs.readFile(path.join(TEST_DIR, 'sub/dir/file.txt'), 'utf8'); + assert.equal(content, 'nested'); + }); + + it('writes base64 content', async () => { + const b64 = Buffer.from('binary data').toString('base64'); + const result = await writeFile(path.join(TEST_DIR, 'binary.bin'), b64, { encoding: 'base64' }); + assert.ok(!result.error); + const content = await fs.readFile(path.join(TEST_DIR, 'binary.bin')); + assert.equal(content.toString(), 'binary data'); + }); + }); + + describe('readFile', () => { + before(async () => { + const lines = Array.from({ length: 50 }, (_, i) => `line ${i + 1}`).join('\n'); + await fs.writeFile(path.join(TEST_DIR, 'multiline.txt'), lines); + }); + + it('reads entire file', async () => { + const result = await readFile(path.join(TEST_DIR, 'test.txt')); + assert.ok(!result.error); + assert.equal(result.content, 'hello world'); + assert.equal(result.truncated, false); + }); + + it('reads line range', async () => { + const result = await readFile(path.join(TEST_DIR, 'multiline.txt'), { startLine: 5, endLine: 10 }); + assert.ok(!result.error); + assert.equal(result.startLine, 5); + assert.equal(result.endLine, 10); + assert.equal(result.returnedLines, 6); + assert.ok(result.content.startsWith('line 5')); + }); + + it('reads as base64', async () => { + const result = await readFile(path.join(TEST_DIR, 'test.txt'), { encoding: 'base64' }); + assert.ok(!result.error); + assert.equal(result.encoding, 'base64'); + assert.equal(Buffer.from(result.content, 'base64').toString(), 'hello world'); + }); + + it('returns error for missing file', async () => { + const result = await readFile(path.join(TEST_DIR, 'nope.txt')); + assert.ok(result.error); + assert.ok(result.error.includes('not found')); + }); + }); + + describe('editFile', () => { + before(async () => { + await fs.writeFile(path.join(TEST_DIR, 'edit.txt'), 'foo bar foo baz'); + }); + + it('replaces first occurrence', async () => { + await fs.writeFile(path.join(TEST_DIR, 'edit-single.txt'), 'foo bar foo baz'); + const result = await editFile(path.join(TEST_DIR, 'edit-single.txt'), 'foo', 'qux'); + assert.ok(!result.error); + assert.equal(result.replacements, 1); + const content = await fs.readFile(path.join(TEST_DIR, 'edit-single.txt'), 'utf8'); + assert.equal(content, 'qux bar foo baz'); + }); + + it('replaces all occurrences', async () => { + await fs.writeFile(path.join(TEST_DIR, 'edit-all.txt'), 'foo bar foo baz'); + const result = await editFile(path.join(TEST_DIR, 'edit-all.txt'), 'foo', 'qux', { replaceAll: true }); + assert.ok(!result.error); + assert.equal(result.replacements, 2); + const content = await fs.readFile(path.join(TEST_DIR, 'edit-all.txt'), 'utf8'); + assert.equal(content, 'qux bar qux baz'); + }); + + it('returns error when string not found', async () => { + const result = await editFile(path.join(TEST_DIR, 'edit.txt'), 'zzz', 'aaa'); + assert.ok(result.error); + assert.ok(result.error.includes('not found')); + }); + }); + + describe('browseDir', () => { + before(async () => { + await fs.mkdir(path.join(TEST_DIR, 'browse/inner'), { recursive: true }); + await fs.writeFile(path.join(TEST_DIR, 'browse/a.txt'), 'a'); + await fs.writeFile(path.join(TEST_DIR, 'browse/inner/b.txt'), 'b'); + }); + + it('lists directory entries', async () => { + const result = await browseDir(path.join(TEST_DIR, 'browse')); + assert.ok(!result.error); + assert.ok(Array.isArray(result.entries)); + const names = result.entries.map(e => e.name); + assert.ok(names.includes('a.txt')); + assert.ok(names.includes('inner')); + }); + + it('supports recursive listing', async () => { + const result = await browseDir(path.join(TEST_DIR, 'browse'), { recursive: true }); + assert.ok(!result.error); + const innerDir = result.entries.find(e => e.name === 'inner'); + assert.ok(innerDir.children); + assert.ok(innerDir.children.some(c => c.name === 'b.txt')); + }); + + it('returns error for missing dir', async () => { + const result = await browseDir(path.join(TEST_DIR, 'nope')); + assert.ok(result.error); + }); + }); +}); diff --git a/helper-apps/cortex-workspace/tests/shell.test.js b/helper-apps/cortex-workspace/tests/shell.test.js new file mode 100644 index 00000000..3ca156fd --- /dev/null +++ b/helper-apps/cortex-workspace/tests/shell.test.js @@ -0,0 +1,96 @@ +import { describe, it, before, after } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs/promises'; +import { execSync, execBackground, getResult, listBackgroundJobs } from '../lib/shell.js'; + +const TEST_DIR = '/tmp/workspace-test-shell'; + +describe('shell', () => { + before(async () => { + await fs.mkdir(TEST_DIR, { recursive: true }); + }); + + after(async () => { + await fs.rm(TEST_DIR, { recursive: true, force: true }); + }); + + describe('execSync', () => { + it('runs a simple command', async () => { + const result = await execSync('echo hello', { cwd: TEST_DIR }); + assert.equal(result.success, true); + assert.equal(result.stdout.trim(), 'hello'); + assert.equal(result.exitCode, 0); + assert.equal(typeof result.durationMs, 'number'); + }); + + it('captures stderr', async () => { + const result = await execSync('echo err >&2', { cwd: TEST_DIR }); + assert.equal(result.stderr.trim(), 'err'); + }); + + it('returns failure for bad command', async () => { + const result = await execSync('exit 42', { cwd: TEST_DIR }); + assert.equal(result.success, false); + assert.equal(result.exitCode, 42); + }); + + it('handles timeout', async () => { + const result = await execSync('sleep 60', { cwd: TEST_DIR, timeout: 500 }); + assert.equal(result.success, false); + assert.equal(result.killed, true); + }); + + it('handles pipelines', async () => { + const result = await execSync('echo "a b c" | tr " " "\\n" | wc -l', { cwd: TEST_DIR }); + assert.equal(result.success, true); + assert.equal(result.stdout.trim(), '3'); + }); + + it('respects cwd', async () => { + const result = await execSync('pwd', { cwd: TEST_DIR }); + assert.equal(result.success, true); + // macOS resolves /tmp -> /private/tmp, so normalize + assert.ok(result.stdout.trim().endsWith(TEST_DIR.replace('/tmp/', ''))); + }); + }); + + describe('execBackground', () => { + it('returns processId immediately', () => { + const result = execBackground('sleep 0.1', { cwd: TEST_DIR }); + assert.ok(result.processId); + assert.equal(typeof result.processId, 'string'); + }); + + it('result is eventually available', async () => { + const { processId } = execBackground('echo bg-done', { cwd: TEST_DIR }); + // Wait for completion + await new Promise(r => setTimeout(r, 500)); + const result = getResult(processId); + assert.equal(result.status, 'completed'); + assert.equal(result.stdout.trim(), 'bg-done'); + }); + + it('shows running status for active process', () => { + const { processId } = execBackground('sleep 1', { cwd: TEST_DIR }); + const result = getResult(processId); + assert.equal(result.status, 'running'); + // Kill the process to clean up + const entry = getResult(processId); + assert.ok(entry); + }); + }); + + describe('getResult', () => { + it('returns error for unknown processId', () => { + const result = getResult('nonexistent'); + assert.ok(result.error); + }); + }); + + describe('listBackgroundJobs', () => { + it('returns array of jobs', () => { + const jobs = listBackgroundJobs(); + assert.ok(Array.isArray(jobs)); + }); + }); +}); diff --git a/helper-apps/cortex-workspace/tests/system.test.js b/helper-apps/cortex-workspace/tests/system.test.js new file mode 100644 index 00000000..748b08cb --- /dev/null +++ b/helper-apps/cortex-workspace/tests/system.test.js @@ -0,0 +1,210 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { getStatus, resetWorkspace, uploadBackupToUrl, __testables } from '../lib/system.js'; +import fs from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { EventEmitter, once } from 'node:events'; + +const TEST_DIR = '/tmp/workspace-test-system'; + +describe('system', () => { + describe('getStatus', () => { + it('returns system info', async () => { + const status = await getStatus(); + assert.equal(typeof status.uptime, 'number'); + assert.ok(status.memory); + assert.equal(typeof status.memory.totalMB, 'number'); + assert.ok(status.cpu); + assert.equal(typeof status.cpu.cores, 'number'); + assert.ok(Array.isArray(status.backgroundJobs)); + }); + }); + + describe('resetWorkspace', () => { + it('removes all files from target directory', async () => { + // Create temp test workspace + await fs.mkdir(`${TEST_DIR}/a`, { recursive: true }); + await fs.writeFile(`${TEST_DIR}/b.txt`, 'b'); + + // Monkey-patch the function to use test dir + // (In real use it always targets /workspace) + // We test the logic by calling with a directory that exists + const result = await resetWorkspace([]); + // This would fail on non-docker because /workspace likely doesn't exist + // but the logic is tested via the browseDir/writeFile tests + assert.ok(result.message || result.error); + + await fs.rm(TEST_DIR, { recursive: true, force: true }); + }); + }); + + describe('uploadBackupToUrl', () => { + it('streams archives through curl config without exposing the SAS URL in argv', async () => { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'workspace-upload-test-')); + const archivePath = path.join(dir, 'workspace.tar.gz'); + await fs.writeFile(archivePath, 'checkpoint-data'); + + let capturedConfig = ''; + try { + __testables.setCurlUploadRunnerForTest(async (configText) => { + capturedConfig = configText; + return { + success: true, + stdout: 'http_code=201 time_total=0.010000 size_upload=15 speed_upload=1500\n', + stderr: '', + }; + }); + + const result = await uploadBackupToUrl( + 'https://account.blob.core.windows.net/container/workspace.tar.gz?sig=secret', + archivePath, + { + 'bad-key': 'value\r\ninjected', + ok_key: 'ok', + }, + ); + + assert.equal(result.error, undefined); + assert.equal(result.uploadMethod, 'curl'); + assert.equal(result.sizeBytes, 15); + assert.match(result.uploadStats, /http_code=201/); + assert.match(capturedConfig, /url = "https:\/\/account\.blob\.core\.windows\.net\/container\/workspace\.tar\.gz\?sig=secret"/); + assert.match(capturedConfig, new RegExp(`upload-file = "${archivePath.replace(/\\/g, '\\\\')}"`)); + assert.match(capturedConfig, /header = "x-ms-blob-type: BlockBlob"/); + assert.match(capturedConfig, /header = "Expect: "/); + assert.match(capturedConfig, /header = "x-ms-meta-badkey: value injected"/); + assert.match(capturedConfig, /header = "x-ms-meta-ok_key: ok"/); + } finally { + __testables.setCurlUploadRunnerForTest(null); + await fs.rm(dir, { recursive: true, force: true }); + } + }); + + it('uploads encrypted streams as bounded Azure blocks', async () => { + const uploaded = []; + try { + __testables.setBlockUploadRunnerForTest(async (event) => { + uploaded.push({ + type: event.type, + blockId: event.blockId, + chunk: event.chunk ? event.chunk.toString('utf8') : null, + }); + }); + + const writable = __testables.createAzureBlockUploadWritable( + 'https://account.blob.core.windows.net/container/workspace.tar.gz?sig=secret', + { blockSize: 4 }, + ); + writable.write(Buffer.from('abc')); + writable.write(Buffer.from('defgh')); + writable.end(); + await once(writable, 'finish'); + + assert.deepEqual(uploaded.map(event => [event.type, event.chunk]), [ + ['block', 'abcd'], + ['block', 'efgh'], + ]); + assert.deepEqual(writable.getUploadState().blockIds, [ + Buffer.from('00000000').toString('base64'), + Buffer.from('00000001').toString('base64'), + ]); + assert.equal(writable.getUploadState().sizeBytes, 8); + } finally { + __testables.setBlockUploadRunnerForTest(null); + } + }); + }); + + describe('checkpoint encryption', () => { + it('requires AES-256-GCM key material', () => { + const keyBase64 = Buffer.alloc(32, 1).toString('base64'); + const parsed = __testables.parseCheckpointEncryption({ + algorithm: 'aes-256-gcm', + keyBase64, + keyId: 'key-1', + }); + + assert.equal(parsed.algorithm, 'aes-256-gcm'); + assert.equal(parsed.key.length, 32); + assert.equal(parsed.iv.length, 12); + assert.equal(parsed.keyId, 'key-1'); + assert.throws( + () => __testables.parseCheckpointEncryption({ algorithm: 'aes-256-gcm', keyBase64: Buffer.alloc(8).toString('base64') }), + /checkpoint encryption key must be 32 bytes/, + ); + }); + }); + + describe('createBackup', () => { + it('excludes secrets and reinstallable caches but preserves shell init files', () => { + const args = __testables.buildCheckpointTarArgs( + '/persist/workspace.tar.gz.tmp', + '/workspace', + __testables.resolveCheckpointCompression('gzip'), + ); + + assert.ok(args.includes('--use-compress-program=gzip -1')); + assert.ok(args.includes('-cf')); + assert.ok(args.includes('--exclude=./files')); + assert.ok(args.includes('--exclude=./.env')); + assert.ok(args.includes('--exclude=./.env.*')); + assert.ok(args.includes('--exclude=./*/node_modules')); + assert.ok(args.includes('--exclude=./.npm')); + assert.ok(args.includes('--exclude=./*/.bun/install/cache')); + assert.ok(args.includes('--exclude=./*/.venv')); + assert.ok(args.includes('--exclude=./*/.next')); + assert.ok(args.includes('--exclude=./*/dist')); + assert.ok(args.includes('--exclude=./*/coverage')); + assert.ok(!args.includes('--exclude=./.bashrc')); + assert.ok(!args.includes('--exclude=./.bash_profile')); + assert.ok(!args.includes('--exclude=./.profile')); + assert.ok(!args.includes('--exclude=./.zshrc')); + assert.ok(!args.includes('--exclude=./.zprofile')); + assert.deepEqual(args.slice(-3), ['-C', '/workspace', '.']); + }); + + it('prefers zstd, then pigz, then gzip for checkpoint compression', () => { + const availability = new Set(['zstd', 'pigz']); + assert.equal( + __testables.resolveCheckpointCompression('auto', command => availability.has(command)).id, + 'zstd', + ); + availability.delete('zstd'); + assert.equal( + __testables.resolveCheckpointCompression('auto', command => availability.has(command)).id, + 'pigz', + ); + availability.delete('pigz'); + assert.equal( + __testables.resolveCheckpointCompression('auto', command => availability.has(command)).id, + 'gzip', + ); + }); + + it('builds tar extraction args for zstd checkpoints', () => { + assert.deepEqual( + __testables.buildCheckpointExtractArgs('-', '/workspace', 'zstd'), + ['--use-compress-program=zstd', '-xf', '-', '--no-same-owner', '-C', '/workspace'], + ); + }); + + it('uses per-call checkpoint temp paths so overlapping backups do not race on rename', () => { + assert.equal( + __testables.buildCheckpointTmpPath('/persist/workspace.tar.gz', 123, 456), + '/persist/workspace.tar.gz.123.456.tmp', + ); + assert.notEqual( + __testables.buildCheckpointTmpPath('/persist/workspace.tar.gz', 123, 456), + __testables.buildCheckpointTmpPath('/persist/workspace.tar.gz', 123, 457), + ); + }); + + it('waits on child close events registered before streaming starts', async () => { + const child = new EventEmitter(); + const close = __testables.waitForChildClose(child); + child.emit('close', 0); + assert.equal(await close, 0); + }); + }); +});