From 444d36606a71874d3d7bda7b75c829e74b2ce292 Mon Sep 17 00:00:00 2001 From: Jason McCartney Date: Tue, 26 May 2026 15:18:29 -0700 Subject: [PATCH] Port workspace runtime from aj-cortex --- config.js | 152 + lib/MongoEntityStore.js | 1139 ++++ lib/blobContainerUtils.js | 126 + lib/crypto.js | 8 +- package-lock.json | 610 ++- package.json | 9 +- .../tools/shared/backends/ACIBackend.js | 378 ++ .../tools/shared/backends/ContainerBackend.js | 86 + .../tools/shared/backends/DockerBackend.js | 253 + .../entity/tools/shared/backends/index.js | 28 + .../entity/tools/shared/sys_entity_tools.js | 285 +- .../system/entity/tools/shared/warmPool.js | 668 +++ .../entity/tools/shared/workspace_client.js | 3919 ++++++++++++++ .../entity/tools/sys_tool_workspace_ssh.js | 384 ++ .../integration/workspace.integration.test.js | 538 ++ .../tools/workspaceBootstrapSecretFix.test.js | 4792 +++++++++++++++++ tests/unit/tools/workspaceReconfigure.test.js | 119 + tests/unit/tools/workspaceSSH.test.js | 56 + tests/unit/tools/workspaceShareName.test.js | 322 ++ 19 files changed, 13725 insertions(+), 147 deletions(-) create mode 100644 lib/MongoEntityStore.js create mode 100644 lib/blobContainerUtils.js create mode 100644 pathways/system/entity/tools/shared/backends/ACIBackend.js create mode 100644 pathways/system/entity/tools/shared/backends/ContainerBackend.js create mode 100644 pathways/system/entity/tools/shared/backends/DockerBackend.js create mode 100644 pathways/system/entity/tools/shared/backends/index.js create mode 100644 pathways/system/entity/tools/shared/warmPool.js create mode 100644 pathways/system/entity/tools/shared/workspace_client.js create mode 100644 pathways/system/entity/tools/sys_tool_workspace_ssh.js create mode 100644 tests/integration/workspace.integration.test.js create mode 100644 tests/unit/tools/workspaceBootstrapSecretFix.test.js create mode 100644 tests/unit/tools/workspaceReconfigure.test.js create mode 100644 tests/unit/tools/workspaceSSH.test.js create mode 100644 tests/unit/tools/workspaceShareName.test.js diff --git a/config.js b/config.js index 6c6e8f80..f8c58854 100644 --- a/config.js +++ b/config.js @@ -959,6 +959,158 @@ var config = convict({ format: String, default: null, env: 'AZURE_FOUNDRY_BING_SEARCH_CONNECTION_ID' + }, + workspaceImage: { + format: String, + default: 'cortex-workspace', + env: 'WORKSPACE_IMAGE' + }, + workspaceImageVersion: { + format: String, + default: '', + env: 'WORKSPACE_IMAGE_VERSION' + }, + workspaceNetwork: { + format: String, + default: 'cortex_workspace', + env: 'WORKSPACE_NETWORK' + }, + workspaceCpus: { + format: String, + default: '1.0', + env: 'WORKSPACE_CPUS' + }, + workspaceMemory: { + format: String, + default: '512m', + env: 'WORKSPACE_MEMORY' + }, + workspaceDiskSize: { + format: String, + default: '10g', + env: 'WORKSPACE_DISK_SIZE' + }, + dockerHost: { + format: String, + default: '', + env: 'DOCKER_HOST', + doc: 'Docker Engine endpoint. Unix socket (unix:///var/run/docker.sock) or TCP (tcp://host:port). Empty = auto-detect local socket.' + }, + workspaceHost: { + format: String, + default: '', + env: 'WORKSPACE_HOST', + doc: 'Hostname/IP for reaching workspace containers. Set when Docker runs on a remote host. Empty = auto (localhost or Docker DNS).' + }, + workspaceIdleTimeoutMs: { + format: Number, + default: 1800000, + env: 'WORKSPACE_IDLE_TIMEOUT_MS', + doc: 'Milliseconds of inactivity before a workspace container is automatically stopped. Default 30 minutes. Set 0 to disable.' + }, + workspaceIdleCheckpointMs: { + format: Number, + default: 900000, + env: 'WORKSPACE_IDLE_CHECKPOINT_MS', + doc: 'Milliseconds of workspace inactivity before an ACI workspace checkpoint is refreshed. Default 15 minutes. Set 0 to checkpoint only at reap time.' + }, + workspaceBackend: { + format: String, + default: 'docker', + env: 'WORKSPACE_BACKEND', + doc: "Container backend: 'docker' (local/remote Docker Engine) or 'aci' (Azure Container Instances)." + }, + workspaceContainerPrefix: { + format: String, + default: 'workspace-local', + env: 'WORKSPACE_CONTAINER_PREFIX', + doc: 'Prefix for workspace ACI container groups. Production should explicitly set "workspace"; non-prod should use env-specific prefixes such as "workspace-dev", "workspace-blue", or "workspace-local".' + }, + warmPoolSize: { + format: Number, + default: 2, + env: 'WARM_POOL_SIZE', + doc: 'Number of pre-provisioned ACI containers in the warm pool. 0 = disabled.' + }, + warmPoolBootstrapSecret: { + format: String, + default: '', + env: 'WARM_POOL_BOOTSTRAP_SECRET', + sensitive: true, + doc: 'Legacy shared warm-pool bootstrap secret. Deprecated and no longer used for new containers.' + }, + warmPoolEnabled: { + format: Boolean, + default: false, + env: 'WARM_POOL_ENABLED', + doc: 'Enable the warm pool for pre-provisioned ACI workspace containers.' + }, + azureSubscriptionId: { + format: String, + default: '', + env: 'AZURE_SUBSCRIPTION_ID' + }, + azureResourceGroup: { + format: String, + default: '', + env: 'AZURE_RESOURCE_GROUP' + }, + azureLocation: { + format: String, + default: 'eastus', + env: 'AZURE_LOCATION' + }, + aciSubnetId: { + format: String, + default: '', + env: 'ACI_SUBNET_ID', + doc: 'Full resource ID of the subnet delegated to ACI (enables private VNet deployment)' + }, + azureAcrServer: { + format: String, + default: '', + env: 'AZURE_ACR_SERVER', + doc: 'Azure Container Registry server (e.g. myacr.azurecr.io)' + }, + azureAcrUsername: { + format: String, + default: '', + env: 'AZURE_ACR_USERNAME', + sensitive: true + }, + azureAcrPassword: { + format: String, + default: '', + env: 'AZURE_ACR_PASSWORD', + sensitive: true + }, + azureStorageAccountName: { + format: String, + default: '', + env: 'AZURE_STORAGE_ACCOUNT_NAME' + }, + azureStorageAccountKey: { + format: String, + default: '', + env: 'AZURE_STORAGE_ACCOUNT_KEY', + sensitive: true + }, + workspaceAzureFilesStorageAccountName: { + format: String, + default: '', + env: 'WORKSPACE_AZURE_FILES_STORAGE_ACCOUNT_NAME' + }, + workspaceAzureFilesStorageAccountKey: { + format: String, + default: '', + env: 'WORKSPACE_AZURE_FILES_STORAGE_ACCOUNT_KEY', + sensitive: true + }, + azureBlobContainerName: { + format: String, + default: '', + env: 'AZURE_BLOB_CONTAINER_NAME', + doc: 'Azure Blob container for user files (blob mount in ACI workspaces)' } }); diff --git a/lib/MongoEntityStore.js b/lib/MongoEntityStore.js new file mode 100644 index 00000000..9ad9fe73 --- /dev/null +++ b/lib/MongoEntityStore.js @@ -0,0 +1,1139 @@ +/** + * MongoDB Entity Store + * + * Manages entity configurations in MongoDB with UUID-based identifiers. + * Entities define AI personas with their identity, tools, and resources. + * + * Schema: + * - id: UUID (primary identifier) + * - name: String (human-readable name, e.g., "Jarvis") + * - isDefault: Boolean (default entity for this deployment) + * - isSystem: Boolean (system entity - hidden from normal entity lists) + * - useMemory: Boolean (enable memory) + * - description: String (entity description for display) + * - identity: String (core identity/persona - renamed from "instructions") + * - avatar: Object (optional visual representation) + * - text: String (optional - text/emoji representation) + * - image: Object (optional - { url, gcs, name }) + * - video: Object (optional - { url, gcs, name }) + * - tools: [String] (tool names - explicit list preferred, ["*"] for all supported for backward compat) + * - resources: [{ url, gcs, name, type }] (attached media/documents) + * - customTools: Object (entity-specific tool definitions) + * - requiredEnvVars: [String] (environment variables required for this entity to be available) + * - personalOwnerId: String (authoritative owner for personal entities only) + * - assocUserIds: [String] (user IDs associated with this entity - for private entities) + * - createdBy: String (userId who created this entity) + * - baseModel: String (optional - base model for the entity, e.g., "gemini-flash-3-vision") + * - reasoningEffort: String (optional - reasoning effort level, e.g., "high", "low") + * - createdAt: Date + * - updatedAt: Date + */ + +import { v4 as uuidv4 } from 'uuid'; +import { MongoClient } from 'mongodb'; +import logger from './logger.js'; + +// Tool migration is optional - will be added later if needed +let migrateToolList = (tools) => tools; +let needsMigration = () => false; + +// Try to import tool migrations (may not exist yet) +try { + const migrations = await import('../pathways/system/entity/tools/shared/tool_migrations.js'); + migrateToolList = migrations.migrateToolList; + needsMigration = migrations.needsMigration; +} catch { + // Tool migrations not available yet - use pass-through +} + +// Default collection name +const DEFAULT_COLLECTION = 'entities'; + +/** + * Singleton instance + * @type {MongoEntityStore|null} + */ +let instance = null; + +function hasMeaningfulPersonalEntityState(entity) { + if (!entity || typeof entity !== 'object') { + return false; + } + + return Boolean( + entity.workspace || + (Array.isArray(entity.resources) && entity.resources.length > 0) || + (entity.customTools && Object.keys(entity.customTools).length > 0) || + (entity.secrets && Object.keys(entity.secrets).length > 0) + ); +} + +function rankPersonalEntityCandidate(entity, userId) { + let score = 0; + + if (entity?.personalOwnerId === userId) score += 1000; + if (entity?.createdBy === userId) score += 500; + if (Array.isArray(entity?.assocUserIds) && entity.assocUserIds.includes(userId)) score += 100; + if (hasMeaningfulPersonalEntityState(entity)) score += 50; + + return score; +} + +export class MongoEntityStore { + /** + * @param {Object} [mongoConfig] + * @param {string} [mongoConfig.collectionName] - Custom collection name + * @param {string} [mongoConfig.databaseName] - Database name (defaults to URI database) + */ + constructor(mongoConfig = {}) { + this.collectionName = mongoConfig.collectionName || DEFAULT_COLLECTION; + this.databaseName = mongoConfig.databaseName || null; + + // Get connection string from environment + this.connectionString = process.env.MONGO_URI || ''; + + // Connection state + this._client = null; + this._db = null; + this._collection = null; + this._connected = false; + + // Cache for entities (loaded on startup) + this._entityCache = new Map(); + this._cacheTimestamps = new Map(); // Track when each entity was last fetched + this._cacheTTL = 10000; // 10 seconds TTL + this._cacheLoaded = false; + } + + /** + * Get or create singleton instance + * @param {Object} [options] + * @returns {MongoEntityStore} + */ + static getInstance(options = {}) { + if (!instance) { + instance = new MongoEntityStore(options); + } + return instance; + } + + /** + * Check if MongoDB is configured + * @returns {boolean} + */ + isConfigured() { + return !!this.connectionString; + } + + /** + * Get or create MongoDB connection + * @private + * @returns {Promise} + */ + async _getCollection() { + if (this._collection && this._connected) { + return this._collection; + } + + if (!this.isConfigured()) { + throw new Error('MongoDB not configured - MONGO_URI not set'); + } + + try { + // Use default connection options - mongodb+srv:// automatically handles TLS + // No explicit TLS options needed (same approach as concierge) + this._client = new MongoClient(this.connectionString); + await this._client.connect(); + + // Get database - priority: explicit config > URI path > fallback + if (this.databaseName) { + this._db = this._client.db(this.databaseName); + } else { + this._db = this._client.db(); + } + + // Verify we have a database name + if (!this._db.databaseName) { + this._db = this._client.db('cortex'); + } + + this._collection = this._db.collection(this.collectionName); + this._connected = true; + + // Ensure unique index on entity id for safe concurrent upserts + try { + await this._collection.createIndex({ id: 1 }, { unique: true, background: true }); + } catch (e) { + // CosmosDB doesn't allow modifying unique indexes after collection creation. + // If the index already exists, this is safe to ignore. + logger.warn(`Could not create unique index on entities (may already exist): ${e.message}`); + } + + // Enforce exactly one personal entity per user without encoding + // semantics into entity ids. Non-personal entities simply omit this field. + try { + await this._collection.createIndex( + { personalOwnerId: 1 }, + { unique: true, sparse: true, background: true }, + ); + } catch (e) { + logger.warn( + `Could not create unique index on entities.personalOwnerId (may already exist or require cleanup first): ${e.message}`, + ); + } + + // Reaper lookup path: one ACI container group name should map to + // at most one runtime workspace record, but keep this non-unique + // to avoid migration risk with old drifted data. + try { + await this._collection.createIndex( + { 'workspace.containerId': 1 }, + { sparse: true, background: true }, + ); + } catch (e) { + logger.warn( + `Could not create sparse index on entities.workspace.containerId (may already exist): ${e.message}`, + ); + } + + logger.info(`Connected to MongoDB entities: ${this._db.databaseName}.${this.collectionName}`); + return this._collection; + } catch (error) { + logger.error(`MongoDB entity store connection failed: ${error.message}`); + throw error; + } + } + + /** + * Close MongoDB connection + */ + async close() { + if (this._client) { + await this._client.close(); + this._client = null; + this._db = null; + this._collection = null; + this._connected = false; + } + this._entityCache.clear(); + this._cacheLoaded = false; + } + + // ==================== ENTITY CRUD OPERATIONS ==================== + + /** + * Load all entities into cache (called on startup) + * @returns {Promise} Entity config object keyed by UUID + */ + async loadAllEntities() { + if (!this.isConfigured()) { + logger.warn('MongoDB not configured - entities will not be available'); + return null; + } + + try { + const collection = await this._getCollection(); + const entities = await collection.find({}).toArray(); + + // Build cache keyed by UUID + const entityConfig = {}; + const now = Date.now(); + const entitiesToMigrate = []; + + for (const entity of entities) { + const { _id, ...entityData } = entity; + + // Check if entity tools need migration + if (needsMigration(entityData.tools)) { + const oldTools = [...entityData.tools]; + entityData.tools = migrateToolList(entityData.tools); + entitiesToMigrate.push({ id: entity.id, tools: entityData.tools }); + logger.info(`Migrating tools for entity ${entityData.name} (${entity.id}): [${oldTools.join(', ')}] -> [${entityData.tools.join(', ')}]`); + } + + // Cache by UUID only with timestamp + this._entityCache.set(entity.id, entityData); + this._cacheTimestamps.set(entity.id, now); + + // Config object keyed by UUID + entityConfig[entity.id] = entityData; + } + + // Persist migrated entities to database + if (entitiesToMigrate.length > 0) { + for (const { id, tools } of entitiesToMigrate) { + try { + await collection.updateOne( + { id }, + { $set: { tools, updatedAt: new Date() } } + ); + } catch (err) { + logger.error(`Failed to persist tool migration for entity ${id}: ${err.message}`); + } + } + logger.info(`Persisted tool migrations for ${entitiesToMigrate.length} entity(ies)`); + } + + this._cacheLoaded = true; + logger.info(`Loaded ${entities.length} entities from MongoDB`); + + return entityConfig; + } catch (error) { + logger.error(`Failed to load entities from MongoDB: ${error.message}`); + return null; + } + } + + /** + * Sync config-defined entities to MongoDB. + * For each config entity, upserts to MongoDB so config always wins. + * Maps `instructions` to `identity` to match MongoDB schema. + * Preserves `createdAt` via $setOnInsert for existing entities. + * + * @param {Object} configEntityMap - Entity config map from config.get('entityConfig') + * @returns {Promise} + */ + async syncConfigEntities(configEntityMap) { + if (!this.isConfigured()) { + logger.warn('MongoDB not configured — skipping config entity sync'); + return; + } + + if (!configEntityMap || typeof configEntityMap !== 'object') { + return; + } + + const entries = Object.entries(configEntityMap); + if (entries.length === 0) { + return; + } + + try { + const collection = await this._getCollection(); + const now = new Date(); + let synced = 0; + + for (const [id, cfg] of entries) { + const doc = { + id, + name: cfg.name || id, + isDefault: cfg.isDefault ?? false, + isSystem: cfg.isSystem ?? false, + useMemory: cfg.useMemory ?? true, + description: cfg.description || '', + identity: cfg.identity || cfg.instructions || '', + avatar: cfg.avatar || null, + voice: cfg.voice || null, + tools: cfg.tools || ['*'], + resources: cfg.resources || cfg.files || [], + customTools: cfg.customTools || {}, + requiredEnvVars: cfg.requiredEnvVars || [], + baseModel: cfg.baseModel || null, + preferredModel: cfg.preferredModel || null, + modelOverride: cfg.modelOverride || null, + reasoningEffort: cfg.reasoningEffort || null, + updatedAt: now, + }; + + // Only overwrite workspace if the config explicitly defines it — + // runtime workspace state must survive config syncs on deploy. + if (Object.hasOwn(cfg, 'workspace')) { + doc.workspace = cfg.workspace || null; + } + + await collection.updateOne( + { id }, + { + $set: doc, + $setOnInsert: { createdAt: now }, + }, + { upsert: true }, + ); + synced++; + } + + logger.info(`Synced ${synced} config entities to MongoDB: [${entries.map(([id]) => id).join(', ')}]`); + } catch (error) { + logger.error(`Failed to sync config entities to MongoDB: ${error.message}`); + } + } + + /** + * Get entity by UUID + * @param {string} entityId - Entity UUID + * @param {Object} [options] + * @param {boolean} [options.fresh=false] - Bypass cache and fetch fresh from MongoDB + * @param {boolean} [options.throwOnError=false] - Throw instead of falling back to cache/null when MongoDB lookup fails + * @returns {Promise} + */ + async getEntity(entityId, options = {}) { + if (!entityId) return null; + + const { fresh = false, throwOnError = false } = options; + + // Check if cache entry is stale (older than TTL) + const cachedTimestamp = this._cacheTimestamps.get(entityId) || 0; + const isStale = Date.now() - cachedTimestamp > this._cacheTTL; + + // Return from cache if: not fresh requested, cache is loaded, entity exists, and not stale + if (!fresh && !isStale && this._cacheLoaded && this._entityCache.has(entityId)) { + const cached = this._entityCache.get(entityId); + return cached ? JSON.parse(JSON.stringify(cached)) : undefined; + } + + if (!this.isConfigured()) { + // Fall back to potentially stale cache if MongoDB not available + if (this._entityCache.has(entityId)) { + const cached = this._entityCache.get(entityId); + return cached ? JSON.parse(JSON.stringify(cached)) : undefined; + } + return null; + } + + try { + const collection = await this._getCollection(); + + // Find by UUID only + const entity = await collection.findOne({ id: entityId }); + + if (entity) { + const { _id, ...entityData } = entity; + + // Check if entity tools need migration + if (needsMigration(entityData.tools)) { + const oldTools = [...entityData.tools]; + entityData.tools = migrateToolList(entityData.tools); + logger.info(`Migrating tools for entity ${entityData.name} (${entity.id}): [${oldTools.join(', ')}] -> [${entityData.tools.join(', ')}]`); + + // Persist migration to database + try { + await collection.updateOne( + { id: entity.id }, + { $set: { tools: entityData.tools, updatedAt: new Date() } } + ); + } catch (err) { + logger.error(`Failed to persist tool migration for entity ${entity.id}: ${err.message}`); + } + } + + // Update cache and timestamp + this._entityCache.set(entity.id, entityData); + this._cacheTimestamps.set(entity.id, Date.now()); + return entityData; + } + + return null; + } catch (error) { + logger.error(`Failed to get entity ${entityId}: ${error.message}`); + if (throwOnError) throw error; + // Fall back to potentially stale cache on error + if (this._entityCache.has(entityId)) { + const cached = this._entityCache.get(entityId); + return cached ? JSON.parse(JSON.stringify(cached)) : undefined; + } + return null; + } + } + + /** + * Find the entity currently assigned to a workspace container group. + * Used by the ACI reaper to avoid scanning every entity on each tick. + * @param {string} containerId + * @returns {Promise} + */ + async getEntityByWorkspaceContainerId(containerId) { + if (!containerId || typeof containerId !== 'string') { + return null; + } + + if (!this.isConfigured()) { + for (const entity of this._entityCache.values()) { + if (entity?.workspace?.containerId === containerId) { + return JSON.parse(JSON.stringify(entity)); + } + } + return null; + } + + try { + const collection = await this._getCollection(); + const matches = await collection + .find({ 'workspace.containerId': containerId }) + .limit(2) + .toArray(); + + if (matches.length === 0) return null; + if (matches.length > 1) { + logger.warn( + `Multiple entities reference workspace.containerId=${containerId}; using the first match`, + ); + } + + const { _id, ...entityData } = matches[0]; + this._entityCache.set(entityData.id, entityData); + this._cacheTimestamps.set(entityData.id, Date.now()); + return entityData; + } catch (error) { + logger.error(`Failed to get entity by workspace container ${containerId}: ${error.message}`); + for (const entity of this._entityCache.values()) { + if (entity?.workspace?.containerId === containerId) { + return JSON.parse(JSON.stringify(entity)); + } + } + throw error; + } + } + + /** + * Get the default entity + * @returns {Promise} + */ + async getDefaultEntity() { + // Check cache first + if (this._cacheLoaded) { + for (const entity of this._entityCache.values()) { + if (entity.isDefault) return JSON.parse(JSON.stringify(entity)); + } + } + + if (!this.isConfigured()) { + return null; + } + + try { + const collection = await this._getCollection(); + const entity = await collection.findOne({ isDefault: true }); + + if (entity) { + const { _id, ...entityData } = entity; + return entityData; + } + + // Fall back to first entity if no default + const firstEntity = await collection.findOne({}); + if (firstEntity) { + const { _id, ...entityData } = firstEntity; + return entityData; + } + + return null; + } catch (error) { + logger.error(`Failed to get default entity: ${error.message}`); + return null; + } + } + + /** + * Get all entities (for sys_get_entities) + * @param {Object} [options] + * @param {boolean} [options.includeSystem=false] - Include system entities + * @param {string} [options.userId] - Filter to entities associated with this user + * @param {boolean} [options.fresh=false] - Bypass cache and fetch fresh from MongoDB + * @returns {Promise} + */ + async getAllEntities(options = {}) { + const { includeSystem = false, userId, fresh = false } = options; + + // Return from cache if loaded + if (!fresh && this._cacheLoaded) { + const entities = []; + const seenIds = new Set(); + + for (const entity of this._entityCache.values()) { + if (entity.id && !seenIds.has(entity.id)) { + // Filter out system entities unless requested + if (!includeSystem && entity.isSystem) { + continue; + } + // Filter by userId if provided + if (userId) { + if (!entity.isSystem) { + const assocUserIds = Array.isArray(entity.assocUserIds) + ? entity.assocUserIds + : []; + const isPublicEntity = assocUserIds.length === 0; + if (!isPublicEntity && !assocUserIds.includes(userId)) { + continue; + } + } + } + seenIds.add(entity.id); + entities.push(entity); + } + } + return entities; + } + + if (!this.isConfigured()) { + return []; + } + + try { + const collection = await this._getCollection(); + + // Build query + const query = {}; + if (!includeSystem) { + query.isSystem = { $ne: true }; + } + if (userId) { + const userFilter = [ + { assocUserIds: { $exists: false } }, + { assocUserIds: { $size: 0 } }, + { assocUserIds: userId } + ]; + + if (includeSystem) { + query.$or = [ + { isSystem: true }, + { isSystem: { $ne: true }, $or: userFilter } + ]; + } else { + query.$or = userFilter; + } + } + + const entities = await collection.find(query).toArray(); + const now = Date.now(); + return entities.map(e => { + const { _id, ...entityData } = e; + this._entityCache.set(entityData.id, entityData); + this._cacheTimestamps.set(entityData.id, now); + return entityData; + }); + } catch (error) { + logger.error(`Failed to get all entities: ${error.message}`); + return []; + } + } + + /** + * Get system entity by name + * @param {string} name - System entity name + * @returns {Promise} + */ + async getSystemEntity(name) { + // Check cache first + if (this._cacheLoaded) { + for (const entity of this._entityCache.values()) { + if (entity.isSystem && entity.name?.toLowerCase() === name.toLowerCase()) { + return JSON.parse(JSON.stringify(entity)); + } + } + } + + if (!this.isConfigured()) { + return null; + } + + try { + const collection = await this._getCollection(); + const escapedName = name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + const entity = await collection.findOne({ + name: { $regex: new RegExp(`^${escapedName}$`, 'i') }, + isSystem: true + }); + + if (entity) { + const { _id, ...entityData } = entity; + this._entityCache.set(entity.id, entityData); + return entityData; + } + + return null; + } catch (error) { + logger.error(`Failed to get system entity ${name}: ${error.message}`); + return null; + } + } + + /** + * Get entities for a specific user + * @param {string} userId - User ID + * @returns {Promise} + */ + async getEntitiesForUser(userId) { + return this.getAllEntities({ userId, includeSystem: false }); + } + + /** + * Add a user association to an entity + * @param {string} entityId - Entity UUID + * @param {string} userId - User ID to associate + * @returns {Promise} + */ + async addUserToEntity(entityId, userId) { + if (!this.isConfigured() || !entityId || !userId) { + return false; + } + + try { + const collection = await this._getCollection(); + + await collection.updateOne( + { id: entityId }, + { + $addToSet: { assocUserIds: userId }, + $set: { updatedAt: new Date() } + } + ); + + // Update local cache + if (this._entityCache.has(entityId)) { + const entity = this._entityCache.get(entityId); + entity.assocUserIds = entity.assocUserIds || []; + if (!entity.assocUserIds.includes(userId)) { + entity.assocUserIds.push(userId); + } + } + + logger.info(`Added user ${userId} to entity ${entityId}`); + return true; + } catch (error) { + logger.error(`Failed to add user to entity: ${error.message}`); + return false; + } + } + + /** + * Remove a user association from an entity + * @param {string} entityId - Entity UUID + * @param {string} userId - User ID to disassociate + * @returns {Promise} + */ + async removeUserFromEntity(entityId, userId) { + if (!this.isConfigured() || !entityId || !userId) { + return false; + } + + try { + const collection = await this._getCollection(); + + await collection.updateOne( + { id: entityId }, + { + $pull: { assocUserIds: userId }, + $set: { updatedAt: new Date() } + } + ); + + // Update local cache + if (this._entityCache.has(entityId)) { + const entity = this._entityCache.get(entityId); + if (entity.assocUserIds && Array.isArray(entity.assocUserIds)) { + entity.assocUserIds = entity.assocUserIds.filter(id => id !== userId); + } + } + + logger.info(`Removed user ${userId} from entity ${entityId}`); + return true; + } catch (error) { + logger.error(`Failed to remove user from entity: ${error.message}`); + return false; + } + } + + /** + * Create or update an entity. + * For updates, fields not present in the input are preserved from the + * existing document — callers can pass partial objects safely without + * accidentally wiping workspace, secrets, or user associations. + * @param {Object} entity - Entity data (partial for updates, full for creates) + * @returns {Promise} Entity ID + */ + async upsertEntity(entity) { + if (!this.isConfigured()) { + logger.warn('MongoDB not configured - cannot store entity'); + return null; + } + + try { + const collection = await this._getCollection(); + const now = new Date(); + + // Generate UUID if not provided + const id = entity.id || uuidv4(); + + // Load existing entity so we can merge — prevents partial updates + // from wiping fields the caller didn't intend to change. + const existingEntity = this._entityCache.get(id) || await this.getEntity(id); + const base = existingEntity || {}; + + const pick = (key, fallback) => + Object.hasOwn(entity, key) ? entity[key] : (base[key] ?? fallback); + + const doc = { + id, + name: pick('name', 'Unnamed Entity') || 'Unnamed Entity', + isDefault: pick('isDefault', false), + isSystem: pick('isSystem', false), + useMemory: pick('useMemory', true), + description: pick('description', ''), + identity: Object.hasOwn(entity, 'identity') + ? (entity.identity || '') + : Object.hasOwn(entity, 'instructions') + ? (entity.instructions || '') + : (base.identity ?? ''), + avatar: pick('avatar', null), + voice: pick('voice', null), + tools: pick('tools', ['*']), + resources: Object.hasOwn(entity, 'resources') + ? (entity.resources || []) + : Object.hasOwn(entity, 'files') + ? (entity.files || []) + : (base.resources ?? []), + customTools: pick('customTools', {}), + requiredEnvVars: pick('requiredEnvVars', []), + personalOwnerId: pick('personalOwnerId', undefined), + assocUserIds: pick('assocUserIds', []), + createdBy: pick('createdBy', null), + baseModel: pick('baseModel', null), + preferredModel: pick('preferredModel', null), + modelOverride: pick('modelOverride', null), + reasoningEffort: pick('reasoningEffort', null), + workspace: pick('workspace', null), + secrets: pick('secrets', null), + updatedAt: now + }; + + if (doc.personalOwnerId == null || doc.personalOwnerId === '') { + delete doc.personalOwnerId; + } + + // If setting as default, unset other defaults first + if (doc.isDefault) { + await collection.updateMany( + { isDefault: true, id: { $ne: id } }, + { $set: { isDefault: false } } + ); + } + + const createdAt = existingEntity?.createdAt || entity.createdAt || now; + + await collection.updateOne( + { id }, + { + $set: doc, + $setOnInsert: { createdAt } + }, + { upsert: true } + ); + + // Update cache + const cachedDoc = { + ...doc, + createdAt + }; + this._entityCache.set(id, cachedDoc); + this._cacheTimestamps.set(id, Date.now()); + + logger.info(`Upserted entity: ${doc.name} (${id})`); + return id; + } catch (error) { + logger.error(`Failed to upsert entity: ${error.message}`); + return null; + } + } + + /** + * Atomically find or create a personal entity for a user. + * Uses findOneAndUpdate with upsert to prevent race conditions + * where concurrent requests both create an entity for the same user. + * @param {string} userId - The user ID + * @param {Object} entityDefaults - Fields to set only on insert + * @returns {Promise<{id: string, name: string, created: boolean}|null>} + */ + async findOrCreatePersonalEntity(userId, entityDefaults) { + if (!this.isConfigured() || !userId) { + return null; + } + + try { + const collection = await this._getCollection(); + const now = new Date(); + const id = uuidv4(); + const candidateQuery = { + isSystem: { $ne: true }, + $or: [ + { personalOwnerId: userId }, + { createdBy: userId }, + ], + }; + const existingCandidates = await collection.find(candidateQuery).toArray(); + const rankedExisting = [...existingCandidates].sort((left, right) => { + const scoreDiff = + rankPersonalEntityCandidate(right, userId) - + rankPersonalEntityCandidate(left, userId); + + if (scoreDiff !== 0) { + return scoreDiff; + } + + const updatedLeft = new Date(left?.updatedAt || left?.createdAt || 0).getTime(); + const updatedRight = new Date(right?.updatedAt || right?.createdAt || 0).getTime(); + return updatedRight - updatedLeft; + }); + const topExisting = rankedExisting[0] || null; + const secondExisting = rankedExisting[1] || null; + + if (topExisting && secondExisting) { + const topScore = rankPersonalEntityCandidate(topExisting, userId); + const secondScore = rankPersonalEntityCandidate(secondExisting, userId); + const bothStateful = + hasMeaningfulPersonalEntityState(topExisting) && + hasMeaningfulPersonalEntityState(secondExisting); + + if (bothStateful || topScore === secondScore) { + logger.error( + `Refusing to create or select a personal entity for ${userId}: ambiguous candidates [${rankedExisting.map(entity => entity.id).join(', ')}]`, + ); + return null; + } + } + + if (topExisting) { + let doc = topExisting; + const ownershipUpdates = {}; + if (doc.personalOwnerId && doc.personalOwnerId !== userId) { + logger.error( + `Refusing to reassign personal entity ${doc.id} from ${doc.personalOwnerId} to ${userId}`, + ); + return null; + } + if (doc.personalOwnerId !== userId) { + ownershipUpdates.personalOwnerId = userId; + } + if (doc.createdBy !== userId) { + ownershipUpdates.createdBy = userId; + } + const missingAssoc = + !Array.isArray(doc.assocUserIds) || !doc.assocUserIds.includes(userId); + + if (Object.keys(ownershipUpdates).length > 0 || missingAssoc) { + const update = { + $set: { + ...ownershipUpdates, + updatedAt: now, + }, + }; + + if (missingAssoc) { + update.$addToSet = { assocUserIds: userId }; + } + + await collection.updateOne({ id: doc.id }, update); + doc = { + ...doc, + ...ownershipUpdates, + assocUserIds: missingAssoc + ? [...new Set([...(doc.assocUserIds || []), userId])] + : doc.assocUserIds, + updatedAt: now, + }; + } + + this._entityCache.set(doc.id, doc); + this._cacheTimestamps.set(doc.id, Date.now()); + logger.info(`Found existing personal entity ${doc.id} for user ${userId}`); + return { id: doc.id, name: doc.name, created: false }; + } + + const filter = { + personalOwnerId: userId, + isSystem: { $ne: true }, + }; + + let result; + try { + result = await collection.findOneAndUpdate( + filter, + { + $setOnInsert: { + id, + name: entityDefaults.name || 'Unnamed Entity', + isDefault: false, + isSystem: false, + useMemory: entityDefaults.useMemory ?? true, + description: entityDefaults.description || '', + identity: entityDefaults.identity || '', + avatar: entityDefaults.avatar || null, + voice: entityDefaults.voice || null, + tools: entityDefaults.tools || ['*'], + resources: entityDefaults.resources || [], + customTools: entityDefaults.customTools || {}, + personalOwnerId: userId, + assocUserIds: entityDefaults.assocUserIds || [userId], + createdBy: userId, + baseModel: entityDefaults.baseModel || null, + preferredModel: entityDefaults.preferredModel || null, + modelOverride: entityDefaults.modelOverride || null, + reasoningEffort: entityDefaults.reasoningEffort || null, + workspace: null, + secrets: null, + createdAt: now, + updatedAt: now, + }, + }, + { + upsert: true, + returnDocument: 'after', + includeResultMetadata: true, + }, + ); + } catch (error) { + if (error?.code === 11000 || /duplicate key/i.test(error?.message || '')) { + const existing = await collection.findOne(filter); + if (!existing) { + throw error; + } + result = { + value: existing, + lastErrorObject: { updatedExisting: true }, + }; + } else { + throw error; + } + } + + let doc = result?.value; + if (!doc) { + return null; + } + const created = Boolean(result?.lastErrorObject?.upserted); + + if (doc.personalOwnerId && doc.personalOwnerId !== userId) { + logger.error( + `Refusing to reassign personal entity ${doc.id} from ${doc.personalOwnerId} to ${userId}`, + ); + return null; + } + if (doc.isSystem) { + logger.error( + `Refusing to use system entity ${doc.id} as a personal entity for ${userId}`, + ); + return null; + } + + const ownershipUpdates = {}; + if (doc.personalOwnerId !== userId) { + ownershipUpdates.personalOwnerId = userId; + } + if (doc.createdBy !== userId) { + ownershipUpdates.createdBy = userId; + } + if (doc.isSystem) { + ownershipUpdates.isSystem = false; + } + const missingAssoc = + !Array.isArray(doc.assocUserIds) || !doc.assocUserIds.includes(userId); + + if (Object.keys(ownershipUpdates).length > 0 || missingAssoc) { + const update = { + $set: { + ...ownershipUpdates, + updatedAt: now, + }, + }; + + if (missingAssoc) { + update.$addToSet = { assocUserIds: userId }; + } + + await collection.updateOne({ id: doc.id }, update); + + doc = { + ...doc, + ...ownershipUpdates, + assocUserIds: missingAssoc + ? [...new Set([...(doc.assocUserIds || []), userId])] + : doc.assocUserIds, + updatedAt: now, + }; + } + + // Update cache + this._entityCache.set(doc.id, doc); + this._cacheTimestamps.set(doc.id, Date.now()); + + if (created) { + logger.info(`Created personal entity ${doc.id} for user ${userId}`); + } else { + logger.info(`Found existing personal entity ${doc.id} for user ${userId}`); + } + + return { id: doc.id, name: doc.name, created }; + } catch (error) { + logger.error(`Failed to find/create personal entity for user ${userId}: ${error.message}`); + return null; + } + } + + /** + * Delete an entity by UUID + * @param {string} entityId - Entity UUID + * @returns {Promise} + */ + async deleteEntity(entityId) { + if (!this.isConfigured() || !entityId) { + return false; + } + + try { + const collection = await this._getCollection(); + + const entity = await this.getEntity(entityId); + if (!entity) { + return false; + } + + await collection.deleteOne({ id: entityId }); + + this._entityCache.delete(entityId); + this._cacheTimestamps.delete(entityId); + + logger.info(`Deleted entity: ${entity.name} (${entityId})`); + return true; + } catch (error) { + logger.error(`Failed to delete entity: ${error.message}`); + return false; + } + } + + /** + * Invalidate cache (force reload on next access) + */ + invalidateCache() { + this._entityCache.clear(); + this._cacheTimestamps.clear(); + this._cacheLoaded = false; + } + + /** + * Check if entities exist in MongoDB + * @returns {Promise} + */ + async hasEntities() { + if (!this.isConfigured()) { + return false; + } + + try { + const collection = await this._getCollection(); + const count = await collection.countDocuments({}, { limit: 1 }); + return count > 0; + } catch (error) { + return false; + } + } +} + +/** + * Get singleton instance + * @param {Object} [options] + * @returns {MongoEntityStore} + */ +export function getEntityStore(options = {}) { + return MongoEntityStore.getInstance(options); +} + +export default MongoEntityStore; diff --git a/lib/blobContainerUtils.js b/lib/blobContainerUtils.js new file mode 100644 index 00000000..74682eba --- /dev/null +++ b/lib/blobContainerUtils.js @@ -0,0 +1,126 @@ +// blobContainerUtils.js +// Utility functions for per-user blob container naming, creation, and SAS generation. + +import { + BlobServiceClient, + StorageSharedKeyCredential, + generateAccountSASQueryParameters, + AccountSASPermissions, + AccountSASResourceTypes, + AccountSASServices, + ContainerSASPermissions, + generateBlobSASQueryParameters, +} from '@azure/storage-blob'; +import logger from './logger.js'; + +function buildContainerName(baseName, sanitized) { + if (!sanitized) return baseName; + return `${baseName}-${sanitized}`; +} + +function sanitizeContainerContextId(contextId) { + return contextId + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/-+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 50); +} + +function sanitizeLegacyContainerContextId(contextId) { + return contextId + .toLowerCase() + .replace(/[^a-z0-9-]/g, '') + .slice(0, 50); +} + +/** + * Derive the per-user blob container name. + * MIRROR: Keep in sync with cortex-file-handler/src/constants.js getUserContainerName() + * @param {string} baseName - Base container name (e.g. 'cortexfiles-local') + * @param {string} [contextId] - User/entity context ID + * @returns {string} Container name: `{baseName}-{contextId}` or baseName if no contextId + */ +export function getUserContainerName(baseName, contextId) { + if (!contextId) return baseName; + return buildContainerName(baseName, sanitizeContainerContextId(contextId)); +} + +/** + * Legacy per-user blob container naming used before scoped contexts preserved + * separators. Keep this for compatibility probes against older blobs. + * MIRROR: Keep in sync with cortex-file-handler/src/constants.js getLegacyUserContainerName() + * @param {string} baseName - Base container name + * @param {string} [contextId] - Legacy compound context ID + * @returns {string} Legacy container name + */ +export function getLegacyUserContainerName(baseName, contextId) { + if (!contextId) return baseName; + return buildContainerName( + baseName, + sanitizeLegacyContainerContextId(contextId), + ); +} + +/** + * Return current and legacy-compatible container names for a context ID. + * MIRROR: Keep in sync with cortex-file-handler/src/constants.js getUserContainerNameCandidates() + * @param {string} baseName - Base container name + * @param {string} [contextId] - Context ID + * @returns {string[]} Candidate container names + */ +export function getUserContainerNameCandidates(baseName, contextId) { + if (!contextId) return [baseName]; + const current = getUserContainerName(baseName, contextId); + const legacy = getLegacyUserContainerName(baseName, contextId); + return legacy === current ? [current] : [current, legacy]; +} + +/** + * Ensure a blob container exists, creating it if necessary. + * @param {string} accountName - Azure storage account name + * @param {string} accountKey - Azure storage account key + * @param {string} containerName - Container name to create + * @returns {Promise} + */ +export async function ensureContainer(accountName, accountKey, containerName) { + const credential = new StorageSharedKeyCredential(accountName, accountKey); + const blobServiceClient = new BlobServiceClient( + `https://${accountName}.blob.core.windows.net`, + credential, + ); + const containerClient = blobServiceClient.getContainerClient(containerName); + try { + await containerClient.createIfNotExists(); + } catch (e) { + // 409 = already exists, which is fine + if (e.statusCode !== 409) { + logger.error(`Failed to ensure container ${containerName}: ${e.message}`); + throw e; + } + } +} + +/** + * Generate a container-scoped SAS token with read/write/delete/list permissions. + * @param {string} accountName - Azure storage account name + * @param {string} accountKey - Azure storage account key + * @param {string} containerName - Target container name + * @param {number} [lifetimeDays=30] - Token lifetime in days + * @returns {string} SAS query string (without leading '?') + */ +export function generateContainerSASToken(accountName, accountKey, containerName, lifetimeDays = 30) { + const credential = new StorageSharedKeyCredential(accountName, accountKey); + + const startsOn = new Date(); + const expiresOn = new Date(startsOn.valueOf() + lifetimeDays * 24 * 60 * 60 * 1000); + + const sasOptions = { + containerName, + permissions: ContainerSASPermissions.parse('rwdl'), // read, write, delete, list + startsOn, + expiresOn, + }; + + return generateBlobSASQueryParameters(sasOptions, credential).toString(); +} diff --git a/lib/crypto.js b/lib/crypto.js index 659b8963..70cfd00a 100644 --- a/lib/crypto.js +++ b/lib/crypto.js @@ -42,16 +42,18 @@ function encrypt(text, key) { function decrypt(message, key) { if (!key) { return message; } try { - // Quick type check - if not string, convert or skip + // Quick type check - only ciphertext is stored as strings if (typeof message !== 'string') { if (Buffer.isBuffer(message)) { message = message.toString('utf8'); } else if (message === null || message === undefined) { - logger.warn(`Decryption skipped: message is ${message === null ? 'null' : 'undefined'}`); return null; + } else if (typeof message === 'object' || typeof message === 'number' || typeof message === 'boolean') { + // Already-deserialized plain data (e.g. Keyv JSON.parse) — not ciphertext + return message; } else { const preview = getMessagePreview(message); - logger.warn(`Decryption skipped: message is not a string (type: ${typeof message}, preview: ${preview})`); + logger.warn(`Decryption skipped: unexpected message type (type: ${typeof message}, preview: ${preview})`); return null; } } diff --git a/package-lock.json b/package-lock.json index 80215cf6..505f441b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,10 @@ "@apollo/server-plugin-response-cache": "^4.1.2", "@apollo/utils.keyvadapter": "^3.0.0", "@aws-sdk/client-s3": "^3.674.0", + "@azure/arm-containerinstance": "^9.1.0", + "@azure/identity": "^4.5.0", "@azure/storage-blob": "^12.24.0", + "@azure/storage-file-share": "^12.24.0", "@azure/storage-queue": "^12.24.0", "@datastructures-js/deque": "^1.0.4", "@dqbd/tiktoken": "^1.0.20", @@ -40,7 +43,9 @@ "handlebars": "^4.7.7", "ioredis": "^5.3.1", "keyv": "^4.5.2", + "mime-db": "^1.52.0", "mime-types": "^2.1.35", + "mongodb": "^6.12.0", "uuid": "^9.0.0", "winston": "^3.11.0", "ws": "^8.12.0", @@ -1234,50 +1239,82 @@ "node": ">=18.0.0" } }, + "node_modules/@azure/arm-containerinstance": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/@azure/arm-containerinstance/-/arm-containerinstance-9.1.0.tgz", + "integrity": "sha512-N9T3/HJwWXvJuz7tin+nO+DYYCTGHILJ5Die3TtdF8Wd1ITfXGqB0vY/wOnspUu/AGojhaIKGmawAfPdw2kX8w==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^1.0.0", + "@azure/core-auth": "^1.3.0", + "@azure/core-client": "^1.7.0", + "@azure/core-lro": "^2.5.0", + "@azure/core-paging": "^1.2.0", + "@azure/core-rest-pipeline": "^1.8.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=14.0.0" + } + }, + "node_modules/@azure/arm-containerinstance/node_modules/@azure/abort-controller": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", + "integrity": "sha512-TrRLIoSQVzfAJX9H1JeFjzAoDGcoK1IYX1UImfceTZpsyYfWr09Ss1aHW1y5TrrR3iq6RZLBwJ3E24uwPhwahw==", + "license": "MIT", + "dependencies": { + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/@azure/core-auth": { - "version": "1.9.0", - "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.9.0.tgz", - "integrity": "sha512-FPwHpZywuyasDSLMqJ6fhbOK3TqUdviZNF8OqRGA4W5Ewib2lEEZ+pBsYcBa88B2NGO/SEnYPGhyBqNlE8ilSw==", + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.10.1.tgz", + "integrity": "sha512-ykRMW8PjVAn+RS6ww5cmK9U2CyH9p4Q88YJwvUslfuMmN98w/2rdGRLPqJYObapBCdzBVeDgYWdJnFPFb7qzpg==", "license": "MIT", "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-util": "^1.11.0", + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", "tslib": "^2.6.2" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.0.0" } }, "node_modules/@azure/core-client": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.9.2.tgz", - "integrity": "sha512-kRdry/rav3fUKHl/aDLd/pDLcB+4pOFwPPTVEExuMyaI5r+JBbMWqRbCY1pn5BniDaU3lRxO9eaQ1AmSMehl/w==", + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.10.1.tgz", + "integrity": "sha512-Nh5PhEOeY6PrnxNPsEHRr9eimxLwgLlpmguQaHKBinFYA/RU9+kOYVOQqOrTsCL+KSxrLLl1gD8Dk5BFW/7l/w==", "license": "MIT", "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.4.0", - "@azure/core-rest-pipeline": "^1.9.1", - "@azure/core-tracing": "^1.0.0", - "@azure/core-util": "^1.6.1", - "@azure/logger": "^1.0.0", + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", "tslib": "^2.6.2" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.0.0" } }, "node_modules/@azure/core-http-compat": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.1.2.tgz", - "integrity": "sha512-5MnV1yqzZwgNLLjlizsU3QqOeQChkIXw781Fwh1xdAqJR5AA32IUaq6xv1BICJvfbHoa+JYcaij2HFkhLbNTJQ==", + "version": "2.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-http-compat/-/core-http-compat-2.4.0.tgz", + "integrity": "sha512-f1P96IB399YiN2ARYHP7EpZi3Bf3wH4SN2lGzrw7JVwm7bbsVYtf2iKSBwTywD2P62NOPZGHFSZi+6jjb75JuA==", "license": "MIT", "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-client": "^1.3.0", - "@azure/core-rest-pipeline": "^1.3.0" + "@azure/abort-controller": "^2.1.2" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.0.0" + }, + "peerDependencies": { + "@azure/core-client": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0" } }, "node_modules/@azure/core-lro": { @@ -1308,47 +1345,47 @@ } }, "node_modules/@azure/core-rest-pipeline": { - "version": "1.18.2", - "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.18.2.tgz", - "integrity": "sha512-IkTf/DWKyCklEtN/WYW3lqEsIaUDshlzWRlZNNwSYtFcCBQz++OtOjxNpm8rr1VcbMS6RpjybQa3u6B6nG0zNw==", + "version": "1.23.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.23.0.tgz", + "integrity": "sha512-Evs1INHo+jUjwHi1T6SG6Ua/LHOQBCLuKEEE6efIpt4ZOoNonaT1kP32GoOcdNDbfqsD2445CPri3MubBy5DEQ==", "license": "MIT", "dependencies": { - "@azure/abort-controller": "^2.0.0", - "@azure/core-auth": "^1.8.0", - "@azure/core-tracing": "^1.0.1", - "@azure/core-util": "^1.11.0", - "@azure/logger": "^1.0.0", - "http-proxy-agent": "^7.0.0", - "https-proxy-agent": "^7.0.0", + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", "tslib": "^2.6.2" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.0.0" } }, "node_modules/@azure/core-tracing": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.2.0.tgz", - "integrity": "sha512-UKTiEJPkWcESPYJz3X5uKRYyOcJD+4nYph+KpfdPRnQJVrZfk0KJgdnaAWKfhsBBtAf/D58Az4AvCJEmWgIBAg==", + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.3.1.tgz", + "integrity": "sha512-9MWKevR7Hz8kNzzPLfX4EAtGM2b8mr50HPDBvio96bURP/9C+HjdH3sBlLSNNrvRAr5/k/svoH457gB5IKpmwQ==", "license": "MIT", "dependencies": { "tslib": "^2.6.2" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.0.0" } }, "node_modules/@azure/core-util": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.11.0.tgz", - "integrity": "sha512-DxOSLua+NdpWoSqULhjDyAZTXFdP/LKkqtYuxxz1SCN289zk3OG8UOpnCQAz/tygyACBtWp/BoO72ptK7msY8g==", + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.13.1.tgz", + "integrity": "sha512-XPArKLzsvl0Hf0CaGyKHUyVgF7oDnhKoP85Xv6M4StF/1AhfORhZudHtOyf2s+FcbuQ9dPRAjB8J2KvRRMUK2A==", "license": "MIT", "dependencies": { - "@azure/abort-controller": "^2.0.0", + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", "tslib": "^2.6.2" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.0.0" } }, "node_modules/@azure/core-xml": { @@ -1364,16 +1401,73 @@ "node": ">=18.0.0" } }, + "node_modules/@azure/identity": { + "version": "4.13.1", + "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.13.1.tgz", + "integrity": "sha512-5C/2WD5Vb1lHnZS16dNQRPMjN6oV/Upba+C9nBIs15PmOi6A3ZGs4Lr2u60zw4S04gi+u3cEXiqTVP7M4Pz3kw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.2", + "@azure/core-rest-pipeline": "^1.17.0", + "@azure/core-tracing": "^1.0.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.0.0", + "@azure/msal-browser": "^5.5.0", + "@azure/msal-node": "^5.1.0", + "open": "^10.1.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@azure/logger": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.1.4.tgz", - "integrity": "sha512-4IXXzcCdLdlXuCG+8UKEwLA1T1NHqUfanhXYHiQTn+6sfWCZXduqbtXDGceg3Ce5QxTGo7EqmbV6Bi+aqKuClQ==", + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.3.0.tgz", + "integrity": "sha512-fCqPIfOcLE+CGqGPd66c8bZpwAji98tZ4JI9i/mlTNTlsIWslCfpg48s/ypyLxZTump5sypjrKn2/kY7q8oAbA==", "license": "MIT", "dependencies": { + "@typespec/ts-http-runtime": "^0.3.0", "tslib": "^2.6.2" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.0.0" + } + }, + "node_modules/@azure/msal-browser": { + "version": "5.11.0", + "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-5.11.0.tgz", + "integrity": "sha512-zkGNYS3TwY8lUpPIafAmsFCYZbgFixY9y/LZB9GUg0IILoHTqpN26j5OrkL1AQThh/YdZsawe4iWXfp85lFVxg==", + "license": "MIT", + "dependencies": { + "@azure/msal-common": "16.6.2" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@azure/msal-common": { + "version": "16.6.2", + "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-16.6.2.tgz", + "integrity": "sha512-hQjjsekAjB00cM1EmatWJlzhEoK2Qhz7Rj5gvM6tYf8iL7RM3tkxlpU9fG0+ofkulzg9AEEA6dIEnSmDr5ZqUA==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@azure/msal-node": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-5.2.2.tgz", + "integrity": "sha512-toS+2AePxqyzb0YOKttDOOiSl3jrkK9aiqIvpurpis0O34QcIS5gToqrgT39p04Dpxw3YoUU0lxJKTpSFFfA6Q==", + "license": "MIT", + "dependencies": { + "@azure/msal-common": "16.6.2", + "jsonwebtoken": "^9.0.0" + }, + "engines": { + "node": ">=20" } }, "node_modules/@azure/storage-blob": { @@ -1400,26 +1494,71 @@ "node": ">=18.0.0" } }, + "node_modules/@azure/storage-common": { + "version": "12.3.0", + "resolved": "https://registry.npmjs.org/@azure/storage-common/-/storage-common-12.3.0.tgz", + "integrity": "sha512-/OFHhy86aG5Pe8dP5tsp+BuJ25JOAl9yaMU3WZbkeoiFMHFtJ7tu5ili7qEdBXNW9G5lDB19trwyI6V49F/8iQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-http-compat": "^2.2.0", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.1.4", + "events": "^3.3.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, + "node_modules/@azure/storage-file-share": { + "version": "12.30.0", + "resolved": "https://registry.npmjs.org/@azure/storage-file-share/-/storage-file-share-12.30.0.tgz", + "integrity": "sha512-OSpZQv/zPvvuSMh/pr9qTrvZYYOlmG4dxesc8ferElrjvADxw1vNId9tEa0rsxPF3kGyuAR5PZuDDdCae37Nag==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", + "@azure/core-http-compat": "^2.0.0", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", + "@azure/core-xml": "^1.4.3", + "@azure/logger": "^1.1.4", + "@azure/storage-common": "^12.2.0", + "events": "^3.0.0", + "tslib": "^2.8.1" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/@azure/storage-queue": { - "version": "12.25.0", - "resolved": "https://registry.npmjs.org/@azure/storage-queue/-/storage-queue-12.25.0.tgz", - "integrity": "sha512-uoobHFbH/o7wIul/sCm32X2YFq6zb1XpNdpKIms9I60mwG3BBaOpEs5pgQV5a5ONG5WMSHlo8E1dNFB5ZZIa1g==", + "version": "12.29.0", + "resolved": "https://registry.npmjs.org/@azure/storage-queue/-/storage-queue-12.29.0.tgz", + "integrity": "sha512-p02H+TbPQWSI/SQ4CG+luoDvpenM+4837NARmOE4oPNOR5vAq7qRyeX72ffyYL2YLnkcyxETh28/bp/TiVIM+g==", "license": "MIT", "dependencies": { "@azure/abort-controller": "^2.1.2", - "@azure/core-auth": "^1.4.0", - "@azure/core-client": "^1.6.2", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.3", "@azure/core-http-compat": "^2.0.0", - "@azure/core-paging": "^1.1.1", - "@azure/core-rest-pipeline": "^1.10.1", - "@azure/core-tracing": "^1.1.2", - "@azure/core-util": "^1.6.1", + "@azure/core-paging": "^1.6.2", + "@azure/core-rest-pipeline": "^1.19.1", + "@azure/core-tracing": "^1.2.0", + "@azure/core-util": "^1.11.0", "@azure/core-xml": "^1.4.3", - "@azure/logger": "^1.0.0", - "tslib": "^2.2.0" + "@azure/logger": "^1.1.4", + "@azure/storage-common": "^12.2.0", + "tslib": "^2.8.1" }, "engines": { - "node": ">=18.0.0" + "node": ">=20.0.0" } }, "node_modules/@colors/colors": { @@ -1539,6 +1678,15 @@ "node": ">= 14" } }, + "node_modules/@mongodb-js/saslprep": { + "version": "1.4.6", + "resolved": "https://registry.npmjs.org/@mongodb-js/saslprep/-/saslprep-1.4.6.tgz", + "integrity": "sha512-y+x3H1xBZd38n10NZF/rEBlvDOOMQ6LKUTHqr8R9VkJ+mmQOYtJFxIlkkK8fZrtOiL6VixbOBWMbZGBdal3Z1g==", + "license": "MIT", + "dependencies": { + "sparse-bitfield": "^3.0.3" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -2559,6 +2707,35 @@ "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", "license": "MIT" }, + "node_modules/@types/webidl-conversions": { + "version": "7.0.3", + "resolved": "https://registry.npmjs.org/@types/webidl-conversions/-/webidl-conversions-7.0.3.tgz", + "integrity": "sha512-CiJJvcRtIgzadHCYXw7dqEnMNRjhGZlYK05Mj9OyktqV8uVT8fD2BFOB7S1uwBE3Kj2Z+4UyPmFw/Ixgw/LAlA==", + "license": "MIT" + }, + "node_modules/@types/whatwg-url": { + "version": "11.0.5", + "resolved": "https://registry.npmjs.org/@types/whatwg-url/-/whatwg-url-11.0.5.tgz", + "integrity": "sha512-coYR071JRaHa+xoEvvYqvnIHaVqaYrLPbsufM9BF63HkwI5Lgmy2QR8Q5K/lYDYo5AK82wOvSOS0UsLTpTG7uQ==", + "license": "MIT", + "dependencies": { + "@types/webidl-conversions": "*" + } + }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.5", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.5.tgz", + "integrity": "sha512-yURCknZhvywvQItHMMmFSo+fq5arCUIyz/CVk7jD89MSai7dkaX8ufjCWp3NttLojoTVbcE72ri+be/TnEbMHw==", + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=20.0.0" + } + }, "node_modules/accepts": { "version": "1.3.8", "resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.8.tgz", @@ -2608,9 +2785,9 @@ } }, "node_modules/agent-base": { - "version": "7.1.3", - "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", - "integrity": "sha512-jRR5wdylq8CkOe6hei19GGZnxM6rBGwFl3Bg0YItGDimvjGtAvdZk4Pu6Cl4u4Igsws4a1fd1Vq3ezrhn4KmFw==", + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", "license": "MIT", "engines": { "node": ">= 14" @@ -2961,12 +3138,36 @@ "node": ">=8" } }, + "node_modules/bson": { + "version": "6.10.4", + "resolved": "https://registry.npmjs.org/bson/-/bson-6.10.4.tgz", + "integrity": "sha512-WIsKqkSC0ABoBJuT1LEX+2HEvNmNKKgnTAyd0fL8qzK4SH2i9NXg+t08YtdZp/V9IZ33cxe3iV4yM0qg8lMQng==", + "license": "Apache-2.0", + "engines": { + "node": ">=16.20.1" + } + }, "node_modules/buffer-equal-constant-time": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", "license": "BSD-3-Clause" }, + "node_modules/bundle-name": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bundle-name/-/bundle-name-4.1.0.tgz", + "integrity": "sha512-tjwM5exMg6BGRI+kNmTntNsvdZS1X8BFYS6tnJ2hdH0kVxM6/eVZ2xy+FqStSWvYmtfFMDLIxurorHwDKfDz5Q==", + "license": "MIT", + "dependencies": { + "run-applescript": "^7.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -3636,6 +3837,34 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/default-browser": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/default-browser/-/default-browser-5.5.0.tgz", + "integrity": "sha512-H9LMLr5zwIbSxrmvikGuI/5KGhZ8E2zH3stkMgM5LpOWDutGM2JZaj460Udnf1a+946zc7YBgrqEWwbk7zHvGw==", + "license": "MIT", + "dependencies": { + "bundle-name": "^4.1.0", + "default-browser-id": "^5.0.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/default-browser-id": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/default-browser-id/-/default-browser-id-5.0.1.tgz", + "integrity": "sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/defer-to-connect": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/defer-to-connect/-/defer-to-connect-2.0.1.tgz", @@ -3646,6 +3875,18 @@ "node": ">=10" } }, + "node_modules/define-lazy-prop": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", + "integrity": "sha512-N+MeXYoqr3pOgn8xfyRPREN7gHakLYjhsHhWGT3fWAiL4IkAt0iDw14QiiEm2bE30c5XX5q0FtAA3CK5f9/BUg==", + "license": "MIT", + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/delayed-stream": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", @@ -4842,6 +5083,21 @@ "node": ">=8" } }, + "node_modules/is-docker": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/is-docker/-/is-docker-3.0.0.tgz", + "integrity": "sha512-eljcgEDlEns/7AXFosB5K/2nCM4P7FQPkGc/DWLy5rmFEWvZayGrik1d9/QIY5nJ4f9YsVvBkA6kJpHn9rISdQ==", + "license": "MIT", + "bin": { + "is-docker": "cli.js" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-error": { "version": "2.2.2", "resolved": "https://registry.npmjs.org/is-error/-/is-error-2.2.2.tgz", @@ -4885,6 +5141,24 @@ "node": ">=0.10.0" } }, + "node_modules/is-inside-container": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-inside-container/-/is-inside-container-1.0.0.tgz", + "integrity": "sha512-KIYLCCJghfHZxqjYBE7rEy0OBuTd5xCHS7tHVgvCLkx7StIoaxwNW3hCALgEUjFfeRk+MG/Qxmp/vtETEF3tRA==", + "license": "MIT", + "dependencies": { + "is-docker": "^3.0.0" + }, + "bin": { + "is-inside-container": "cli.js" + }, + "engines": { + "node": ">=14.16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-number": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", @@ -4937,6 +5211,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-wsl": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-3.1.1.tgz", + "integrity": "sha512-e6rvdUCiQCAuumZslxRJWR/Doq4VpPR82kqclvcS0efgt430SlGIk05vdCN58+VrzgtIcfNODjozVielycD4Sw==", + "license": "MIT", + "dependencies": { + "is-inside-container": "^1.0.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/iterall": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/iterall/-/iterall-1.3.0.tgz", @@ -4968,6 +5257,28 @@ "integrity": "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ==", "license": "MIT" }, + "node_modules/jsonwebtoken": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", + "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==", + "license": "MIT", + "dependencies": { + "jws": "^4.0.1", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, "node_modules/just-extend": { "version": "6.2.0", "resolved": "https://registry.npmjs.org/just-extend/-/just-extend-6.2.0.tgz", @@ -4976,23 +5287,23 @@ "license": "MIT" }, "node_modules/jwa": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", - "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", "license": "MIT", "dependencies": { - "buffer-equal-constant-time": "1.0.1", + "buffer-equal-constant-time": "^1.0.1", "ecdsa-sig-formatter": "1.0.11", "safe-buffer": "^5.0.1" } }, "node_modules/jws": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", - "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", "license": "MIT", "dependencies": { - "jwa": "^2.0.0", + "jwa": "^2.0.1", "safe-buffer": "^5.0.1" } }, @@ -5051,12 +5362,54 @@ "dev": true, "license": "MIT" }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", + "license": "MIT" + }, "node_modules/lodash.isarguments": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/lodash.isarguments/-/lodash.isarguments-3.1.0.tgz", "integrity": "sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==", "license": "MIT" }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", + "license": "MIT" + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", + "license": "MIT" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "license": "MIT" + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", + "license": "MIT" + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "license": "MIT" + }, "node_modules/lodash.sortby": { "version": "4.7.0", "resolved": "https://registry.npmjs.org/lodash.sortby/-/lodash.sortby-4.7.0.tgz", @@ -5211,6 +5564,12 @@ "url": "https://github.com/sindresorhus/mem?sponsor=1" } }, + "node_modules/memory-pager": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/memory-pager/-/memory-pager-1.5.0.tgz", + "integrity": "sha512-ZS4Bp4r/Zoeq6+NLJpP+0Zzm0pR8whtGPf1XExKLJBAczGMnSi3It14OiNCStjQjM6NU1okjQGSxgEZN8eBYKg==", + "license": "MIT" + }, "node_modules/merge-descriptors": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.3.tgz", @@ -5321,6 +5680,62 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/mongodb": { + "version": "6.21.0", + "resolved": "https://registry.npmjs.org/mongodb/-/mongodb-6.21.0.tgz", + "integrity": "sha512-URyb/VXMjJ4da46OeSXg+puO39XH9DeQpWCslifrRn9JWugy0D+DvvBvkm2WxmHe61O/H19JM66p1z7RHVkZ6A==", + "license": "Apache-2.0", + "dependencies": { + "@mongodb-js/saslprep": "^1.3.0", + "bson": "^6.10.4", + "mongodb-connection-string-url": "^3.0.2" + }, + "engines": { + "node": ">=16.20.1" + }, + "peerDependencies": { + "@aws-sdk/credential-providers": "^3.188.0", + "@mongodb-js/zstd": "^1.1.0 || ^2.0.0", + "gcp-metadata": "^5.2.0", + "kerberos": "^2.0.1", + "mongodb-client-encryption": ">=6.0.0 <7", + "snappy": "^7.3.2", + "socks": "^2.7.1" + }, + "peerDependenciesMeta": { + "@aws-sdk/credential-providers": { + "optional": true + }, + "@mongodb-js/zstd": { + "optional": true + }, + "gcp-metadata": { + "optional": true + }, + "kerberos": { + "optional": true + }, + "mongodb-client-encryption": { + "optional": true + }, + "snappy": { + "optional": true + }, + "socks": { + "optional": true + } + } + }, + "node_modules/mongodb-connection-string-url": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/mongodb-connection-string-url/-/mongodb-connection-string-url-3.0.2.tgz", + "integrity": "sha512-rMO7CGo/9BFwyZABcKAWL8UJwH/Kc2x0g72uhDWzG48URRax5TCIcJ7Rc3RZqffZzO/Gwff/jyKwCU9TN8gehA==", + "license": "Apache-2.0", + "dependencies": { + "@types/whatwg-url": "^11.0.2", + "whatwg-url": "^14.1.0 || ^13.0.0" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", @@ -5491,6 +5906,24 @@ "fn.name": "1.x.x" } }, + "node_modules/open": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/open/-/open-10.2.0.tgz", + "integrity": "sha512-YgBpdJHPyQ2UE5x+hlSXcnejzAvD0b22U2OuAP+8OnlJT+PjWPxtgmGqKKc+RgTM63U9gN0YzrYc71R2WT/hTA==", + "license": "MIT", + "dependencies": { + "default-browser": "^5.2.1", + "define-lazy-prop": "^3.0.0", + "is-inside-container": "^1.0.0", + "wsl-utils": "^0.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/p-cancelable": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/p-cancelable/-/p-cancelable-3.0.0.tgz", @@ -6018,6 +6451,18 @@ "node": ">=0.10.0" } }, + "node_modules/run-applescript": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/run-applescript/-/run-applescript-7.1.0.tgz", + "integrity": "sha512-DPe5pVFaAsinSaV6QjQ6gdiedWDcRCbUuiQfQa2wmWV7+xC9bGulGI8+TdRmoFkAPaBXk8CrAbnlY2ISniJ47Q==", + "license": "MIT", + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/run-parallel": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/run-parallel/-/run-parallel-1.2.0.tgz", @@ -6081,7 +6526,6 @@ "version": "7.6.3", "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", - "dev": true, "license": "ISC", "bin": { "semver": "bin/semver.js" @@ -6353,6 +6797,15 @@ "node": ">=0.10.0" } }, + "node_modules/sparse-bitfield": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/sparse-bitfield/-/sparse-bitfield-3.0.3.tgz", + "integrity": "sha512-kvzhi7vqKTfkh0PZU+2D2PIllw2ymqJKujUcyPMd9Y75Nv4nPbGJZXNhxsgdQab2BmlDct1YnfQCguEvHr7VsQ==", + "license": "MIT", + "dependencies": { + "memory-pager": "^1.0.2" + } + }, "node_modules/sprintf-js": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", @@ -6935,6 +7388,21 @@ } } }, + "node_modules/wsl-utils": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/wsl-utils/-/wsl-utils-0.1.0.tgz", + "integrity": "sha512-h3Fbisa2nKGPxCpm89Hk33lBLsnaGBvctQopaBSOW/uIs6FTe1ATyAnKFJrzVs9vpGdsTe73WF3V4lIsk4Gacw==", + "license": "MIT", + "dependencies": { + "is-wsl": "^3.1.0" + }, + "engines": { + "node": ">=18" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/xxhash-wasm": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/xxhash-wasm/-/xxhash-wasm-1.1.0.tgz", diff --git a/package.json b/package.json index ee262a28..b57459c6 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,10 @@ "@apollo/server-plugin-response-cache": "^4.1.2", "@apollo/utils.keyvadapter": "^3.0.0", "@aws-sdk/client-s3": "^3.674.0", + "@azure/arm-containerinstance": "^9.1.0", + "@azure/identity": "^4.5.0", "@azure/storage-blob": "^12.24.0", + "@azure/storage-file-share": "^12.24.0", "@azure/storage-queue": "^12.24.0", "@datastructures-js/deque": "^1.0.4", "@dqbd/tiktoken": "^1.0.20", @@ -64,7 +67,9 @@ "handlebars": "^4.7.7", "ioredis": "^5.3.1", "keyv": "^4.5.2", + "mime-db": "^1.52.0", "mime-types": "^2.1.35", + "mongodb": "^6.12.0", "uuid": "^9.0.0", "winston": "^3.11.0", "ws": "^8.12.0", @@ -82,11 +87,13 @@ }, "ava": { "files": [ - "tests/**/*.test.js" + "tests/**/*.test.js", + "!tests/integration/clientToolCallbacks.test.js" ], "require": [ "dotenv/config" ], + "timeout": "60s", "concurrency": 1 }, "overrides": { diff --git a/pathways/system/entity/tools/shared/backends/ACIBackend.js b/pathways/system/entity/tools/shared/backends/ACIBackend.js new file mode 100644 index 00000000..eb3c9ee3 --- /dev/null +++ b/pathways/system/entity/tools/shared/backends/ACIBackend.js @@ -0,0 +1,378 @@ +// ACIBackend.js +// Azure Container Instances backend for workspace containers. +// Each workspace = one ACI container group with public IP, local workspace +// scratch, local checkpoint scratch, and a blob mount (via blobfuse2 inside +// the container image). +// +// Lifecycle strategy: ACI containers are destroyed after inactivity. Cortex +// stores workspace checkpoints in Blob Storage and restores them into a warm +// container when the workspace is needed again. + +import { config } from '../../../../../../config.js'; +import logger from '../../../../../../lib/logger.js'; +import ContainerBackend from './ContainerBackend.js'; + +function workspaceContainerPrefix() { + return config.get('workspaceContainerPrefix') || 'workspace-local'; +} + +function isLocalAciEnvironment() { + const env = String(config.get('env') || '').toLowerCase(); + return env === 'development' || env === 'test' || env === 'local' || env === 'debug'; +} + +function assertPrivateSubnetForAci(subnetId) { + if (subnetId || isLocalAciEnvironment()) return; + throw new Error('ACI_SUBNET_ID is required for ACI workspaces outside development/test/local environments'); +} + +export default class ACIBackend extends ContainerBackend { + constructor() { + super(); + this._clientPromise = null; + this._shareClientPromise = null; + } + + get backendName() { + return 'aci'; + } + + get healthTimeoutMs() { + return 180000; + } + + get wakeHealthTimeoutMs() { + return this.healthTimeoutMs; + } + + /** Lazy-init the ACI management client (dynamic import to avoid loading Azure SDK for Docker users). */ + async _getClient() { + if (!this._clientPromise) { + this._clientPromise = (async () => { + const { ContainerInstanceManagementClient } = await import('@azure/arm-containerinstance'); + const subscriptionId = config.get('azureSubscriptionId'); + if (!subscriptionId) throw new Error('AZURE_SUBSCRIPTION_ID is required for ACI backend'); + + // Prefer explicit service principal from AZURE_SERVICE_PRINCIPAL_CREDENTIALS, + // fall back to DefaultAzureCredential (managed identity, CLI, etc.) + let credential; + const spCredentials = config.get('azureServicePrincipalCredentials'); + if (spCredentials) { + const parsed = typeof spCredentials === 'string' ? JSON.parse(spCredentials) : spCredentials; + const tenantId = parsed.tenant_id || parsed.tenantId; + const clientId = parsed.client_id || parsed.clientId; + const clientSecret = parsed.client_secret || parsed.clientSecret; + if (tenantId && clientId && clientSecret) { + const { ClientSecretCredential } = await import('@azure/identity'); + credential = new ClientSecretCredential(tenantId, clientId, clientSecret); + } + } + if (!credential) { + const { DefaultAzureCredential } = await import('@azure/identity'); + credential = new DefaultAzureCredential(); + } + + return new ContainerInstanceManagementClient(credential, subscriptionId); + })(); + } + return this._clientPromise; + } + + /** + * Resolve the storage account used for workspace Azure Files shares. + * Prefers the dedicated workspace-files account so the workspace volume + * can live in the same region as the ACI backend, while per-user blob + * storage stays on AZURE_STORAGE_ACCOUNT_NAME (which may be in a + * different region). Falls back to the blob account when not set. + */ + _getWorkspaceFilesAccount() { + const accountName = config.get('workspaceAzureFilesStorageAccountName') + || config.get('azureStorageAccountName'); + const accountKey = config.get('workspaceAzureFilesStorageAccountKey') + || config.get('azureStorageAccountKey'); + return { accountName, accountKey }; + } + + /** Lazy-init the Azure Files share service client. */ + async _getShareClient() { + if (!this._shareClientPromise) { + this._shareClientPromise = (async () => { + const { ShareServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-file-share'); + const { accountName, accountKey } = this._getWorkspaceFilesAccount(); + if (!accountName || !accountKey) throw new Error('WORKSPACE_AZURE_FILES_STORAGE_ACCOUNT_NAME/KEY (or AZURE_STORAGE_ACCOUNT_NAME/KEY as fallback) are required for ACI backend'); + const cred = new StorageSharedKeyCredential(accountName, accountKey); + return new ShareServiceClient(`https://${accountName}.file.core.windows.net`, cred); + })(); + } + return this._shareClientPromise; + } + + /** Ensure an Azure Files share exists for this workspace. */ + async _ensureFileShare(shareName) { + const serviceClient = await this._getShareClient(); + const shareClient = serviceClient.getShareClient(shareName); + try { + await shareClient.create(); + logger.info(`[ACIBackend] Created Azure Files share: ${shareName}`); + } catch (e) { + if (e.statusCode === 409) { + // 409 can mean the share exists OR a soft-deleted share has the same name. + try { + await shareClient.getProperties(); + } catch { + throw new Error( + `Azure Files share '${shareName}' is soft-deleted and cannot be reused. ` + + `Restore or purge it, or disable soft-delete on the storage account.` + ); + } + } else { + throw e; + } + } + } + + async createAndStart({ containerName, image, env, cpus, memoryMB, diskSize, shareName: explicitShareName, mountAzureFiles = false, tags = {} }) { + const subnetId = config.get('aciSubnetId'); + assertPrivateSubnetForAci(subnetId); + + const client = await this._getClient(); + const resourceGroup = config.get('azureResourceGroup'); + const location = config.get('azureLocation'); + const acrServer = config.get('azureAcrServer'); + const acrUsername = config.get('azureAcrUsername'); + const acrPassword = config.get('azureAcrPassword'); + + if (!resourceGroup) throw new Error('AZURE_RESOURCE_GROUP is required for ACI backend'); + + const shareName = explicitShareName || null; + let storageAccountName = null; + let storageAccountKey = null; + if (mountAzureFiles) { + if (!shareName) throw new Error('shareName is required when mounting Azure Files'); + ({ accountName: storageAccountName, accountKey: storageAccountKey } = this._getWorkspaceFilesAccount()); + await this._ensureFileShare(shareName); + } + + const fullImage = image.includes('/') ? image : (acrServer ? `${acrServer}/${image}` : image); + + const environmentVariables = env.map(e => { + const eqIdx = e.indexOf('='); + const name = e.slice(0, eqIdx); + const value = e.slice(eqIdx + 1); + const sensitiveKeys = ['WORKSPACE_SECRET', 'AZURE_BLOB_SAS_TOKEN']; + if (sensitiveKeys.includes(name)) { + return { name, secureValue: value }; + } + return { name, value }; + }); + + logger.info(`[ACIBackend] Creating container group ${containerName} in ${location}`); + + const useVNet = !!subnetId; + + const containerGroupDef = { + location, + osType: 'Linux', + tags: { + managedBy: 'cortex', + cortexId: config.get('cortexId'), + workspaceContainerPrefix: workspaceContainerPrefix(), + ...(shareName ? { shareName } : {}), + imageVersion: config.get('workspaceImageVersion') || '', + ...tags, + }, + // Runtime workspace auth/env are injected by /reconfigure after the + // ACI resource is created. If the process dies, an ACI-level restart + // would boot with the original bootstrap env and silently revert the + // claimed entity workspace. Let Cortex reprovision instead. + restartPolicy: 'Never', + ipAddress: { + type: useVNet ? 'Private' : 'Public', + ports: [{ port: 3100, protocol: 'TCP' }], + ...(useVNet ? {} : { dnsNameLabel: containerName }), + }, + ...(useVNet ? { subnetIds: [{ id: subnetId }] } : {}), + imageRegistryCredentials: acrServer ? [{ + server: acrServer, + username: acrUsername, + password: acrPassword, + }] : undefined, + containers: [{ + name: containerName, + image: fullImage, + resources: { + requests: { + cpu: cpus, + memoryInGB: memoryMB / 1024, + }, + }, + ports: [{ port: 3100, protocol: 'TCP' }], + environmentVariables, + securityContext: { + privileged: true, // Required for blobfuse2 FUSE mount + }, + volumeMounts: [ + { name: 'workspace-vol', mountPath: '/workspace' }, + { name: 'persist-vol', mountPath: '/persist' }, + ], + }], + volumes: [ + { + name: 'workspace-vol', + emptyDir: {}, + }, + { + name: 'persist-vol', + ...(mountAzureFiles ? { azureFile: { + shareName, + storageAccountName, + storageAccountKey, + } } : { emptyDir: {} }), + }, + ], + }; + + // Retry if ACI is still transitioning (e.g. a previous delete is in progress) + let result; + for (let attempt = 0; attempt < 6; attempt++) { + try { + const poller = await client.containerGroups.beginCreateOrUpdate( + resourceGroup, + containerName, + containerGroupDef, + ); + result = await poller.pollUntilDone(); + break; + } catch (e) { + if (e.message?.includes('still transitioning') && attempt < 5) { + logger.info(`[ACIBackend] Container group ${containerName} transitioning, retrying in 10s...`); + await new Promise(r => setTimeout(r, 10000)); + } else { + throw e; + } + } + } + + const url = this._getContainerGroupUrl(result); + + if (!url) { + throw new Error(`ACI container group created but no ${useVNet ? 'private' : 'public'} IP assigned`); + } + + const containerId = containerName; + logger.info(`[ACIBackend] Container group ${containerName} created at ${url}`); + return { containerId, url }; + } + + _getContainerGroupUrl(containerGroup) { + const ip = containerGroup.ipAddress?.ip; + const fqdn = containerGroup.ipAddress?.fqdn; + return ip + ? `http://${ip}:3100` + : (fqdn ? `http://${fqdn}:3100` : null); + } + + async stop(containerId, containerName) { + const client = await this._getClient(); + const resourceGroup = config.get('azureResourceGroup'); + const groupName = containerName || containerId; + + logger.info(`[ACIBackend] Stopping container group ${groupName}`); + await client.containerGroups.stop(resourceGroup, groupName); + } + + async start(containerId, containerName) { + const client = await this._getClient(); + const resourceGroup = config.get('azureResourceGroup'); + const groupName = containerName || containerId; + + logger.info(`[ACIBackend] Starting container group ${groupName}`); + const poller = await client.containerGroups.beginStart(resourceGroup, groupName); + await poller.pollUntilDone(); + + const containerGroup = await client.containerGroups.get(resourceGroup, groupName); + const url = this._getContainerGroupUrl(containerGroup); + if (!url) { + throw new Error(`ACI container group ${groupName} started but no IP assigned`); + } + + logger.info(`[ACIBackend] Container group ${groupName} started at ${url}`); + return { url }; + } + + /** + * Set an `entityId` Azure tag on a container group. Used to mark pool + * containers as claimed so the orphan reconciler can distinguish + * claimed-but-pool-named containers from true orphans without a Mongo lookup. + */ + async setEntityTag(containerName, entityId) { + const client = await this._getClient(); + const resourceGroup = config.get('azureResourceGroup'); + const current = await client.containerGroups.get(resourceGroup, containerName); + await client.containerGroups.update(resourceGroup, containerName, { + tags: { + ...(current.tags || {}), + entityId, + workspaceRole: 'entity', + }, + }); + } + + async listWorkspaceContainers({ prefix = workspaceContainerPrefix() } = {}) { + const client = await this._getClient(); + const resourceGroup = config.get('azureResourceGroup'); + const matchPrefix = `${prefix}-`; + const containers = []; + + for await (const group of client.containerGroups.listByResourceGroup(resourceGroup)) { + if (!group.name?.startsWith(matchPrefix)) continue; + containers.push({ + name: group.name, + location: group.location || null, + tags: group.tags || {}, + image: group.containers?.[0]?.image || null, + provisioningState: group.provisioningState || null, + instanceViewState: group.instanceView?.state || null, + createdAt: group.tags?.createdAt || null, + startedAt: group.containers?.[0]?.instanceView?.currentState?.startTime || null, + ip: group.ipAddress?.ip || null, + fqdn: group.ipAddress?.fqdn || null, + }); + } + + return containers; + } + + async remove(containerId, containerName) { + const client = await this._getClient(); + const resourceGroup = config.get('azureResourceGroup'); + const groupName = containerName || containerId; + + logger.info(`[ACIBackend] Deleting container group ${groupName}`); + try { + const poller = await client.containerGroups.beginDelete(resourceGroup, groupName); + await poller.pollUntilDone(); + } catch (e) { + if (e.statusCode === 404) { + // Already gone + } else { + throw e; + } + } + } + + async destroyVolume(shareName) { + try { + const serviceClient = await this._getShareClient(); + const shareClient = serviceClient.getShareClient(shareName); + await shareClient.delete(); + logger.info(`[ACIBackend] Deleted Azure Files share: ${shareName}`); + } catch (e) { + if (e.statusCode === 404) { + // Share doesn't exist + } else { + logger.warn(`[ACIBackend] Failed to delete share ${shareName}: ${e.message}`); + } + } + } +} diff --git a/pathways/system/entity/tools/shared/backends/ContainerBackend.js b/pathways/system/entity/tools/shared/backends/ContainerBackend.js new file mode 100644 index 00000000..633ae0f1 --- /dev/null +++ b/pathways/system/entity/tools/shared/backends/ContainerBackend.js @@ -0,0 +1,86 @@ +// ContainerBackend.js +// Abstract base class for workspace container backends (Docker, ACI, etc.) + +/** + * Base class defining the container backend interface. + * Subclasses must implement all methods. + */ +export default class ContainerBackend { + /** + * Create and start a new workspace container. + * @param {Object} opts + * @param {string} opts.containerName - Unique container name (e.g. 'workspace-{entityId}') + * @param {string} opts.image - Container image name/tag + * @param {string[]} opts.env - Environment variables (KEY=VALUE format) + * @param {number} opts.cpus - CPU cores (e.g. 1.0) + * @param {number} opts.memoryMB - Memory in megabytes + * @param {string} opts.diskSize - Disk size string (e.g. '10g') — Docker-specific, may be ignored + * @param {string} [opts.shareName] - Persistent volume/share name. Docker uses + * it for the workspace volume; ACI uses it only with mountAzureFiles for + * one-time legacy Azure Files migration. + * @param {boolean} [opts.mountAzureFiles] - ACI-only legacy migration mount. + * @returns {Promise<{containerId: string, url: string}>} + */ + async createAndStart(opts) { + throw new Error('createAndStart() not implemented'); + } + + /** + * Start a stopped container. + * @param {string} containerId - Container/resource identifier + * @param {string} containerName - Human-readable container name + * @returns {Promise} + */ + async start(containerId, containerName) { + throw new Error('start() not implemented'); + } + + /** + * Stop a running container (preserves state for restart). + * @param {string} containerId - Container/resource identifier + * @param {string} containerName - Human-readable container name + * @returns {Promise} + */ + async stop(containerId, containerName) { + throw new Error('stop() not implemented'); + } + + /** + * Remove a container entirely. + * @param {string} containerId - Container/resource identifier + * @param {string} containerName - Human-readable container name + * @returns {Promise} + */ + async remove(containerId, containerName) { + throw new Error('remove() not implemented'); + } + + /** + * Destroy persistent storage associated with a container. + * @param {string} shareName - Volume/share name to destroy (may differ from containerName) + * @returns {Promise} + */ + async destroyVolume(shareName) { + throw new Error('destroyVolume() not implemented'); + } + + /** @returns {string} Backend identifier ('docker' or 'aci') */ + get backendName() { + throw new Error('backendName not implemented'); + } + + /** @returns {number} Milliseconds to wait for container health check after create */ + get healthTimeoutMs() { + throw new Error('healthTimeoutMs not implemented'); + } + + /** + * Milliseconds to wait for health check during wake. + * Short for backends where stop/start are no-ops (container is already hot). + * Same as healthTimeoutMs for backends that truly restart containers. + * @returns {number} + */ + get wakeHealthTimeoutMs() { + return this.healthTimeoutMs; + } +} diff --git a/pathways/system/entity/tools/shared/backends/DockerBackend.js b/pathways/system/entity/tools/shared/backends/DockerBackend.js new file mode 100644 index 00000000..db93bd7e --- /dev/null +++ b/pathways/system/entity/tools/shared/backends/DockerBackend.js @@ -0,0 +1,253 @@ +// DockerBackend.js +// Docker Engine API backend for workspace containers. +// Extracted from workspace_client.js — talks to Docker daemon via Unix socket or TCP. + +import fs from 'node:fs'; +import http from 'node:http'; +import net from 'node:net'; +import { config } from '../../../../../../config.js'; +import logger from '../../../../../../lib/logger.js'; +import ContainerBackend from './ContainerBackend.js'; + +// Detect if Cortex is running inside Docker (production) or on the host (local dev) +const isInDocker = fs.existsSync('/.dockerenv'); + +/** + * Resolve Docker Engine connection. + * Supports three topologies: + * 1. Local dev — auto-detect Unix socket on the host + * 2. Same VM — Cortex in Docker, socket mounted into container + * 3. Remote host — DOCKER_HOST=tcp://host:port (e.g. VM on same VNet) + */ +function resolveDockerConnection() { + const dockerHost = config.get('dockerHost'); + + if (dockerHost && dockerHost.startsWith('tcp://')) { + const url = new URL(dockerHost.replace('tcp://', 'http://')); + return { type: 'tcp', hostname: url.hostname, port: Number(url.port) || 2375 }; + } + + if (dockerHost && dockerHost.startsWith('unix://')) { + const socketPath = dockerHost.replace('unix://', ''); + return { type: 'socket', socketPath }; + } + + // Auto-detect local socket + const socketPath = [ + '/var/run/docker.sock', + `${process.env.HOME}/.docker/run/docker.sock`, + ].find(p => fs.existsSync(p)) || '/var/run/docker.sock'; + return { type: 'socket', socketPath }; +} + +/** + * Parse memory limit string (e.g. '512m', '1g') to bytes. + */ +function parseMemoryLimit(str) { + const match = str.toLowerCase().match(/^(\d+(?:\.\d+)?)\s*([kmg]?)b?$/); + if (!match) return 512 * 1024 * 1024; // default 512MB + + const num = parseFloat(match[1]); + const unit = match[2]; + + switch (unit) { + case 'k': return Math.round(num * 1024); + case 'm': return Math.round(num * 1024 * 1024); + case 'g': return Math.round(num * 1024 * 1024 * 1024); + default: return Math.round(num); + } +} + +/** + * Find a free port on the host for local dev port mapping. + */ +function findFreePort() { + return new Promise((resolve, reject) => { + const srv = net.createServer(); + srv.listen(0, () => { + const port = srv.address().port; + srv.close(() => resolve(port)); + }); + srv.on('error', reject); + }); +} + +export default class DockerBackend extends ContainerBackend { + constructor() { + super(); + this._conn = resolveDockerConnection(); + this._workspaceHost = config.get('workspaceHost') || ''; + this._isRemoteDocker = this._conn.type === 'tcp' || !!this._workspaceHost; + } + + get backendName() { + return 'docker'; + } + + get healthTimeoutMs() { + return 30000; + } + + /** + * Make a Docker Engine API request via Unix socket or TCP. + */ + _api(method, path, body = null) { + return new Promise((resolve, reject) => { + const payload = body ? JSON.stringify(body) : null; + + const reqOptions = { + path, + method, + headers: { + 'Content-Type': 'application/json', + ...(payload ? { 'Content-Length': Buffer.byteLength(payload) } : {}), + }, + timeout: 30000, + }; + + if (this._conn.type === 'tcp') { + reqOptions.hostname = this._conn.hostname; + reqOptions.port = this._conn.port; + } else { + reqOptions.socketPath = this._conn.socketPath; + } + + const req = http.request(reqOptions, (res) => { + let data = ''; + res.on('data', chunk => { data += chunk; }); + res.on('end', () => { + if (res.statusCode >= 400) { + reject(new Error(`Docker API ${method} ${path}: ${res.statusCode} ${data}`)); + return; + } + if (res.statusCode === 204 || !data) { + resolve({}); + return; + } + try { + resolve(JSON.parse(data)); + } catch { + resolve({}); + } + }); + }); + + req.on('error', (e) => reject(new Error(`Docker API ${method} ${path}: ${e.message}`))); + req.on('timeout', () => { req.destroy(); reject(new Error(`Docker API ${method} ${path}: timeout`)); }); + + if (payload) req.write(payload); + req.end(); + }); + } + + async createAndStart({ containerName, image, env, cpus, memoryMB, diskSize, shareName }) { + const network = config.get('workspaceNetwork'); + const volumeName = `${shareName || containerName}-data`; + + const mode = this._isRemoteDocker ? 'remote' : (isInDocker ? 'docker' : 'local'); + logger.info(`[DockerBackend] Provisioning ${containerName} [${mode} mode]`); + + // Remove existing container if it exists (for re-provisioning) + try { + await this._api('DELETE', `/containers/${containerName}?force=true`); + } catch { + // Container doesn't exist, that's fine + } + + // Convert memory/cpu to Docker units + const memoryBytes = memoryMB * 1024 * 1024; + const nanoCpus = Math.round(cpus * 1e9); + + // Determine networking mode + const usePortMapping = mode !== 'docker'; + let hostPort = null; + if (mode === 'local') { + hostPort = await findFreePort(); + } + + const portBindings = usePortMapping + ? { '3100/tcp': [{ HostPort: hostPort ? String(hostPort) : '0' }] } + : undefined; + + // Note: StorageOpt only works on Linux with overlay2/xfs+pquota; skip on macOS/Docker Desktop + const useStorageOpt = (isInDocker || this._isRemoteDocker) && diskSize; + const createBody = { + Image: image, + Hostname: containerName, + Env: env, + ExposedPorts: { '3100/tcp': {} }, + HostConfig: { + NanoCpus: nanoCpus, + Memory: memoryBytes, + ...(useStorageOpt ? { StorageOpt: { size: diskSize } } : {}), + RestartPolicy: { Name: 'unless-stopped' }, + Binds: [`${volumeName}:/workspace`], + ...(portBindings ? { PortBindings: portBindings } : {}), + }, + ...(mode === 'docker' ? { + NetworkingConfig: { + EndpointsConfig: { + [network]: {}, + }, + }, + } : {}), + }; + + const createRes = await this._api('POST', `/containers/create?name=${containerName}`, createBody); + const containerId = createRes.Id; + + // Start container + await this._api('POST', `/containers/${containerId}/start`); + + // Resolve URL to reach this workspace container + let url; + if (mode === 'docker') { + url = `http://${containerName}:3100`; + } else if (mode === 'remote') { + const info = await this._api('GET', `/containers/${containerId}/json`); + const bindings = info.NetworkSettings?.Ports?.['3100/tcp']; + const assignedPort = bindings?.[0]?.HostPort; + if (!assignedPort) { + throw new Error('Docker did not assign a host port for workspace container'); + } + url = `http://${this._workspaceHost}:${assignedPort}`; + } else { + url = `http://localhost:${hostPort}`; + } + + return { containerId, url }; + } + + async start(containerId, containerName) { + await this._api('POST', `/containers/${containerId}/start`); + } + + async stop(containerId, containerName) { + await this._api('POST', `/containers/${containerId}/stop?t=10`); + } + + async remove(containerId, containerName) { + try { + await this._api('POST', `/containers/${containerName}/stop?t=10`); + } catch { + // May already be stopped + } + try { + await this._api('DELETE', `/containers/${containerName}?force=true`); + } catch { + // May already be removed + } + } + + async destroyVolume(shareName) { + const volumeName = `${shareName}-data`; + try { + await this._api('DELETE', `/volumes/${volumeName}`); + } catch { + // Volume may not exist + } + } +} + +// Export helpers for backward compat / unit testing +export { parseMemoryLimit }; diff --git a/pathways/system/entity/tools/shared/backends/index.js b/pathways/system/entity/tools/shared/backends/index.js new file mode 100644 index 00000000..11256592 --- /dev/null +++ b/pathways/system/entity/tools/shared/backends/index.js @@ -0,0 +1,28 @@ +// backends/index.js +// Factory for workspace container backends. +// Singleton — lazily creates the backend on first call. + +import { config } from '../../../../../../config.js'; + +let _backend = null; + +/** + * Get the configured container backend (singleton). + * Uses dynamic import so Azure SDK only loads when ACI backend is selected. + * @returns {Promise} + */ +export async function getBackend() { + if (_backend) return _backend; + + const backendType = config.get('workspaceBackend'); + + if (backendType === 'aci') { + const { default: ACIBackend } = await import('./ACIBackend.js'); + _backend = new ACIBackend(); + } else { + const { default: DockerBackend } = await import('./DockerBackend.js'); + _backend = new DockerBackend(); + } + + return _backend; +} diff --git a/pathways/system/entity/tools/shared/sys_entity_tools.js b/pathways/system/entity/tools/shared/sys_entity_tools.js index b790df0d..a216202a 100644 --- a/pathways/system/entity/tools/shared/sys_entity_tools.js +++ b/pathways/system/entity/tools/shared/sys_entity_tools.js @@ -2,47 +2,73 @@ // Shared tool definitions that can be used by any entity import { config } from '../../../../../config.js'; import logger from '../../../../../lib/logger.js'; +import { getEntityStore } from '../../../../../lib/MongoEntityStore.js'; export const CUSTOM_TOOLS = {}; +const ALWAYS_VISIBLE_LOCAL_TOOL_KEYS = new Set(['workspacessh']); +const WORKSPACE_SSH_TOOL_KEY = 'workspacessh'; + +export const toOpenAiToolDefinition = (tool) => { + const { + icon, + pathwayParams, + silent, + defaultUserMessage, + allowResultCompaction, + ...definitionWithoutExtras + } = tool.definition; + return definitionWithoutExtras; +}; + +const isWorkspaceSshTool = ([toolName, tool]) => { + const functionName = tool?.definition?.function?.name; + return toolName.toLowerCase() === WORKSPACE_SSH_TOOL_KEY || + (typeof functionName === 'string' && functionName.toLowerCase() === WORKSPACE_SSH_TOOL_KEY); +}; + +const removeDefaultEntityTools = (tools, entityConfig) => { + if (!entityConfig?.isDefault) return tools; + return Object.fromEntries( + Object.entries(tools).filter((entry) => !isWorkspaceSshTool(entry)) + ); +}; // Helper function to get tools for a specific entity export const getToolsForEntity = (entityConfig) => { // Get system tools from config const systemTools = config.get('entityTools') || {}; - + // Convert all tool names to lowercase in system tools const normalizedSystemTools = Object.fromEntries( Object.entries(systemTools).map(([key, value]) => [key.toLowerCase(), value]) ); - + // Convert custom tools to lowercase if they exist - const normalizedCustomTools = entityConfig?.customTools ? + const normalizedCustomTools = entityConfig?.customTools ? Object.fromEntries( Object.entries(entityConfig.customTools).map(([key, value]) => [key.toLowerCase(), value]) ) : {}; - + // Convert CUSTOM_TOOLS to lowercase const normalizedCUSTOM_TOOLS = Object.fromEntries( Object.entries(CUSTOM_TOOLS).map(([key, value]) => [key.toLowerCase(), value]) ); - + // Merge system tools with custom tools (custom tools override system tools) const allTools = { ...normalizedSystemTools, ...normalizedCustomTools, ...normalizedCUSTOM_TOOLS }; - + // If no tools property specified or array contains *, return all tools if (!entityConfig?.tools || entityConfig.tools.includes('*')) { + const entityTools = removeDefaultEntityTools(allTools, entityConfig); return { - entityTools: allTools, - entityToolsOpenAiFormat: Object.values(allTools).map(tool => { - const { icon, pathwayParams, ...definitionWithoutExtras } = tool.definition; - return definitionWithoutExtras; - }) + entityTools, + entityToolsOpenAiFormat: Object.values(entityTools).map(toOpenAiToolDefinition) }; } // Get the list of tool names for this entity and convert to lowercase for case-insensitive comparison const entityToolNames = entityConfig.tools.map(name => name.toLowerCase()); - + // Add custom tools to the list of allowed tools if they exist if (entityConfig.customTools) { Object.keys(entityConfig.customTools).forEach(toolName => { @@ -51,61 +77,172 @@ export const getToolsForEntity = (entityConfig) => { } }); } - + // Filter the tools to only include those specified for this entity - const filteredTools = Object.fromEntries( - Object.entries(allTools).filter(([toolName]) => + const filteredTools = removeDefaultEntityTools(Object.fromEntries( + Object.entries(allTools).filter(([toolName]) => entityToolNames.includes(toolName.toLowerCase()) ) - ); + ), entityConfig); return { entityTools: filteredTools, - entityToolsOpenAiFormat: Object.values(filteredTools).map(tool => { - const { icon, pathwayParams, ...definitionWithoutExtras } = tool.definition; - return definitionWithoutExtras; - }) + entityToolsOpenAiFormat: Object.values(filteredTools).map(toOpenAiToolDefinition) }; }; -// Load entity configurations -export const loadEntityConfig = (entityId) => { +export const buildLocalToolCatalog = (entityTools = {}) => { + return Object.fromEntries( + Object.entries(entityTools).map(([toolKey, tool]) => { + const toolFunction = tool.definition?.function || {}; + const parameters = Object.keys(toolFunction.parameters?.properties || {}); + return [toolKey, { + name: toolKey, + displayName: toolFunction.name || toolKey, + originalName: toolFunction.name || toolKey, + server: 'cortex', + description: toolFunction.description || '', + parameters, + source: 'local', + }]; + }) + ); +}; + +export const getAlwaysVisibleLocalToolDefinitions = (entityTools = {}) => { + return Object.entries(entityTools) + .filter(([toolName]) => ALWAYS_VISIBLE_LOCAL_TOOL_KEYS.has(toolName.toLowerCase())) + .map(([, tool]) => toOpenAiToolDefinition(tool)); +}; + +// Check if an entity has all required environment variables set +const hasRequiredEnvVars = (entity) => { + if (!entity.requiredEnvVars || entity.requiredEnvVars.length === 0) { + return true; + } + return entity.requiredEnvVars.every(varName => !!process.env[varName]); +}; + +const buildPersonalEntityDefaults = (userId, defaultEntity = null, personalEntityName = null) => ({ + name: personalEntityName || defaultEntity?.name || 'Jarvis', + tools: defaultEntity?.tools || ['*'], + useMemory: defaultEntity?.useMemory ?? true, + description: defaultEntity?.description || '', + identity: defaultEntity?.identity || '', + avatar: defaultEntity?.avatar || null, + voice: defaultEntity?.voice || null, + resources: defaultEntity?.resources || [], + customTools: defaultEntity?.customTools || {}, + assocUserIds: [userId], + baseModel: defaultEntity?.baseModel || null, + preferredModel: defaultEntity?.preferredModel || null, + modelOverride: defaultEntity?.modelOverride || null, + reasoningEffort: defaultEntity?.reasoningEffort || null, +}); + +/** + * Resolve a stale explicit entityId to a canonical entity for the current user. + * Used by sys_entity_agent to repair replayed entity ids before tools/workspaces run. + * + * @param {string} entityId - Explicit entity UUID from the caller + * @param {Object} [options] + * @param {string} [options.userId] - User context id for personal entity repair + * @param {string} [options.personalEntityName] - Preferred personal entity name + * @returns {Promise<{entityId: string|null, entityConfig: Object|null, repaired: boolean, disabled?: boolean}>} + */ +export const resolveExplicitEntityConfig = async (entityId, options = {}) => { + const { userId = null, personalEntityName = null } = options; + try { - entityId = entityId.toLowerCase(); - - const entityConfig = config.get('entityConfig'); - if (!entityConfig) { - logger.warn('No entity config found in config'); - return null; + const entityStore = getEntityStore(); + if (!entityStore.isConfigured() || !entityId) { + return { entityId, entityConfig: null, repaired: false }; } - // Handle both array and object formats - const configArray = Array.isArray(entityConfig) ? entityConfig : Object.entries(entityConfig).map(([id, config]) => ({ - id, - ...config - })); + const explicitEntity = await entityStore.getEntity(entityId, { fresh: true }); + if (explicitEntity) { + if (!hasRequiredEnvVars(explicitEntity)) { + logger.warn( + `Explicit entityId ${entityId} is disabled - preserving disabled entity failure`, + ); + return { entityId, entityConfig: null, repaired: false, disabled: true }; + } + return { entityId, entityConfig: explicitEntity, repaired: false }; + } - // If entityId is provided, look for that specific entity - if (entityId) { - const entity = configArray.find(e => e.id === entityId); - if (entity) { - return entity; + if (userId) { + const defaultEntity = await entityStore.getDefaultEntity(); + const personalEntity = await entityStore.findOrCreatePersonalEntity( + userId, + buildPersonalEntityDefaults(userId, defaultEntity, personalEntityName), + ); + + if (personalEntity?.id) { + const canonicalEntity = await entityStore.getEntity(personalEntity.id, { + fresh: true, + }); + if (canonicalEntity && hasRequiredEnvVars(canonicalEntity)) { + logger.warn( + `Repairing stale entityId ${entityId} to canonical personal entity ${personalEntity.id} for user ${userId}`, + ); + return { + entityId: personalEntity.id, + entityConfig: canonicalEntity, + repaired: personalEntity.id !== entityId, + }; + } } - logger.warn(`Entity ${entityId} not found in config`); } - // If no entityId or entity not found, look for default entity - const defaultEntity = configArray.find(e => e.isDefault === true); - if (defaultEntity) { - return defaultEntity; + const defaultEntity = await entityStore.getDefaultEntity(); + if (defaultEntity && hasRequiredEnvVars(defaultEntity)) { + logger.warn( + `Falling back from stale entityId ${entityId} to default entity config without binding entityId`, + ); + return { + entityId: '', + entityConfig: defaultEntity, + repaired: true, + }; + } + + return { entityId, entityConfig: null, repaired: false }; + } catch (error) { + logger.error(`Error resolving explicit entity config: ${error.message}`); + return { entityId, entityConfig: null, repaired: false }; + } +}; + +/** + * Load entity configuration from MongoDB (source of truth after boot sync). + * + * @param {string} entityId - Entity UUID or name + * @param {Object} [options] + * @param {boolean} [options.fresh=false] - Bypass local cache and read fresh from MongoDB + * @returns {Promise} Entity config or null + */ +export const loadEntityConfig = async (entityId, options = {}) => { + try { + const entityStore = getEntityStore(); + if (!entityStore.isConfigured()) { + logger.warn('MongoDB not configured — cannot load entity'); + return null; } - // If no default entity found, return the first entity - if (configArray.length > 0) { - return configArray[0]; + if (entityId) { + const entity = await entityStore.getEntity(entityId, options); + if (entity) { + if (!hasRequiredEnvVars(entity)) { + logger.warn(`Entity ${entityId} is disabled - missing required environment variables`); + return null; + } + return entity; + } } - return null; + // No specific entity requested (or not found) — return the default + const defaultEntity = await entityStore.getDefaultEntity(); + return defaultEntity || null; } catch (error) { logger.error(`Error loading entity config: ${error.message}`); return null; @@ -113,36 +250,40 @@ export const loadEntityConfig = (entityId) => { }; /** - * Fetches the list of available entities with their descriptions and active tools - * @returns {Array} Array of objects containing entity information and their active tools + * Fetches the list of available entities with their descriptions and active tools. + * Reads from MongoDB only (source of truth after boot sync). + * + * @param {Object} [options] + * @param {string} [options.userId] - Filter to entities for this user + * @param {boolean} [options.fresh=false] - Bypass local cache and read fresh from MongoDB + * @returns {Promise} Array of objects containing entity information and their active tools */ -export const getAvailableEntities = () => { +export const getAvailableEntities = async (options = {}) => { try { - const entityConfig = config.get('entityConfig'); - if (!entityConfig) { - logger.warn('No entity config found in config'); + const entityStore = getEntityStore(); + if (!entityStore.isConfigured()) { + logger.warn('MongoDB not configured — cannot list entities'); return []; } - // Handle both array and object formats - const configArray = Array.isArray(entityConfig) ? entityConfig : Object.entries(entityConfig).map(([id, config]) => ({ - id, - ...config - })); - - return configArray.map(entity => { - const { entityTools } = getToolsForEntity(entity); - return { - id: entity.id, - name: entity.name || entity.id, - description: entity.description || '', - isDefault: entity.isDefault || false, - activeTools: Object.keys(entityTools).map(toolName => ({ - name: toolName, - description: entityTools[toolName].definition?.function?.description || '' - })) - }; - }); + const mongoEntities = await entityStore.getAllEntities(options); + return mongoEntities + .filter(entity => hasRequiredEnvVars(entity)) + .map(entity => { + const { entityTools } = getToolsForEntity(entity); + return { + id: entity.id, + name: entity.name || entity.id, + description: entity.description || '', + isDefault: entity.isDefault || false, + activeTools: Object.keys(entityTools).map(toolName => ({ + name: toolName, + description: entityTools[toolName].definition?.function?.description || '' + })), + secretKeys: entity.secrets ? Object.keys(entity.secrets) : [], + reasoningEffort: entity.reasoningEffort || null, + }; + }); } catch (error) { logger.error(`Error fetching available entities: ${error.message}`); return []; diff --git a/pathways/system/entity/tools/shared/warmPool.js b/pathways/system/entity/tools/shared/warmPool.js new file mode 100644 index 00000000..18b579c4 --- /dev/null +++ b/pathways/system/entity/tools/shared/warmPool.js @@ -0,0 +1,668 @@ +// warmPool.js +// Pre-provisions ACI containers so the first workspace request is instant (~1-2s +// reconfigure call instead of 90-120s cold provision). ACI-only. +// +// Security model: "born clean, die on release" +// - Pool containers start fresh — no entity data, no secrets, just a bootstrap secret +// - When claimed: /reconfigure rotates the secret, injects env vars, mounts blob storage +// - When entity is done: container is destroyed (never recycled entity-to-entity) +// +// Pool state is backed by Redis so it survives restarts and is shared across hosts. + +import crypto from 'node:crypto'; +import os from 'node:os'; +import logger from '../../../../../lib/logger.js'; +import { config } from '../../../../../config.js'; +import { parseMemoryToMB, resolveWorkspaceImage } from './workspace_client.js'; + +const REPLENISH_INTERVAL_MS = 60_000; +const REPLENISH_LOCK_TTL_MS = 6 * 60 * 1000; // must exceed STUCK_PROVISION_TIMEOUT_MS +const STUCK_PROVISION_TIMEOUT_MS = 5 * 60 * 1000; + +let _replenishTimer = null; +let _hostId = null; + +// Redis key helpers +function keyPrefix() { + return `${config.get('cortexId')}-warmpool`; +} +function containersKey() { return `${keyPrefix()}:containers`; } +function readyKey() { return `${keyPrefix()}:ready`; } +function replenishLockKey() { return `${keyPrefix()}:replenish-lock`; } +function workspaceContainerPrefix() { return config.get('workspaceContainerPrefix') || 'workspace-local'; } + +function isPoolEntryActive(entry) { + return entry?.status === 'READY' || entry?.status === 'PROVISIONING'; +} + +function isStuckProvisioning(entry, now = Date.now()) { + if (entry?.status !== 'PROVISIONING') return false; + const createdAt = new Date(entry.createdAt).getTime(); + return Number.isFinite(createdAt) && now - createdAt > STUCK_PROVISION_TIMEOUT_MS; +} + +// --------------------------------------------------------------------------- +// Lazy singleton Redis client (follows pattern from lib/fileUtils.js) +// --------------------------------------------------------------------------- +let _redisClient = null; + +async function getRedisClient() { + if (_redisClient) return _redisClient; + + try { + const connectionString = config.get('storageConnectionString'); + if (!connectionString) return null; + + const Redis = (await import('ioredis')).default; + _redisClient = new Redis(connectionString, { + maxRetriesPerRequest: null, + enableReadyCheck: true, + lazyConnect: false, + connectTimeout: 10000, + }); + + _redisClient.on('error', (error) => { + logger.error(`[WarmPool] Redis client error: ${error.message}`); + }); + + return _redisClient; + } catch (e) { + logger.error(`[WarmPool] Failed to create Redis client: ${e.message}`); + return null; + } +} + +// --------------------------------------------------------------------------- +// Public API +// --------------------------------------------------------------------------- + +/** + * Initialize the warm pool. Called once at startup. + * Discovers existing pool containers from Redis, then kicks off replenishment. + */ +export async function initWarmPool() { + const poolSize = config.get('warmPoolSize'); + + if (!poolSize || poolSize <= 0) { + logger.info('[WarmPool] Disabled (WARM_POOL_SIZE=0 or unset)'); + return; + } + + if (config.get('workspaceBackend') !== 'aci') { + logger.info('[WarmPool] Disabled (only supported on ACI backend)'); + return; + } + + if (!config.get('warmPoolEnabled')) { + logger.info('[WarmPool] Disabled (WARM_POOL_ENABLED=false or unset)'); + return; + } + + const redis = await getRedisClient(); + if (!redis) { + logger.warn('[WarmPool] Disabled — Redis not available'); + return; + } + + _hostId = `${config.get('cortexId')}-${os.hostname()}-${process.pid}`; + logger.info(`[WarmPool] Initializing with target size ${poolSize}, hostId=${_hostId}`); + + // Discover existing containers from Redis and health-check them + await discoverExistingContainers(redis).catch(e => + logger.error(`[WarmPool] Discovery failed: ${e.message}`) + ); + + // Initial replenishment (background — don't block startup) + replenish(redis).catch(e => logger.error(`[WarmPool] Initial replenish failed: ${e.message}`)); + + // Periodic replenishment + _replenishTimer = setInterval(() => { + replenish(redis).catch(e => logger.error(`[WarmPool] Periodic replenish failed: ${e.message}`)); + }, REPLENISH_INTERVAL_MS); + + if (_replenishTimer.unref) _replenishTimer.unref(); +} + +/** + * Claim a READY container from the pool. + * Uses atomic SPOP on the Redis ready set. + * Returns container info and removes it from the registry. + * Triggers background replenishment. + * + * @returns {Promise<{ success: boolean, containerName?: string, url?: string, bootstrapSecret?: string, containerId?: string }>} + */ +export async function claimContainer(entityId) { + const redis = await getRedisClient(); + if (!redis) return { success: false }; + + try { + let containerName = null; + let entry = null; + + while (true) { + // Atomic pop of one random member from the ready set + containerName = await redis.spop(readyKey()); + if (!containerName) return { success: false }; + + // Get container details from the registry hash + const raw = await redis.hget(containersKey(), containerName); + if (!raw) { + // Race condition — another host already cleaned it up + continue; + } + + entry = JSON.parse(raw); + const validation = await validatePoolContainerInAci(containerName, entry); + if (validation.valid !== false) break; + + await redis.hdel(containersKey(), containerName); + logger.warn(`[WarmPool] Removed stale READY entry ${containerName}; ${validation.reason}`); + if (validation.removeContainer) { + const { default: ACIBackend } = await import('./backends/ACIBackend.js'); + const backend = new ACIBackend(); + backend.remove(entry.containerId || containerName, containerName).catch(() => {}); + } + } + + // Remove from registry + await redis.hdel(containersKey(), containerName); + + logger.info(`[WarmPool] Claimed container ${containerName}`); + + // Tag the Azure resource with the claiming entityId so the orphan + // reconciler can distinguish claimed pool containers from true orphans + // without a cross-database Mongo lookup. Best-effort — never fail the + // claim because tagging fell over. + if (entityId) { + try { + const { default: ACIBackend } = await import('./backends/ACIBackend.js'); + const backend = new ACIBackend(); + await backend.setEntityTag(containerName, entityId); + } catch (e) { + logger.warn(`[WarmPool] Failed to tag ${containerName} with entityId=${entityId}: ${e.message}`); + } + } + + // Trigger background replenishment + replenish(redis).catch(e => logger.error(`[WarmPool] Post-claim replenish failed: ${e.message}`)); + + return { + success: true, + containerName, + url: entry.url, + bootstrapSecret: entry.bootstrapSecret, + containerId: entry.containerId, + imageVersion: entry.imageVersion || null, + }; + } catch (e) { + logger.error(`[WarmPool] Claim failed: ${e.message}`); + return { success: false }; + } +} + +/** + * Get pool status for monitoring. + */ +export async function getPoolStatus() { + const redis = await getRedisClient(); + if (!redis) { + return { ready: 0, provisioning: 0, total: 0, targetSize: config.get('warmPoolSize'), entries: [] }; + } + + try { + const all = await redis.hgetall(containersKey()); + const entries = []; + let ready = 0; + let provisioning = 0; + + for (const [name, raw] of Object.entries(all)) { + const entry = JSON.parse(raw); + if (entry.status === 'READY') ready++; + if (entry.status === 'PROVISIONING') provisioning++; + entries.push({ name, status: entry.status, createdAt: entry.createdAt }); + } + + return { ready, provisioning, total: entries.length, targetSize: config.get('warmPoolSize'), entries }; + } catch (e) { + logger.error(`[WarmPool] Failed to get pool status: ${e.message}`); + return { ready: 0, provisioning: 0, total: 0, targetSize: config.get('warmPoolSize'), entries: [] }; + } +} + +export async function getWarmPoolActiveContainerNames(redisClient = null) { + const redis = redisClient || await getRedisClient(); + if (!redis) return new Set(); + + try { + const all = await redis.hgetall(containersKey()); + const active = new Set(); + for (const [name, raw] of Object.entries(all)) { + const entry = JSON.parse(raw); + if (entry.status === 'READY' || entry.status === 'PROVISIONING') { + active.add(name); + } + } + return active; + } catch (e) { + logger.warn(`[WarmPool] Failed to read active pool containers: ${e.message}`); + return new Set(); + } +} + +export async function removeWarmPoolEntry(containerName, redisClient = null) { + if (!containerName) return; + const redis = redisClient || await getRedisClient(); + if (!redis) return; + + await redis.hdel(containersKey(), containerName); + await redis.srem(readyKey(), containerName); +} + +/** + * Graceful shutdown: do NOT destroy pool containers. + * They persist in Redis for other hosts / next restart to discover. + * Only clear the replenish timer. + */ +export async function drainPool() { + if (_replenishTimer) { + clearInterval(_replenishTimer); + _replenishTimer = null; + } + + logger.info('[WarmPool] Shutdown — pool containers preserved in Redis for next startup'); +} + +// --------------------------------------------------------------------------- +// Startup Discovery +// --------------------------------------------------------------------------- + +/** + * Discover existing pool containers from Redis on startup. + * Health-check READY entries, clean up stale/failed ones. + */ +async function discoverExistingContainers(redis) { + const all = await redis.hgetall(containersKey()); + const containerNames = Object.keys(all); + + if (containerNames.length === 0) { + logger.info('[WarmPool] No existing pool containers in Redis'); + return; + } + + logger.info(`[WarmPool] Discovered ${containerNames.length} pool container(s) in Redis`); + + const { default: ACIBackend } = await import('./backends/ACIBackend.js'); + const backend = new ACIBackend(); + const inventory = await getPoolContainerInventory(backend); + + for (const [containerName, raw] of Object.entries(all)) { + const entry = JSON.parse(raw); + + if (entry.status === 'READY') { + const inventoryResult = validatePoolContainerInventoryEntry(containerName, inventory, entry); + if (inventoryResult.valid === false) { + const removed = await redis.hdel(containersKey(), containerName); + await redis.srem(readyKey(), containerName); + logger.warn(`[WarmPool] Removed stale ${entry.status} entry ${containerName}; ${inventoryResult.reason}`); + if (removed && inventoryResult.removeContainer) { + backend.remove(entry.containerId || containerName, containerName).catch(() => {}); + } + continue; + } + } + + if (entry.status === 'FAILED') { + // Clean up failed entries + await redis.hdel(containersKey(), containerName); + await redis.srem(readyKey(), containerName); + logger.info(`[WarmPool] Removed FAILED entry ${containerName}`); + continue; + } + + if (entry.status === 'PROVISIONING') { + const age = Date.now() - new Date(entry.createdAt).getTime(); + if (age > STUCK_PROVISION_TIMEOUT_MS) { + // Stuck provisioning — clean up + logger.warn(`[WarmPool] Removing stuck PROVISIONING container ${containerName} (age=${Math.round(age / 1000)}s)`); + const removed = await redis.hdel(containersKey(), containerName); + await redis.srem(readyKey(), containerName); + // Only destroy if we actually owned the removal (hdel returned 1). + // If 0, claim() already took it — don't destroy a claimed container's share. + if (removed) { + backend.remove(entry.containerId || containerName, containerName).catch(() => {}); + } + continue; + } + // Still provisioning and not stuck — leave it + continue; + } + + if (entry.status === 'READY') { + // Health-check the container + const healthy = await checkHealth(entry.url); + if (healthy) { + // Ensure it's in the ready set + await redis.sadd(readyKey(), containerName); + logger.info(`[WarmPool] Existing container ${containerName} is healthy`); + } else { + // Dead — remove from Redis and destroy + logger.warn(`[WarmPool] Container ${containerName} failed health check — removing`); + const removed = await redis.hdel(containersKey(), containerName); + await redis.srem(readyKey(), containerName); + if (removed) { + backend.remove(entry.containerId || containerName, containerName).catch(() => {}); + } + } + } + } +} + +// --------------------------------------------------------------------------- +// Replenishment with distributed lock +// --------------------------------------------------------------------------- + +/** + * Replenish the pool to the target size. + * Uses a Redis lock to ensure only one host replenishes at a time. + */ +async function replenish(redis) { + const targetSize = config.get('warmPoolSize'); + if (!targetSize || targetSize <= 0) return; + + // Count active entries (READY + PROVISIONING) + const all = await redis.hgetall(containersKey()); + let active = 0; + let hasStuckProvisioning = false; + for (const raw of Object.values(all)) { + const entry = JSON.parse(raw); + if (isPoolEntryActive(entry)) active++; + if (isStuckProvisioning(entry)) hasStuckProvisioning = true; + } + + const deficit = targetSize - active; + if (deficit <= 0 && !hasStuckProvisioning) return; + + // Acquire distributed lock + const lockAcquired = await redis.set( + replenishLockKey(), + _hostId, + 'NX', + 'PX', + REPLENISH_LOCK_TTL_MS, + ); + + if (!lockAcquired) { + logger.debug('[WarmPool] Another host is replenishing — skipping'); + return; + } + + try { + await pruneStalePoolEntries(redis); + + // Double-check after acquiring lock + const allAfterLock = await redis.hgetall(containersKey()); + let activeAfterLock = 0; + for (const raw of Object.values(allAfterLock)) { + const entry = JSON.parse(raw); + if (isPoolEntryActive(entry)) activeAfterLock++; + } + + const confirmedDeficit = targetSize - activeAfterLock; + if (confirmedDeficit <= 0) return; + + // Clean up any FAILED entries + for (const [name, raw] of Object.entries(allAfterLock)) { + const entry = JSON.parse(raw); + if (entry.status === 'FAILED') { + await redis.hdel(containersKey(), name); + await redis.srem(readyKey(), name); + } + } + + logger.info(`[WarmPool] Replenishing ${confirmedDeficit} container(s) (active=${activeAfterLock}, target=${targetSize})`); + + const provisions = []; + for (let i = 0; i < confirmedDeficit; i++) { + provisions.push(provisionPoolContainer(redis)); + } + + await Promise.allSettled(provisions); + } finally { + // Release lock only if we still own it + try { + const currentOwner = await redis.get(replenishLockKey()); + if (currentOwner === _hostId) { + await redis.del(replenishLockKey()); + } + } catch { + // Best-effort lock release + } + } +} + +async function getPoolContainerInventory(backend = null) { + try { + let resolvedBackend = backend; + if (!resolvedBackend) { + const { default: ACIBackend } = await import('./backends/ACIBackend.js'); + resolvedBackend = new ACIBackend(); + } + if (typeof resolvedBackend.listWorkspaceContainers !== 'function') { + return { available: false, containersByName: new Map(), error: 'ACI inventory unavailable' }; + } + + const containers = await resolvedBackend.listWorkspaceContainers(); + return { + available: true, + containersByName: new Map(containers + .filter(container => container.name) + .map(container => [container.name, container])), + error: null, + }; + } catch (e) { + logger.warn(`[WarmPool] Skipping ACI pool inventory validation: ${e.message}`); + return { available: false, containersByName: new Map(), error: e.message }; + } +} + +function validatePoolContainerInventoryEntry(containerName, inventory, entry = null) { + if (!inventory.available) return { valid: null, reason: inventory.error || 'ACI inventory unavailable' }; + const container = inventory.containersByName.get(containerName); + if (!container) { + return { valid: false, reason: 'not present in ACI inventory for current workspace prefix' }; + } + if (container.tags?.workspaceRole !== 'pool') { + return { valid: false, reason: `ACI inventory role is ${container.tags?.workspaceRole || 'unset'}, not pool` }; + } + const expectedVersion = config.get('workspaceImageVersion'); + const actualVersion = container.tags?.imageVersion || entry?.imageVersion || null; + if (expectedVersion && actualVersion && actualVersion !== expectedVersion) { + return { + valid: false, + reason: `stale image (${actualVersion} vs ${expectedVersion})`, + removeContainer: true, + }; + } + return { valid: true, reason: null }; +} + +async function validatePoolContainerInAci(containerName, entry = null) { + const inventory = await getPoolContainerInventory(); + return validatePoolContainerInventoryEntry(containerName, inventory, entry); +} + +async function pruneStalePoolEntries(redis) { + const all = await redis.hgetall(containersKey()); + if (Object.keys(all).length === 0) return 0; + + const inventory = await getPoolContainerInventory(); + let pruned = 0; + for (const [containerName, raw] of Object.entries(all)) { + const entry = JSON.parse(raw); + + if (isStuckProvisioning(entry)) { + logger.warn(`[WarmPool] Removing stuck PROVISIONING container ${containerName} (age=${Math.round((Date.now() - new Date(entry.createdAt).getTime()) / 1000)}s)`); + const removed = await redis.hdel(containersKey(), containerName); + await redis.srem(readyKey(), containerName); + if (removed) { + pruned += 1; + const { default: ACIBackend } = await import('./backends/ACIBackend.js'); + const backend = new ACIBackend(); + backend.remove(entry.containerId || containerName, containerName).catch(() => {}); + } + continue; + } + + if (entry.status !== 'READY') continue; + if (!inventory.available) continue; + + const result = validatePoolContainerInventoryEntry(containerName, inventory, entry); + if (result.valid !== false) continue; + + await redis.hdel(containersKey(), containerName); + await redis.srem(readyKey(), containerName); + pruned += 1; + logger.warn(`[WarmPool] Removed stale ${entry.status} entry ${containerName}; ${result.reason}`); + } + return pruned; +} + +// --------------------------------------------------------------------------- +// Pool container provisioning +// --------------------------------------------------------------------------- + +/** + * Provision a single pool container. + * Container naming: {WORKSPACE_CONTAINER_PREFIX}-pool-{12-char-uuid} + */ +async function provisionPoolContainer(redis) { + const bootstrapSecret = crypto.randomBytes(32).toString('hex'); + const shortId = crypto.randomUUID().replace(/-/g, '').slice(0, 12); + const containerName = `${workspaceContainerPrefix()}-pool-${shortId}`; + + const entry = { + url: null, + bootstrapSecret, + status: 'PROVISIONING', + containerId: null, + createdAt: new Date().toISOString(), + hostId: _hostId, + imageVersion: config.get('workspaceImageVersion') || null, + }; + + // Register in Redis as PROVISIONING + await redis.hset(containersKey(), containerName, JSON.stringify(entry)); + + try { + const { default: ACIBackend } = await import('./backends/ACIBackend.js'); + const backend = new ACIBackend(); + + const image = resolveWorkspaceImage(); + const cpus = parseFloat(config.get('workspaceCpus')); + const memory = config.get('workspaceMemory'); + const memoryMB = parseMemoryToMB(memory); + + const env = [ + `WORKSPACE_SECRET=${bootstrapSecret}`, + `PORT=3100`, + ]; + + logger.info(`[WarmPool] Provisioning pool container ${containerName}`); + + const { containerId, url } = await backend.createAndStart({ + containerName, + image, + env, + cpus, + memoryMB, + diskSize: config.get('workspaceDiskSize'), + tags: { + workspaceRole: 'pool', + createdAt: entry.createdAt, + }, + }); + + // Wait for health + const healthOk = await waitForHealth(url, backend.healthTimeoutMs); + + if (!healthOk) { + throw new Error('Pool container failed to become healthy'); + } + + // Update entry to READY in Redis + entry.url = url; + entry.containerId = containerId; + entry.status = 'READY'; + await redis.hset(containersKey(), containerName, JSON.stringify(entry)); + await redis.sadd(readyKey(), containerName); + + logger.info(`[WarmPool] Pool container ${containerName} ready at ${url}`); + } catch (e) { + logger.error(`[WarmPool] Failed to provision pool container ${containerName}: ${e.message}`); + + // Mark as FAILED in Redis + entry.status = 'FAILED'; + await redis.hset(containersKey(), containerName, JSON.stringify(entry)); + await redis.srem(readyKey(), containerName); + + // Best-effort cleanup of the failed pool container. + try { + const { default: ACIBackend } = await import('./backends/ACIBackend.js'); + const backend = new ACIBackend(); + await backend.remove(containerName, containerName); + } catch { + // Ignore cleanup errors + } + } +} + +// Test-only exports for targeted unit coverage of provisioning semantics. +export const __testables = { + getWarmPoolActiveContainerNames, + pruneStalePoolEntries, + provisionPoolContainer, + replenish, + validatePoolContainerInventoryEntry, +}; + +// --------------------------------------------------------------------------- +// Health checking +// --------------------------------------------------------------------------- + +/** + * Quick health check — single attempt with short timeout. + * Used by discovery to validate existing containers. + */ +async function checkHealth(baseUrl) { + try { + const res = await fetch(`${baseUrl}/health`, { + signal: AbortSignal.timeout(5000), + }); + return res.ok; + } catch { + return false; + } +} + +/** + * Poll a container's /health endpoint until it responds OK. + * Used during provisioning where we need to wait for startup. + */ +async function waitForHealth(baseUrl, maxWaitMs) { + const start = Date.now(); + const interval = 1000; + + while (Date.now() - start < maxWaitMs) { + try { + const res = await fetch(`${baseUrl}/health`, { + signal: AbortSignal.timeout(3000), + }); + if (res.ok) return true; + } catch { + // Not ready yet + } + await new Promise(r => setTimeout(r, interval)); + } + + return false; +} diff --git a/pathways/system/entity/tools/shared/workspace_client.js b/pathways/system/entity/tools/shared/workspace_client.js new file mode 100644 index 00000000..598cc4ce --- /dev/null +++ b/pathways/system/entity/tools/shared/workspace_client.js @@ -0,0 +1,3919 @@ +// workspace_client.js +// Shared module for workspace tools: HTTP client, auto-provisioning, backend abstraction. +import crypto from 'node:crypto'; +import fs from 'node:fs'; +import { Readable } from 'node:stream'; +import { pipeline } from 'node:stream/promises'; +import { Agent } from 'undici'; +import logger from '../../../../../lib/logger.js'; +import { config } from '../../../../../config.js'; +import { decrypt, encrypt } from '../../../../../lib/crypto.js'; +import { loadEntityConfig } from './sys_entity_tools.js'; +import { getEntityStore } from '../../../../../lib/MongoEntityStore.js'; +import { getBackend } from './backends/index.js'; +import { getUserContainerName, ensureContainer, generateContainerSASToken } from '../../../../../lib/blobContainerUtils.js'; +import { initWarmPool, claimContainer, getWarmPoolActiveContainerNames, removeWarmPoolEntry } from './warmPool.js'; + +/** + * Resolve the full workspace image reference (name:tag). + * Combines workspaceImage + workspaceImageVersion so ACI/Docker always + * pulls an exact version — never a cached `:latest`. + */ +export function resolveWorkspaceImage() { + const base = config.get('workspaceImage'); + const version = config.get('workspaceImageVersion'); + // If the image already has a tag (e.g. from WORKSPACE_IMAGE env), use it as-is + if (base.includes(':')) return base; + // If no version configured, fall back to :latest (local dev) + if (!version) return `${base}:latest`; + return `${base}:${version}`; +} + +// In-memory lock to prevent concurrent provisioning for the same entity +const provisioningLocks = new Map(); +const reprovisionLocks = new Map(); +const WORKSPACE_TRANSITION_STATUSES = new Set(['starting', 'provisioning']); +const WORKSPACE_TRANSITION_WAIT_MS = 90_000; +const WORKSPACE_TRANSITION_POLL_MS = 2_000; +const WORKSPACE_PROVISIONING_LOCK_TTL_MS = 5 * 60 * 1000; +const WORKSPACE_CHECKPOINT_PATH = '/persist/workspace.tar.gz'; +const WORKSPACE_CHECKPOINT_ENCRYPTION_ALGORITHM = 'aes-256-gcm'; + +// Local activity mirror: entityId → timestamp (ms). Redis is the durable source +// of reaper candidates; this map only helps the current process notice fresher +// activity while it is evaluating a candidate. +const lastActivity = new Map(); +const longFetchDispatchers = new Map(); + +let _activityRedisClient = null; +let _activityRedisClientConnectPromise = null; +let _activityRedisClientOverride; +let _workspaceCheckpointUploadOverride; +let _workspaceCheckpointContainerClientOverride; +let _workspaceLegacyShareUploadOverride; +const ACTIVITY_REAPER_HOST_ID = `${process.pid}-${crypto.randomUUID()}`; + +function isValidWorkspaceEntityId(entityId) { + return typeof entityId === 'string' && entityId.trim().length > 0; +} + +function invalidWorkspaceEntityResult() { + return { success: false, error: 'Workspace entityId is required' }; +} + +function getLongFetchDispatcher(timeoutMs) { + const boundedTimeoutMs = Math.max(1, Number(timeoutMs) || 30_000); + if (!longFetchDispatchers.has(boundedTimeoutMs)) { + longFetchDispatchers.set(boundedTimeoutMs, new Agent({ + headersTimeout: boundedTimeoutMs, + bodyTimeout: boundedTimeoutMs, + })); + } + return longFetchDispatchers.get(boundedTimeoutMs); +} + +function workspaceActivityKey(entityId) { + return `${config.get('cortexId')}-workspace:activity:${entityId}`; +} + +function workspaceActivityIndexKey() { + return `${config.get('cortexId')}-workspace:activity-index`; +} + +function workspaceReaperLockKey(entityId) { + return `${config.get('cortexId')}-workspace:reaper-lock:${entityId}`; +} + +function workspaceProvisioningLockKey(entityId) { + return `${config.get('cortexId')}-workspace:provisioning-lock:${entityId}`; +} + +function workspaceActivityTtlMs(idleTimeoutMs = config.get('workspaceIdleTimeoutMs')) { + return Math.max((Number(idleTimeoutMs) || 0) * 2, 60 * 60 * 1000); +} + +function sleep(ms) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +function workspaceContainerPrefix() { + return config.get('workspaceContainerPrefix') || 'workspace-local'; +} + +function workspaceContainerNameForEntity(entityId) { + return `${workspaceContainerPrefix()}-${entityId}`; +} + +function sanitizeCheckpointPathPart(value) { + return String(value || 'default') + .toLowerCase() + .replace(/[^a-z0-9-]/g, '-') + .replace(/-+/g, '-') + .replace(/^-+|-+$/g, '') + .slice(0, 80) || 'default'; +} + +function workspaceCheckpointIdentityHash(value) { + return crypto + .createHash('sha256') + .update(String(value || '')) + .digest('hex'); +} + +function workspaceCheckpointPathPart(value) { + const hint = sanitizeCheckpointPathPart(value).slice(0, 40); + const hash = workspaceCheckpointIdentityHash(value); + return hint && hint !== 'default' ? `${hint}-${hash}` : hash; +} + +function workspaceCheckpointBlobPath(entityId, filename = 'workspace.tar.gz') { + const cortexId = workspaceCheckpointPathPart(config.get('cortexId')); + const entityKey = workspaceCheckpointPathPart(entityId); + return `workspace-checkpoints/${cortexId}/${entityKey}/${filename}`; +} + +function legacyWorkspaceCheckpointBlobPath(entityId, filename = 'workspace.tar.gz') { + const cortexId = sanitizeCheckpointPathPart(config.get('cortexId')); + const safeEntityId = sanitizeCheckpointPathPart(entityId); + return `workspace-checkpoints/${cortexId}/${safeEntityId}/${filename}`; +} + +function workspaceCheckpointBlobPathCandidates(entityId, filename = 'workspace.tar.gz') { + return [ + workspaceCheckpointBlobPath(entityId, filename), + legacyWorkspaceCheckpointBlobPath(entityId, filename), + ].filter((blobPath, index, paths) => paths.indexOf(blobPath) === index); +} + +function workspaceCheckpointBlobMetadata(entityId, extra = {}) { + const cortexId = String(config.get('cortexId') || ''); + return { + entityId: String(entityId || ''), + entityHash: workspaceCheckpointIdentityHash(entityId), + cortexId, + cortexHash: workspaceCheckpointIdentityHash(cortexId), + ...extra, + }; +} + +function workspaceCheckpointEncryptionKeyId(keyBase64) { + return crypto + .createHash('sha256') + .update(String(keyBase64 || '')) + .digest('hex') + .slice(0, 32); +} + +function encryptWorkspaceCheckpointKey(keyBase64) { + const systemKey = config.get('redisEncryptionKey'); + return encrypt(keyBase64, systemKey); +} + +function decryptWorkspaceCheckpointKey(encryptedKey) { + const systemKey = config.get('redisEncryptionKey'); + return decrypt(encryptedKey, systemKey); +} + +function readWorkspaceCheckpointEncryptionKey(entityConfig) { + const existing = entityConfig?.workspace?.checkpointEncryptionKey; + if (!existing?.encryptedKey) return null; + const keyBase64 = decryptWorkspaceCheckpointKey(existing.encryptedKey); + if (!keyBase64) { + throw new Error('Workspace checkpoint encryption key could not be decrypted'); + } + return { + algorithm: existing.algorithm || WORKSPACE_CHECKPOINT_ENCRYPTION_ALGORITHM, + keyBase64, + keyId: existing.keyId || workspaceCheckpointEncryptionKeyId(keyBase64), + entityConfig, + }; +} + +async function getOrCreateWorkspaceCheckpointEncryptionKey(entityId, entityConfig) { + const existing = readWorkspaceCheckpointEncryptionKey(entityConfig); + if (existing) return existing; + + const current = (await loadEntityConfig(entityId, { fresh: true })) || entityConfig; + const currentKey = readWorkspaceCheckpointEncryptionKey(current); + if (currentKey) return currentKey; + + const keyBase64 = crypto.randomBytes(32).toString('base64'); + const keyRecord = { + algorithm: WORKSPACE_CHECKPOINT_ENCRYPTION_ALGORITHM, + keyId: workspaceCheckpointEncryptionKeyId(keyBase64), + encryptedKey: encryptWorkspaceCheckpointKey(keyBase64), + createdAt: new Date().toISOString(), + }; + const updated = { + ...current, + workspace: { + ...(current?.workspace || {}), + checkpointEncryptionKey: keyRecord, + }, + }; + await getEntityStore().upsertEntity(updated); + return { + algorithm: keyRecord.algorithm, + keyBase64, + keyId: keyRecord.keyId, + entityConfig: updated, + }; +} + +function buildWorkspaceCheckpointRestoreEncryption(entityConfig) { + const checkpointEncryption = entityConfig?.workspace?.checkpointEncryption; + if (!checkpointEncryption) return null; + const keyRecord = entityConfig?.workspace?.checkpointEncryptionKey; + if (!keyRecord?.encryptedKey) { + throw new Error('Workspace checkpoint is encrypted but no checkpoint encryption key is stored'); + } + const keyBase64 = decryptWorkspaceCheckpointKey(keyRecord.encryptedKey); + if (!keyBase64) { + throw new Error('Workspace checkpoint encryption key could not be decrypted'); + } + return { + algorithm: checkpointEncryption.algorithm || keyRecord.algorithm || WORKSPACE_CHECKPOINT_ENCRYPTION_ALGORITHM, + keyBase64, + keyId: checkpointEncryption.keyId || keyRecord.keyId || workspaceCheckpointEncryptionKeyId(keyBase64), + ivBase64: checkpointEncryption.ivBase64, + tagBase64: checkpointEncryption.tagBase64, + compression: checkpointEncryption.compression || entityConfig?.workspace?.checkpointCompression || 'gzip', + }; +} + +function checkpointEncryptionMetadata(encryption = {}) { + if (!encryption?.algorithm || !encryption?.ivBase64 || !encryption?.tagBase64) return {}; + const metadata = { + checkpointEncryptionAlgorithm: encryption.algorithm, + checkpointEncryptionKeyId: encryption.keyId || '', + checkpointEncryptionIv: encryption.ivBase64, + checkpointEncryptionTag: encryption.tagBase64, + }; + if (encryption.compression) { + metadata.checkpointCompression = encryption.compression; + } + return metadata; +} + +function checkpointEncryptionFromMetadata(metadata = {}) { + const algorithm = metadataValue(metadata, 'checkpointEncryptionAlgorithm'); + const ivBase64 = metadataValue(metadata, 'checkpointEncryptionIv'); + const tagBase64 = metadataValue(metadata, 'checkpointEncryptionTag'); + if (!algorithm || !ivBase64 || !tagBase64) return null; + return { + algorithm, + keyId: metadataValue(metadata, 'checkpointEncryptionKeyId') || null, + ivBase64, + tagBase64, + compression: metadataValue(metadata, 'checkpointCompression') || 'gzip', + }; +} + +function metadataValue(metadata, key) { + const lowerKey = key.toLowerCase(); + for (const [candidateKey, value] of Object.entries(metadata || {})) { + if (candidateKey.toLowerCase() === lowerKey) { + return value == null ? '' : String(value); + } + } + return null; +} + +function validateWorkspaceCheckpointMetadata(metadata, entityId) { + const expected = workspaceCheckpointBlobMetadata(entityId); + const entityIdValue = metadataValue(metadata, 'entityId'); + const entityHashValue = metadataValue(metadata, 'entityHash'); + const cortexIdValue = metadataValue(metadata, 'cortexId'); + const cortexHashValue = metadataValue(metadata, 'cortexHash'); + + if (!entityIdValue && !entityHashValue) { + throw new Error('Workspace checkpoint is missing entity identity metadata'); + } + if (entityIdValue && entityIdValue !== expected.entityId) { + throw new Error('Workspace checkpoint entity metadata does not match requested entity'); + } + if (entityHashValue && entityHashValue !== expected.entityHash) { + throw new Error('Workspace checkpoint entity hash does not match requested entity'); + } + if (cortexIdValue && cortexIdValue !== expected.cortexId) { + throw new Error('Workspace checkpoint cortex metadata does not match this Cortex instance'); + } + if (cortexHashValue && cortexHashValue !== expected.cortexHash) { + throw new Error('Workspace checkpoint cortex hash does not match this Cortex instance'); + } + + return true; +} + +function shellQuote(value) { + return `'${String(value).replace(/'/g, `'\\''`)}'`; +} + +function getWorkspaceCheckpointMetadata(workspace = {}) { + if (!workspace?.checkpointBlobPath) return {}; + const metadata = { + checkpointBlobPath: workspace.checkpointBlobPath, + checkpointPreviousBlobPath: workspace.checkpointPreviousBlobPath || null, + checkpointSizeBytes: workspace.checkpointSizeBytes || null, + checkpointSizeMB: workspace.checkpointSizeMB || null, + checkpointedAt: workspace.checkpointedAt || null, + }; + if (workspace.checkpointEncryption) { + metadata.checkpointEncryption = workspace.checkpointEncryption; + } + if (workspace.checkpointEncryptionKey) { + metadata.checkpointEncryptionKey = workspace.checkpointEncryptionKey; + } + return metadata; +} + +function getLegacyShareName(workspace = {}) { + if (workspace?.legacyShareName) return workspace.legacyShareName; + if (workspace?.shareName) return workspace.shareName; + if (workspace?.checkpointBlobPath) return null; + if (workspace?.containerId && workspace?.imageVersion && !isPersistentCheckpointWorkspace(workspace.imageVersion)) { + return workspace.containerId; + } + if (workspace?.containerId && !workspace?.url && !workspace?.imageVersion) { + return workspace.containerId; + } + return null; +} + +function isActivityRedisConfigured() { + return _activityRedisClientOverride !== undefined || Boolean(config.get('storageConnectionString')); +} + +async function getActivityRedisClient() { + if (_activityRedisClientOverride !== undefined) return _activityRedisClientOverride; + if (_activityRedisClient?.status === 'ready') return _activityRedisClient; + if (_activityRedisClientConnectPromise) return _activityRedisClientConnectPromise; + if (_activityRedisClient) { + try { + _activityRedisClient.disconnect(); + } catch { + // Best effort cleanup before replacing a stale socket. + } + _activityRedisClient = null; + } + + try { + const connectionString = config.get('storageConnectionString'); + if (!connectionString) return null; + + const Redis = (await import('ioredis')).default; + const client = new Redis(connectionString, { + maxRetriesPerRequest: 1, + enableOfflineQueue: false, + enableReadyCheck: true, + lazyConnect: true, + connectTimeout: 10000, + }); + + client.on('error', (error) => { + logger.error(`[WorkspaceActivity] Redis client error: ${error.message}`); + }); + + _activityRedisClient = client; + _activityRedisClientConnectPromise = client.connect() + .then(() => { + _activityRedisClientConnectPromise = null; + return client; + }) + .catch((e) => { + _activityRedisClientConnectPromise = null; + if (_activityRedisClient === client) { + _activityRedisClient = null; + } + try { + client.disconnect(); + } catch { + // Best effort cleanup. + } + throw e; + }); + + return await _activityRedisClientConnectPromise; + } catch (e) { + logger.error(`[WorkspaceActivity] Failed to create Redis client: ${e.message}`); + if (_activityRedisClient) { + try { + _activityRedisClient.disconnect(); + } catch { + // Best effort cleanup. + } + _activityRedisClient = null; + } + _activityRedisClientConnectPromise = null; + return null; + } +} + +async function writeWorkspaceActivityToRedis(entityId, timestamp, idleTimeoutMs) { + const redis = await getActivityRedisClient(); + if (!redis) return false; + + await redis.set( + workspaceActivityKey(entityId), + String(timestamp), + 'PX', + workspaceActivityTtlMs(idleTimeoutMs), + ); + await redis.zadd(workspaceActivityIndexKey(), timestamp, entityId); + return true; +} + +async function readWorkspaceActivityFromRedis(entityId, redis) { + if (!redis) return { ok: true, timestamp: 0 }; + + try { + const value = await redis.get(workspaceActivityKey(entityId)); + const timestamp = Number(value); + return { ok: true, timestamp: Number.isFinite(timestamp) ? timestamp : 0 }; + } catch (e) { + logger.warn(`Failed to read workspace activity from Redis for ${entityId}: ${e.message}`); + return { ok: false, timestamp: 0 }; + } +} + +async function readLatestWorkspaceActivityTimestamp(entityId, minimumTimestamp = 0) { + const localTimestamp = lastActivity.get(entityId) || 0; + const redis = await getActivityRedisClient(); + if (!redis) { + return { + ok: !isActivityRedisConfigured(), + timestamp: Math.max(minimumTimestamp, localTimestamp), + }; + } + + const redisActivity = await readWorkspaceActivityFromRedis(entityId, redis); + if (!redisActivity.ok) { + return { ok: false, timestamp: Math.max(minimumTimestamp, localTimestamp) }; + } + return { + ok: true, + timestamp: Math.max(minimumTimestamp, localTimestamp, redisActivity.timestamp), + }; +} + +async function removeWorkspaceActivityFromRedis(entityId) { + const redis = await getActivityRedisClient(); + if (!redis) return; + + try { + await redis.del(workspaceActivityKey(entityId)); + await redis.zrem(workspaceActivityIndexKey(), entityId); + } catch (e) { + logger.warn(`Failed to clear workspace activity in Redis for ${entityId}: ${e.message}`); + } +} + +function recordWorkspaceActivity(entityId) { + const timestamp = Date.now(); + lastActivity.set(entityId, timestamp); + writeWorkspaceActivityToRedis(entityId, timestamp).catch((e) => { + logger.warn(`Failed to record workspace activity in Redis for ${entityId}: ${e.message}`); + }); +} + +async function getWorkspaceReaperCandidates(redis, now, minimumIdleMs) { + if (!redis) { + if (isActivityRedisConfigured()) { + logger.warn('Skipping idle workspace reap; Redis activity index is unavailable'); + return []; + } + + return Array.from(lastActivity.entries()) + .filter(([, timestamp]) => now - timestamp >= minimumIdleMs) + .map(([entityId]) => entityId); + } + + const cutoff = now - minimumIdleMs; + try { + return await redis.zrangebyscore(workspaceActivityIndexKey(), 0, cutoff); + } catch (e) { + logger.warn(`Failed to read workspace activity index from Redis: ${e.message}`); + return []; + } +} + +async function acquireWorkspaceReaperLock(entityId) { + const redis = await getActivityRedisClient(); + if (!redis) { + if (isActivityRedisConfigured()) { + logger.warn(`Skipping idle reap for entity ${entityId}; Redis activity lock is unavailable`); + return { acquired: false, redis: null }; + } + return { acquired: true, redis: null }; + } + + try { + const result = await redis.set( + workspaceReaperLockKey(entityId), + ACTIVITY_REAPER_HOST_ID, + 'PX', + 60_000, + 'NX', + ); + return { acquired: result === 'OK', redis }; + } catch (e) { + logger.warn(`Failed to acquire workspace reaper lock for ${entityId}: ${e.message}`); + return { acquired: false, redis }; + } +} + +async function releaseWorkspaceReaperLock(entityId, redis) { + if (!redis) return; + + try { + const key = workspaceReaperLockKey(entityId); + const owner = await redis.get(key); + if (owner === ACTIVITY_REAPER_HOST_ID) { + await redis.del(key); + } + } catch (e) { + logger.warn(`Failed to release workspace reaper lock for ${entityId}: ${e.message}`); + } +} + +async function acquireWorkspaceProvisioningLock(entityId) { + const redis = await getActivityRedisClient(); + if (!redis) { + return { acquired: true, redis: null, key: null, token: null }; + } + + const key = workspaceProvisioningLockKey(entityId); + const token = `${ACTIVITY_REAPER_HOST_ID}-${crypto.randomUUID()}`; + try { + const result = await redis.set(key, token, 'PX', WORKSPACE_PROVISIONING_LOCK_TTL_MS, 'NX'); + return { acquired: result === 'OK', redis, key, token }; + } catch (e) { + logger.warn(`Failed to acquire workspace provisioning lock for ${entityId}: ${e.message}`); + return { acquired: true, redis: null, key: null, token: null }; + } +} + +async function releaseWorkspaceProvisioningLock(lock) { + if (!lock?.redis || !lock.key || !lock.token) return; + + try { + const owner = await lock.redis.get(lock.key); + if (owner === lock.token) { + await lock.redis.del(lock.key); + } + } catch (e) { + logger.warn(`Failed to release workspace provisioning lock ${lock.key}: ${e.message}`); + } +} + +/** + * Parse memory limit string (e.g. '512m', '1g') to megabytes. + * Backend-agnostic — returns MB for use by any backend. + */ +export function parseMemoryToMB(str) { + const match = str.toLowerCase().match(/^(\d+(?:\.\d+)?)\s*([kmg]?)b?$/); + if (!match) return 512; // default 512MB + + const num = parseFloat(match[1]); + const unit = match[2]; + + switch (unit) { + case 'k': return Math.round(num / 1024); + case 'm': return Math.round(num); + case 'g': return Math.round(num * 1024); + default: return Math.round(num / (1024 * 1024)); // assume bytes + } +} + +/** + * Make an authenticated HTTP request to an entity's workspace client. + * Auto-provisions the workspace if not yet configured. + * + * @param {string} entityId - Entity UUID + * @param {string} endpoint - Path (e.g. '/shell', '/read') + * @param {Object} [body] - JSON body for POST requests + * @param {Object} [options] + * @param {string} [options.method] - HTTP method (default: POST, or GET if no body) + * @param {number} [options.timeoutMs] - Request timeout in ms (default: 30000) + * @returns {Promise} Parsed JSON response + */ +export async function workspaceRequest(entityId, endpoint, body = null, options = {}) { + if (!isValidWorkspaceEntityId(entityId)) { + logger.warn('Workspace request skipped: missing entityId'); + return invalidWorkspaceEntityResult(); + } + + const method = options.method || (body ? 'POST' : 'GET'); + const timeoutMs = options.timeoutMs || 30000; + const shouldRecordActivity = options.recordActivity !== false; + const markActivity = () => { + if (shouldRecordActivity) recordWorkspaceActivity(entityId); + }; + const onWorkspaceLifecycle = typeof options.onWorkspaceLifecycle === 'function' + ? options.onWorkspaceLifecycle + : null; + const workspaceResult = await ensureWorkspaceReady(entityId, options); + if (!workspaceResult.success) { + return workspaceResult; + } + let { entityConfig } = workspaceResult; + + const { url, secret } = entityConfig.workspace; + markActivity(); + + try { + const fetchOptions = { + method, + headers: { + 'x-workspace-secret': secret, + 'Content-Type': 'application/json', + }, + signal: AbortSignal.timeout(timeoutMs), + }; + + if (body && method !== 'GET') { + fetchOptions.body = JSON.stringify(body); + } + + const response = await fetch(`${url}${endpoint}`, fetchOptions); + + markActivity(); + + if (response.status === 401) { + // Secret mismatch — likely ACI restarted the container, reverting + // its in-memory secret to the bootstrap secret from the env var. + // Try reconfiguring with the bootstrap secret first (fast path), + // then fall back to full reprovision if that fails. + const workspace = entityConfig.workspace; + + if (workspace.bootstrapSecret) { + logger.warn(`Workspace auth failed for ${entityId} — attempting reconfigure with bootstrap secret`); + try { + const backend = await getBackend(); + await reconfigureForEntity(entityId, entityConfig, { + containerName: workspace.containerId, + shareName: workspace.shareName || null, + legacyShareName: workspace.legacyShareName || null, + url: workspace.url, + bootstrapSecret: workspace.bootstrapSecret, + containerId: workspace.containerId, + claimedFromPool: workspace.claimedFromPool, + }, backend, { destroyOnFailure: false }); + + // Retry the request with the fresh secret + entityConfig = await loadEntityConfig(entityId); + const retryOptions = { + method, + headers: { + 'x-workspace-secret': entityConfig.workspace.secret, + 'Content-Type': 'application/json', + }, + signal: AbortSignal.timeout(timeoutMs), + }; + if (body && method !== 'GET') { + retryOptions.body = JSON.stringify(body); + } + const retryResponse = await fetch(`${entityConfig.workspace.url}${endpoint}`, retryOptions); + markActivity(); + if (retryResponse.status === 401) { + // Reconfigure succeeded but auth still fails — something else is wrong + logger.warn(`Workspace auth still failing after reconfigure for ${entityId} — full reprovision`); + } else { + const retryData = await retryResponse.json(); + if (retryData.error) { + return { success: false, error: retryData.error }; + } + return { success: true, ...retryData }; + } + } catch (reconfigErr) { + logger.warn(`Reconfigure failed for ${entityId}: ${reconfigErr.message} — falling back to full reprovision`); + } + } + + // Full reprovision fallback + logger.warn(`Workspace auth failed for ${entityId} — re-provisioning`); + try { + await getEntityStore().upsertEntity({ + ...entityConfig, + workspace: { ...entityConfig.workspace, status: 'error' }, + }); + } catch { /* best effort */ } + + const provisionResult = await provisionWorkspace(entityId, entityConfig, options); + if (!provisionResult.success) { + return { success: false, error: `Workspace auth failed and re-provision failed: ${provisionResult.error}` }; + } + + // Retry the request with fresh config + entityConfig = await loadEntityConfig(entityId); + if (!entityConfig?.workspace?.url) { + return { success: false, error: 'Re-provision completed but config not available' }; + } + try { + const retryOptions = { + method, + headers: { + 'x-workspace-secret': entityConfig.workspace.secret, + 'Content-Type': 'application/json', + }, + signal: AbortSignal.timeout(timeoutMs), + }; + if (body && method !== 'GET') { + retryOptions.body = JSON.stringify(body); + } + const retryResponse = await fetch(`${entityConfig.workspace.url}${endpoint}`, retryOptions); + markActivity(); + if (retryResponse.status === 401) { + return { success: false, error: 'Authentication failed after re-provision' }; + } + const retryData = await retryResponse.json(); + if (retryData.error) { + return { success: false, error: retryData.error }; + } + return { success: true, ...retryData }; + } catch (retryErr) { + return { success: false, error: `Workspace re-provisioned but request still failed: ${retryErr.message}` }; + } + } + + const data = await response.json(); + + if (data.error) { + return { success: false, error: data.error }; + } + + return { success: true, ...data }; + } catch (e) { + // Detect connection-level failures (ECONNREFUSED, ENOTFOUND, ECONNRESET, "fetch failed", etc.) + const causeCode = e.cause?.code; + const isConnectionError = + e.code === 'ECONNREFUSED' || e.code === 'ENOTFOUND' || + causeCode === 'ECONNREFUSED' || causeCode === 'ENOTFOUND' || causeCode === 'ECONNRESET' || + (e.name === 'TypeError' && e.message === 'fetch failed'); + + if (isConnectionError) { + // Container is dead — re-provision and retry the request in the same call + logger.warn(`Workspace for ${entityId} unreachable — re-provisioning`); + try { + await getEntityStore().upsertEntity({ + ...entityConfig, + workspace: { ...entityConfig.workspace, status: 'error' }, + }); + } catch { /* best effort */ } + + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { type: 'start', phase: 'reconnect', message: 'Reconnecting workspace' }); + const provisionResult = await provisionWorkspace(entityId, entityConfig, options); + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { + type: 'finish', + phase: 'reconnect', + success: provisionResult.success, + error: provisionResult.error, + }); + if (!provisionResult.success) { + return { success: false, error: `Workspace died and re-provision failed: ${provisionResult.error}` }; + } + + // Retry the request with fresh config + entityConfig = await loadEntityConfig(entityId); + if (!entityConfig?.workspace?.url) { + return { success: false, error: 'Re-provision completed but config not available' }; + } + try { + const retryOptions = { + method, + headers: { + 'x-workspace-secret': entityConfig.workspace.secret, + 'Content-Type': 'application/json', + }, + signal: AbortSignal.timeout(timeoutMs), + }; + if (body && method !== 'GET') { + retryOptions.body = JSON.stringify(body); + } + const retryResponse = await fetch(`${entityConfig.workspace.url}${endpoint}`, retryOptions); + markActivity(); + const retryData = await retryResponse.json(); + if (retryData.error) { + return { success: false, error: retryData.error }; + } + return { success: true, ...retryData }; + } catch (retryErr) { + return { success: false, error: `Workspace re-provisioned but request still failed: ${retryErr.message}` }; + } + } + + if (e.name === 'TimeoutError' || e.name === 'AbortError') { + return { success: false, error: `Request timed out after ${Math.round(timeoutMs / 1000)}s` }; + } + + logger.error(`Workspace request failed for entity ${entityId}: ${e.message}`); + return { success: false, error: `Workspace request failed: ${e.message}` }; + } +} + +/** + * Provision a workspace container for an entity via the configured backend. + * + * @param {string} entityId - Entity UUID + * @param {Object} entityConfig - Current entity config + * @returns {Promise<{success: boolean, error?: string}>} + */ +async function provisionWorkspace(entityId, entityConfig, options = {}) { + if (!isValidWorkspaceEntityId(entityId)) { + logger.warn('Workspace provisioning skipped: missing entityId'); + return invalidWorkspaceEntityResult(); + } + + // Acquire per-entity lock + if (provisioningLocks.has(entityId)) { + // Wait for existing provisioning to finish + try { + await provisioningLocks.get(entityId); + return { success: true }; + } catch { + return { success: false, error: 'Concurrent provisioning failed' }; + } + } + + let distributedLock = await acquireWorkspaceProvisioningLock(entityId); + if (!distributedLock.acquired) { + logger.info(`Workspace provisioning already in progress for entity ${entityId}; waiting for readiness`); + const transitionResult = await waitForWorkspaceTransition(entityId, options); + if (transitionResult.success && transitionResult.entityConfig?.workspace?.status === 'running' && transitionResult.entityConfig.workspace.url) { + return transitionResult; + } + + distributedLock = await acquireWorkspaceProvisioningLock(entityId); + if (!distributedLock.acquired) { + return transitionResult.success + ? { success: false, error: 'Workspace provisioning lock is still held' } + : transitionResult; + } + entityConfig = (await loadEntityConfig(entityId, { fresh: true })) || entityConfig; + } + + const provisionPromise = _doProvision(entityId, entityConfig); + provisioningLocks.set(entityId, provisionPromise); + + try { + const result = await provisionPromise; + return result; + } finally { + provisioningLocks.delete(entityId); + await releaseWorkspaceProvisioningLock(distributedLock); + } +} + +async function emitWorkspaceLifecycle(onWorkspaceLifecycle, event) { + if (!onWorkspaceLifecycle) return; + try { + await onWorkspaceLifecycle(event); + } catch (e) { + logger.warn(`Workspace lifecycle message failed: ${e.message}`); + } +} + +async function getWorkspaceCheckpointContainerClient(storageConfig = getWorkspaceCheckpointStorageConfig(), options = {}) { + if (!options.ignoreOverride && _workspaceCheckpointContainerClientOverride) { + return _workspaceCheckpointContainerClientOverride; + } + + const { accountName, accountKey, containerName } = storageConfig; + if (!accountName || !accountKey || !containerName) { + throw new Error('Workspace checkpoint storage account credentials and AZURE_BLOB_CONTAINER_NAME are required for workspace checkpoints'); + } + + if (options.ensure !== false) { + await ensureContainer(accountName, accountKey, containerName); + } + const { BlobServiceClient, StorageSharedKeyCredential } = await import('@azure/storage-blob'); + const credential = new StorageSharedKeyCredential(accountName, accountKey); + const blobServiceClient = new BlobServiceClient( + `https://${accountName}.blob.core.windows.net`, + credential, + ); + return blobServiceClient.getContainerClient(containerName); +} + +function getWorkspaceCheckpointStorageConfig() { + const workspaceAccountName = config.get('workspaceAzureFilesStorageAccountName'); + const workspaceAccountKey = config.get('workspaceAzureFilesStorageAccountKey'); + const useWorkspaceStorage = Boolean(workspaceAccountName || workspaceAccountKey); + + return { + accountName: useWorkspaceStorage ? workspaceAccountName : config.get('azureStorageAccountName'), + accountKey: useWorkspaceStorage ? workspaceAccountKey : config.get('azureStorageAccountKey'), + containerName: config.get('azureBlobContainerName'), + }; +} + +function getLegacyWorkspaceCheckpointStorageConfig() { + return { + accountName: config.get('azureStorageAccountName'), + accountKey: config.get('azureStorageAccountKey'), + containerName: config.get('azureBlobContainerName'), + }; +} + +function sameWorkspaceCheckpointStorage(left, right) { + return left?.accountName === right?.accountName && left?.containerName === right?.containerName; +} + +function getWorkspaceCheckpointStorageConfigs() { + const preferred = getWorkspaceCheckpointStorageConfig(); + const legacy = getLegacyWorkspaceCheckpointStorageConfig(); + if (!legacy.accountName || !legacy.accountKey || sameWorkspaceCheckpointStorage(preferred, legacy)) { + return [preferred]; + } + return [preferred, legacy]; +} + +async function workspaceCheckpointBlobExists(blobPath, storageConfig) { + if (!storageConfig.accountName || !storageConfig.accountKey || !storageConfig.containerName) { + return false; + } + try { + const containerClient = await getWorkspaceCheckpointContainerClient(storageConfig, { + ensure: false, + ignoreOverride: true, + }); + return await containerClient.getBlockBlobClient(blobPath).exists(); + } catch (e) { + logger.warn(`Could not check workspace checkpoint blob ${blobPath} in ${storageConfig.accountName}: ${e.message}`); + return false; + } +} + +async function validateWorkspaceCheckpointBlobIdentity(blobPath, entityId, storageConfig) { + const containerClient = await getWorkspaceCheckpointContainerClient(storageConfig, { + ensure: false, + ignoreOverride: true, + }); + const properties = await containerClient.getBlockBlobClient(blobPath).getProperties(); + validateWorkspaceCheckpointMetadata(properties.metadata || {}, entityId); +} + +async function readWorkspaceCheckpointBlobFields(blobPath, entityId, storageConfig) { + const containerClient = await getWorkspaceCheckpointContainerClient(storageConfig, { + ensure: false, + }); + const properties = await containerClient.getBlockBlobClient(blobPath).getProperties(); + validateWorkspaceCheckpointMetadata(properties.metadata || {}, entityId); + + const metadataCheckpointedAt = metadataValue(properties.metadata || {}, 'checkpointedAt'); + const checkpointedAtMs = parseTimestampMs(metadataCheckpointedAt) + || parseTimestampMs(properties.lastModified); + const sizeBytes = properties.contentLength || null; + return { + checkpointBlobPath: blobPath, + checkpointPreviousBlobPath: blobPath.endsWith('/workspace.tar.gz') + ? blobPath.replace(/\/workspace\.tar\.gz$/, '/workspace.prev.tar.gz') + : workspaceCheckpointBlobPath(entityId, 'workspace.prev.tar.gz'), + checkpointSizeBytes: sizeBytes, + checkpointSizeMB: sizeBytes + ? Math.round(sizeBytes / 1024 / 1024 * 100) / 100 + : null, + checkpointedAt: checkpointedAtMs + ? new Date(checkpointedAtMs).toISOString() + : null, + checkpointEncryption: checkpointEncryptionFromMetadata(properties.metadata || {}), + checkpointCompression: metadataValue(properties.metadata || {}, 'checkpointCompression') || null, + }; +} + +async function readExistingWorkspaceCheckpointBlobFields(entityId, workspace = {}) { + const candidatePaths = workspace.checkpointBlobPath + ? [workspace.checkpointBlobPath] + : workspaceCheckpointBlobPathCandidates(entityId); + + for (const candidatePath of candidatePaths) { + for (const storageConfig of getWorkspaceCheckpointStorageConfigs()) { + try { + return await readWorkspaceCheckpointBlobFields(candidatePath, entityId, storageConfig); + } catch (e) { + if (e.statusCode === 404 || e.code === 'BlobNotFound') continue; + logger.warn(`Could not read workspace checkpoint blob ${candidatePath} in ${storageConfig.accountName}: ${e.message}`); + } + } + } + return null; +} + +async function createWorkspaceCheckpointSasUrl(blobPath, permissions, storageConfig, options = {}) { + const { accountName, accountKey, containerName } = storageConfig; + if (!accountName || !accountKey || !containerName) { + throw new Error('Workspace checkpoint storage account credentials and AZURE_BLOB_CONTAINER_NAME are required for workspace checkpoints'); + } + + const { + StorageSharedKeyCredential, + BlobSASPermissions, + generateBlobSASQueryParameters, + } = await import('@azure/storage-blob'); + const credential = new StorageSharedKeyCredential(accountName, accountKey); + const startsOn = new Date(Date.now() - 5 * 60 * 1000); + const expiresOn = new Date(Date.now() + (options.sasTtlMs || 30 * 60 * 1000)); + const sas = generateBlobSASQueryParameters({ + containerName, + blobName: blobPath, + permissions: BlobSASPermissions.parse(permissions), + startsOn, + expiresOn, + }, credential).toString(); + + return `https://${accountName}.blob.core.windows.net/${containerName}/${encodeURI(blobPath)}?${sas}`; +} + +async function copyWorkspaceCheckpointFromLegacyStorage(blobPath, legacyStorage, preferredStorage, options = {}) { + const sourceUrl = await createWorkspaceCheckpointSasUrl(blobPath, 'r', legacyStorage, { + ...options, + sasTtlMs: options.sasTtlMs || 60 * 60 * 1000, + }); + const preferredContainer = await getWorkspaceCheckpointContainerClient(preferredStorage, { ignoreOverride: true }); + const destinationBlob = preferredContainer.getBlockBlobClient(blobPath); + const poller = await destinationBlob.beginCopyFromURL(sourceUrl, { intervalInMs: 1000 }); + const result = await poller.pollUntilDone(); + if (result.copyStatus && result.copyStatus !== 'success') { + throw new Error(`copy status ${result.copyStatus}`); + } +} + +async function getReadableWorkspaceCheckpointStorageConfig(blobPath, options = {}) { + const preferred = getWorkspaceCheckpointStorageConfig(); + if (await workspaceCheckpointBlobExists(blobPath, preferred)) { + if (options.entityId) { + await validateWorkspaceCheckpointBlobIdentity(blobPath, options.entityId, preferred); + } + return preferred; + } + + const legacy = getLegacyWorkspaceCheckpointStorageConfig(); + if (!legacy.accountName || !legacy.accountKey || sameWorkspaceCheckpointStorage(preferred, legacy)) { + return preferred; + } + if (!await workspaceCheckpointBlobExists(blobPath, legacy)) { + return preferred; + } + if (options.entityId) { + await validateWorkspaceCheckpointBlobIdentity(blobPath, options.entityId, legacy); + } + + try { + await copyWorkspaceCheckpointFromLegacyStorage(blobPath, legacy, preferred, options); + logger.info(`Copied legacy workspace checkpoint blob ${blobPath} into preferred storage account ${preferred.accountName}`); + return preferred; + } catch (e) { + logger.warn(`Could not copy legacy workspace checkpoint blob ${blobPath} into preferred storage: ${e.message}; restoring from legacy storage`); + return legacy; + } +} + +function workspaceCheckpointBlobPathsForDestroy(entityId, workspace = {}) { + const paths = new Set(); + if (workspace.checkpointBlobPath) paths.add(workspace.checkpointBlobPath); + if (workspace.checkpointPreviousBlobPath) paths.add(workspace.checkpointPreviousBlobPath); + if (paths.size > 0) { + paths.add(workspaceCheckpointBlobPath(entityId)); + paths.add(workspaceCheckpointBlobPath(entityId, 'workspace.prev.tar.gz')); + } + return [...paths]; +} + +async function deleteWorkspaceCheckpointBlobs(entityId, workspace = {}) { + const blobPaths = workspaceCheckpointBlobPathsForDestroy(entityId, workspace); + if (blobPaths.length === 0) return { deleted: 0, attempted: 0 }; + + let deleted = 0; + let attempted = 0; + for (const storageConfig of getWorkspaceCheckpointStorageConfigs()) { + const containerClient = await getWorkspaceCheckpointContainerClient(storageConfig); + for (const blobPath of blobPaths) { + attempted++; + const result = await containerClient.getBlockBlobClient(blobPath).deleteIfExists(); + if (result.succeeded) deleted += 1; + } + } + return { deleted, attempted }; +} + +async function recoverExistingWorkspaceCheckpoint(entityId, entityConfig) { + if (entityConfig?.workspace?.checkpointBlobPath) return entityConfig; + + let checkpointFields = null; + try { + checkpointFields = await readExistingWorkspaceCheckpointBlobFields(entityId, entityConfig?.workspace); + } catch (e) { + logger.warn(`Could not validate existing workspace checkpoint for ${entityId}: ${e.message}`); + return entityConfig; + } + if (!checkpointFields) return entityConfig; + + logger.info(`Recovered existing workspace checkpoint metadata for ${entityId}`); + return { + ...entityConfig, + workspace: { + ...(entityConfig.workspace || {}), + ...checkpointFields, + }, + }; +} + +async function recoverFreshWorkspaceCheckpointFromBlob(entityId, entityConfig, minimumTimestamp) { + if (!entityConfig?.workspace?.checkpointBlobPath) return null; + + const checkpointFields = await readExistingWorkspaceCheckpointBlobFields(entityId, entityConfig.workspace); + if (!checkpointFields || !checkpointFields.checkpointedAt) return null; + if (minimumTimestamp && parseTimestampMs(checkpointFields.checkpointedAt) < minimumTimestamp) return null; + + const current = (await loadEntityConfig(entityId, { fresh: true })) || entityConfig; + if (!current?.workspace) return null; + + const updated = { + ...current, + workspace: { + ...current.workspace, + ...checkpointFields, + }, + }; + await getEntityStore().upsertEntity(updated); + return updated; +} + +async function createWorkspaceCheckpointReadUrl(blobPath, options = {}) { + if (options.checkpointSasUrl) return options.checkpointSasUrl; + + const storageConfig = await getReadableWorkspaceCheckpointStorageConfig(blobPath, options); + const { accountName, accountKey, containerName } = storageConfig; + if (!accountName || !accountKey || !containerName) { + throw new Error('Workspace checkpoint storage account credentials and AZURE_BLOB_CONTAINER_NAME are required for workspace checkpoint restore'); + } + + return createWorkspaceCheckpointSasUrl(blobPath, 'r', storageConfig, options); +} + +async function createWorkspaceCheckpointWriteUrl(blobPath, options = {}) { + if (options.checkpointWriteSasUrl) return options.checkpointWriteSasUrl; + + const { accountName, accountKey, containerName } = getWorkspaceCheckpointStorageConfig(); + if (!accountName || !accountKey || !containerName) { + throw new Error('Workspace checkpoint storage account credentials and AZURE_BLOB_CONTAINER_NAME are required for workspace checkpoint upload'); + } + + await ensureContainer(accountName, accountKey, containerName); + return createWorkspaceCheckpointSasUrl(blobPath, 'cw', { accountName, accountKey, containerName }, options); +} + +async function copyExistingWorkspaceCheckpointToPrevious(entityId, currentBlobPath, previousBlobPath, options = {}) { + if (!currentBlobPath || !previousBlobPath) return false; + const storageConfig = getWorkspaceCheckpointStorageConfig(); + if (!await workspaceCheckpointBlobExists(currentBlobPath, storageConfig)) return false; + await validateWorkspaceCheckpointBlobIdentity(currentBlobPath, entityId, storageConfig); + const sourceUrl = await createWorkspaceCheckpointSasUrl(currentBlobPath, 'r', storageConfig, { + ...options, + sasTtlMs: options.sasTtlMs || 60 * 60 * 1000, + }); + const containerClient = await getWorkspaceCheckpointContainerClient(storageConfig, { ignoreOverride: true }); + const destinationBlob = containerClient.getBlockBlobClient(previousBlobPath); + const poller = await destinationBlob.beginCopyFromURL(sourceUrl, { intervalInMs: 1000 }); + const result = await poller.pollUntilDone(); + if (result.copyStatus && result.copyStatus !== 'success') { + throw new Error(`copy status ${result.copyStatus}`); + } + return true; +} + +function getWorkspaceFilesStorageAccount() { + return { + accountName: config.get('workspaceAzureFilesStorageAccountName') + || config.get('azureStorageAccountName'), + accountKey: config.get('workspaceAzureFilesStorageAccountKey') + || config.get('azureStorageAccountKey'), + }; +} + +async function uploadWorkspaceArchiveFromLegacyShare(entityId, workspace, archivePath, blobPath) { + const shareName = workspace?.legacyShareName || workspace?.shareName; + if (!shareName) { + return null; + } + if (archivePath !== WORKSPACE_CHECKPOINT_PATH) { + throw new Error(`Cannot copy non-standard legacy checkpoint path from Azure Files: ${archivePath}`); + } + + if (_workspaceLegacyShareUploadOverride) { + return await _workspaceLegacyShareUploadOverride({ + entityId, + workspace, + archivePath, + blobPath, + shareName, + }); + } + + const { accountName, accountKey } = getWorkspaceFilesStorageAccount(); + if (!accountName || !accountKey) { + throw new Error('Workspace Azure Files storage credentials are required for legacy checkpoint migration'); + } + + const { + ShareServiceClient, + StorageSharedKeyCredential, + } = await import('@azure/storage-file-share'); + const credential = new StorageSharedKeyCredential(accountName, accountKey); + const shareServiceClient = new ShareServiceClient( + `https://${accountName}.file.core.windows.net`, + credential, + ); + const fileClient = shareServiceClient + .getShareClient(shareName) + .rootDirectoryClient + .getFileClient('workspace.tar.gz'); + + const fileProperties = await fileClient.getProperties(); + const download = await fileClient.download(); + if (!download.readableStreamBody) { + throw new Error(`Legacy checkpoint archive is not readable from Azure Files share ${shareName}`); + } + + const containerClient = await getWorkspaceCheckpointContainerClient(); + const blockBlobClient = containerClient.getBlockBlobClient(blobPath); + await blockBlobClient.uploadStream(download.readableStreamBody, 4 * 1024 * 1024, 5, { + blobHTTPHeaders: { blobContentType: 'application/gzip' }, + metadata: workspaceCheckpointBlobMetadata(entityId, { + checkpointedAt: new Date().toISOString(), + source: 'legacy-azure-files', + }), + }); + + return { + blobPath, + sizeBytes: fileProperties.contentLength || null, + }; +} + +function isFetchTimeoutError(error) { + const name = error?.name || error?.cause?.name; + const code = error?.code || error?.cause?.code; + const message = error?.message || ''; + return name === 'TimeoutError' + || name === 'AbortError' + || code === 'ABORT_ERR' + || code === 'UND_ERR_ABORTED' + || code === 'UND_ERR_HEADERS_TIMEOUT' + || /aborted due to timeout|operation was aborted|timeout/i.test(message); +} + +async function uploadWorkspaceArchiveFromContainer(entityId, workspace, archivePath, blobPath, timeoutMs, options = {}) { + const archiveUrl = await createWorkspaceCheckpointWriteUrl(blobPath, options); + const metadata = { + ...workspaceCheckpointBlobMetadata(entityId), + checkpointedAt: new Date().toISOString(), + }; + + const uploadFromWorkspace = async (currentWorkspace) => fetch(`${currentWorkspace.url}/upload-url`, { + method: 'POST', + headers: { + 'x-workspace-secret': currentWorkspace.secret, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ archiveUrl, archivePath, metadata }), + signal: AbortSignal.timeout(timeoutMs), + dispatcher: getLongFetchDispatcher(timeoutMs), + }); + + const retryUploadWithFreshWorkspace = async (reason, { recoverAuth = false } = {}) => { + const freshConfig = await loadEntityConfig(entityId, { fresh: true }); + let retryWorkspace = freshConfig?.workspace || workspace; + if (recoverAuth && retryWorkspace?.bootstrapSecret && retryWorkspace?.url) { + const recoveredConfig = await recoverWorkspaceAuthWithBootstrapSecret(entityId, freshConfig || { workspace: retryWorkspace }); + retryWorkspace = recoveredConfig?.workspace || retryWorkspace; + } + if (!retryWorkspace?.url || !retryWorkspace?.secret) return null; + + logger.warn(`Retrying workspace checkpoint upload for ${entityId} after /upload-url ${reason}`); + return await uploadFromWorkspace(retryWorkspace); + }; + + let uploadResponse; + try { + uploadResponse = await uploadFromWorkspace(workspace); + } catch (e) { + if (isFetchTimeoutError(e)) { + throw new Error(`/upload-url timed out after ${Math.round(timeoutMs / 1000)}s`); + } + const legacyUpload = await uploadWorkspaceArchiveFromLegacyShare(entityId, workspace, archivePath, blobPath); + if (legacyUpload) { + logger.info(`Copied legacy Azure Files checkpoint for ${entityId} directly to Blob after /upload-url failed: ${e.message}`); + return legacyUpload; + } + uploadResponse = await retryUploadWithFreshWorkspace(`fetch failed: ${e.message}`); + } + if (!uploadResponse) { + throw new Error('/upload-url failed and no workspace retry target is available'); + } + + let uploadBody = await uploadResponse.json().catch(() => ({})); + if (uploadResponse.status === 401) { + uploadResponse = await retryUploadWithFreshWorkspace(`returned 401: ${uploadBody.error || uploadResponse.statusText}`, { + recoverAuth: true, + }); + if (!uploadResponse) { + throw new Error(`/upload-url returned 401: ${uploadBody.error || 'workspace auth recovery unavailable'}`); + } + uploadBody = await uploadResponse.json().catch(() => ({})); + } + + if (!uploadResponse.ok || uploadBody.error) { + const isOldWorkspaceImage = uploadResponse.status === 404; + if (!isOldWorkspaceImage) { + throw new Error(`/upload-url returned ${uploadResponse.status}: ${uploadBody.error || uploadResponse.statusText}`); + } + + const legacyUpload = await uploadWorkspaceArchiveFromLegacyShare(entityId, workspace, archivePath, blobPath); + if (legacyUpload) { + logger.info(`Copied legacy Azure Files checkpoint for ${entityId} directly to Blob`); + return legacyUpload; + } + throw new Error('Workspace image does not support /upload-url and no legacy Azure Files share is available for checkpoint upload'); + } + + return { + blobPath, + sizeBytes: uploadBody.sizeBytes || null, + }; +} + +async function uploadStreamingWorkspaceCheckpointFromContainer(entityId, entityConfig, workspace, blobPath, timeoutMs, options = {}) { + const archiveUrl = await createWorkspaceCheckpointWriteUrl(blobPath, options); + const encryption = await getOrCreateWorkspaceCheckpointEncryptionKey(entityId, entityConfig); + const checkpointedAt = options.checkpointedAt || new Date().toISOString(); + const metadata = { + ...workspaceCheckpointBlobMetadata(entityId), + checkpointedAt, + checkpointEncrypted: 'true', + checkpointEncryptionAlgorithm: encryption.algorithm, + checkpointEncryptionKeyId: encryption.keyId, + }; + + const uploadFromWorkspace = async (currentWorkspace) => fetch(`${currentWorkspace.url}/backup-upload-url`, { + method: 'POST', + headers: { + 'x-workspace-secret': currentWorkspace.secret, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + archiveUrl, + metadata, + encryption: { + algorithm: encryption.algorithm, + keyBase64: encryption.keyBase64, + keyId: encryption.keyId, + }, + }), + signal: AbortSignal.timeout(timeoutMs), + dispatcher: getLongFetchDispatcher(timeoutMs), + }); + + const retryUploadWithFreshWorkspace = async (reason, { recoverAuth = false } = {}) => { + const freshConfig = await loadEntityConfig(entityId, { fresh: true }); + let retryWorkspace = freshConfig?.workspace || workspace; + if (recoverAuth && retryWorkspace?.bootstrapSecret && retryWorkspace?.url) { + const recoveredConfig = await recoverWorkspaceAuthWithBootstrapSecret(entityId, freshConfig || { workspace: retryWorkspace }); + retryWorkspace = recoveredConfig?.workspace || retryWorkspace; + } + if (!retryWorkspace?.url || !retryWorkspace?.secret) return null; + + logger.warn(`Retrying encrypted workspace checkpoint upload for ${entityId} after /backup-upload-url ${reason}`); + return await uploadFromWorkspace(retryWorkspace); + }; + + let uploadResponse; + try { + uploadResponse = await uploadFromWorkspace(workspace); + } catch (e) { + if (isFetchTimeoutError(e)) { + throw new Error(`/backup-upload-url timed out after ${Math.round(timeoutMs / 1000)}s`); + } + uploadResponse = await retryUploadWithFreshWorkspace(`fetch failed: ${e.message}`); + } + if (!uploadResponse) { + throw new Error('/backup-upload-url failed and no workspace retry target is available'); + } + + let uploadBody = await uploadResponse.json().catch(() => ({})); + if (uploadResponse.status === 401) { + uploadResponse = await retryUploadWithFreshWorkspace(`returned 401: ${uploadBody.error || uploadResponse.statusText}`, { + recoverAuth: true, + }); + if (!uploadResponse) { + throw new Error(`/backup-upload-url returned 401: ${uploadBody.error || 'workspace auth recovery unavailable'}`); + } + uploadBody = await uploadResponse.json().catch(() => ({})); + } + + if (uploadResponse.status === 404) { + return { unsupported: true }; + } + if (!uploadResponse.ok || uploadBody.error) { + throw new Error(`/backup-upload-url returned ${uploadResponse.status}: ${uploadBody.error || uploadResponse.statusText}`); + } + if (!uploadBody.encrypted || !uploadBody.encryption?.ivBase64 || !uploadBody.encryption?.tagBase64) { + throw new Error('/backup-upload-url did not return checkpoint encryption metadata'); + } + + return { + blobPath, + sizeBytes: uploadBody.sizeBytes || null, + durationMs: uploadBody.durationMs || null, + encryption: { + algorithm: uploadBody.encryption.algorithm || encryption.algorithm, + keyId: uploadBody.encryption.keyId || encryption.keyId, + ivBase64: uploadBody.encryption.ivBase64, + tagBase64: uploadBody.encryption.tagBase64, + compression: uploadBody.encryption.compression || uploadBody.compression || 'gzip', + }, + entityConfig: encryption.entityConfig, + timestamp: checkpointedAt, + }; +} + +async function uploadWorkspaceCheckpoint(entityId, workspace, backupBody, timeoutMs, options = {}) { + const checkpointBlobPath = workspaceCheckpointBlobPath(entityId); + if (!backupBody) { + const previousBlobPath = workspaceCheckpointBlobPath(entityId, 'workspace.prev.tar.gz'); + let copiedPrevious = false; + try { + copiedPrevious = await copyExistingWorkspaceCheckpointToPrevious(entityId, checkpointBlobPath, previousBlobPath, options); + } catch (e) { + logger.warn(`Failed to copy previous workspace checkpoint for ${entityId}: ${e.message}`); + } + const checkpointedAt = new Date().toISOString(); + const streamingUpload = await uploadStreamingWorkspaceCheckpointFromContainer( + entityId, + options.entityConfig || { id: entityId, workspace }, + workspace, + checkpointBlobPath, + timeoutMs, + { ...options, checkpointedAt }, + ); + if (streamingUpload.unsupported) { + return { unsupported: true }; + } + return { + path: WORKSPACE_CHECKPOINT_PATH, + blobPath: streamingUpload.blobPath, + previousBlobPath: copiedPrevious ? previousBlobPath : null, + sizeBytes: streamingUpload.sizeBytes, + sizeMB: streamingUpload.sizeBytes + ? Math.round(streamingUpload.sizeBytes / 1024 / 1024 * 100) / 100 + : null, + timestamp: streamingUpload.timestamp || checkpointedAt, + durationMs: streamingUpload.durationMs || null, + encryption: streamingUpload.encryption, + compression: streamingUpload.encryption?.compression || null, + entityConfig: streamingUpload.entityConfig, + }; + } + const checkpointUpload = await uploadWorkspaceArchiveFromContainer( + entityId, + workspace, + backupBody.path || WORKSPACE_CHECKPOINT_PATH, + checkpointBlobPath, + timeoutMs, + options, + ); + + let previousBlobPath = null; + if (backupBody.previousPath) { + try { + previousBlobPath = workspaceCheckpointBlobPath(entityId, 'workspace.prev.tar.gz'); + await uploadWorkspaceArchiveFromContainer( + entityId, + workspace, + backupBody.previousPath, + previousBlobPath, + timeoutMs, + options, + ); + } catch (e) { + logger.warn(`Failed to upload previous workspace checkpoint for ${entityId}: ${e.message}`); + previousBlobPath = null; + } + } + + return { + path: backupBody.path || WORKSPACE_CHECKPOINT_PATH, + blobPath: checkpointUpload.blobPath, + previousBlobPath, + sizeBytes: checkpointUpload.sizeBytes || backupBody.sizeBytes || null, + sizeMB: backupBody.sizeMB || ( + checkpointUpload.sizeBytes + ? Math.round(checkpointUpload.sizeBytes / 1024 / 1024 * 100) / 100 + : null + ), + timestamp: backupBody.timestamp || new Date().toISOString(), + durationMs: backupBody.durationMs || null, + }; +} + +function workspaceCheckpointFields(checkpoint) { + if (!checkpoint?.blobPath) return null; + const checkpointedAtMs = parseTimestampMs(checkpoint.timestamp) || Date.now(); + const fields = { + checkpointBlobPath: checkpoint.blobPath, + checkpointPreviousBlobPath: checkpoint.previousBlobPath || null, + checkpointSizeBytes: checkpoint.sizeBytes || null, + checkpointSizeMB: checkpoint.sizeMB || null, + checkpointedAt: new Date(checkpointedAtMs).toISOString(), + }; + if (checkpoint.encryption) { + fields.checkpointEncryption = checkpoint.encryption; + } + if (checkpoint.compression || checkpoint.encryption?.compression) { + fields.checkpointCompression = checkpoint.compression || checkpoint.encryption.compression; + } + return fields; +} + +async function persistWorkspaceCheckpoint(entityId, entityConfig, checkpoint, options = {}) { + const fields = workspaceCheckpointFields(checkpoint); + if (!fields) return entityConfig; + + const current = (await loadEntityConfig(entityId, { fresh: true })) || entityConfig; + if (!current?.workspace) return current; + + const nextWorkspace = { + ...current.workspace, + ...fields, + }; + if (options.clearShareName) { + delete nextWorkspace.shareName; + } + if (options.legacyShareName) { + nextWorkspace.legacyShareName = options.legacyShareName; + } + + const updated = { + ...current, + workspace: nextWorkspace, + }; + await getEntityStore().upsertEntity(updated); + return updated; +} + +async function markWorkspaceCheckpointFresh(entityId, entityConfig, timestamp = new Date().toISOString()) { + const current = (await loadEntityConfig(entityId, { fresh: true })) || entityConfig; + if (!current?.workspace?.checkpointBlobPath) return current; + + const updated = { + ...current, + workspace: { + ...current.workspace, + checkpointedAt: timestamp, + }, + }; + await getEntityStore().upsertEntity(updated); + return updated; +} + +async function checkpointAndPersistWorkspace(entityId, entityConfig, options = {}) { + const checkpointResult = await checkpointWorkspace(entityId, entityConfig, options); + if (!checkpointResult.success || checkpointResult.skipped || !checkpointResult.checkpoint?.blobPath) { + return checkpointResult; + } + + const updatedEntityConfig = await persistWorkspaceCheckpoint( + entityId, + entityConfig, + checkpointResult.checkpoint, + options, + ); + return { + ...checkpointResult, + entityConfig: updatedEntityConfig, + }; +} + +async function recoverWorkspaceAuthWithBootstrapSecret(entityId, entityConfig) { + let workspace = entityConfig?.workspace; + if (!workspace?.bootstrapSecret || !workspace?.url) return null; + + logger.warn(`Workspace auth failed for ${entityId} — attempting reconfigure with bootstrap secret`); + const backend = await getBackend(); + await reconfigureForEntity(entityId, entityConfig, { + containerName: workspace.containerId, + shareName: workspace.shareName || null, + legacyShareName: workspace.legacyShareName || null, + url: workspace.url, + bootstrapSecret: workspace.bootstrapSecret, + containerId: workspace.containerId, + claimedFromPool: workspace.claimedFromPool, + }, backend, { destroyOnFailure: false }); + + return await loadEntityConfig(entityId, { fresh: true }); +} + +async function fetchWorkspaceJson(url, secret, endpoint, options = {}) { + const timeoutMs = options.timeoutMs || 30000; + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + let response; + try { + response = await fetch(`${url}${endpoint}`, { + method: options.method || 'GET', + headers: { + 'x-workspace-secret': secret, + 'Content-Type': 'application/json', + }, + ...(options.body ? { body: JSON.stringify(options.body) } : {}), + signal: controller.signal, + }); + } catch (e) { + throw new Error(`${endpoint} fetch failed: ${e.message}`); + } finally { + clearTimeout(timeout); + } + const body = await response.json().catch(() => ({})); + return { response, body }; +} + +async function fetchWorkspaceJsonWithAuthRecovery(entityId, entityConfig, endpoint, options = {}) { + let currentConfig = entityConfig; + let workspace = currentConfig?.workspace; + if (!workspace?.url || !workspace?.secret) { + throw new Error(`${endpoint} fetch failed: workspace URL or secret is missing`); + } + + let result = await fetchWorkspaceJson(workspace.url, workspace.secret, endpoint, options); + if (result.response.status !== 401 || !workspace.bootstrapSecret || !entityId) { + return { + ...result, + entityConfig: currentConfig, + workspace, + authRecovered: false, + }; + } + + const recoveredConfig = await recoverWorkspaceAuthWithBootstrapSecret(entityId, currentConfig); + if (!recoveredConfig?.workspace?.url || !recoveredConfig?.workspace?.secret) { + return { + ...result, + entityConfig: currentConfig, + workspace, + authRecovered: false, + }; + } + + currentConfig = recoveredConfig; + workspace = recoveredConfig.workspace; + result = await fetchWorkspaceJson(workspace.url, workspace.secret, endpoint, options); + return { + ...result, + entityConfig: currentConfig, + workspace, + authRecovered: true, + }; +} + +async function checkpointWorkspace(entityId, entityConfig, options = {}) { + let workspace = entityConfig?.workspace; + if (!workspace?.url || !workspace?.secret) { + return { success: false, error: 'Workspace URL or secret is missing' }; + } + + const timeoutMs = options.timeoutMs || 900000; + const onWorkspaceLifecycle = typeof options.onWorkspaceLifecycle === 'function' + ? options.onWorkspaceLifecycle + : null; + let activePhase = null; + const startPhase = async (phase, message) => { + activePhase = phase; + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { type: 'start', phase, message }); + }; + const finishPhase = async (success, error = null) => { + if (!activePhase) return; + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { type: 'finish', phase: activePhase, success, error }); + activePhase = null; + }; + try { + let healthResult = await fetchWorkspaceJsonWithAuthRecovery(entityId, entityConfig, '/health', { + timeoutMs: Math.min(timeoutMs, 30000), + }); + entityConfig = healthResult.entityConfig; + workspace = healthResult.workspace; + const { response: healthResponse, body: healthBody } = healthResult; + if (!healthResponse.ok) { + return { success: false, error: `/health returned ${healthResponse.status}: ${healthBody.error || healthResponse.statusText}` }; + } + + let statusResult = await fetchWorkspaceJsonWithAuthRecovery(entityId, entityConfig, '/status', { + timeoutMs: Math.min(timeoutMs, 30000), + }); + entityConfig = statusResult.entityConfig; + workspace = statusResult.workspace; + const { response: statusResponse, body: statusBody } = statusResult; + if (!statusResponse.ok) { + return { success: false, error: `/status returned ${statusResponse.status}: ${statusBody.error || statusResponse.statusText}` }; + } + const workspaceVersion = healthBody.version || statusBody.version; + if (!isPersistentCheckpointWorkspace(workspaceVersion)) { + return { + success: true, + skipped: true, + reason: `workspace version ${workspaceVersion || 'unknown'} stores files directly on Azure Files`, + }; + } + + const uploadCheckpoint = options.uploadCheckpoint || _workspaceCheckpointUploadOverride || uploadWorkspaceCheckpoint; + const canUseStreamingCheckpoint = uploadCheckpoint === uploadWorkspaceCheckpoint + && isEncryptedStreamingCheckpointWorkspace(workspaceVersion); + if (canUseStreamingCheckpoint) { + await startPhase('checkpointUpload', 'Backing up and saving workspace'); + const checkpoint = await uploadCheckpoint(entityId, workspace, null, timeoutMs, { + ...options, + entityConfig, + }); + if (!checkpoint?.unsupported) { + await finishPhase(true); + return { success: true, checkpoint }; + } + await finishPhase(true); + logger.info(`Workspace image for ${entityId} does not support /backup-upload-url; falling back to two-step checkpoint upload`); + } + + await startPhase('checkpointBackup', 'Backing up workspace'); + const backupResult = await fetchWorkspaceJsonWithAuthRecovery(entityId, entityConfig, '/backup', { + method: 'POST', + timeoutMs, + }); + entityConfig = backupResult.entityConfig; + workspace = backupResult.workspace; + const { response, body } = backupResult; + if (!response.ok) { + await finishPhase(false, `/backup returned ${response.status}: ${body.error || response.statusText}`); + return { success: false, error: `/backup returned ${response.status}: ${body.error || response.statusText}` }; + } + if (body.error) { + await finishPhase(false, body.error); + return { success: false, error: body.error }; + } + await finishPhase(true); + + await startPhase('checkpointUpload', 'Saving workspace backup'); + const checkpoint = await uploadCheckpoint(entityId, workspace, body, timeoutMs, options); + await finishPhase(true); + return { success: true, checkpoint }; + } catch (e) { + await finishPhase(false, e.message); + return { success: false, error: `Checkpoint failed: ${e.message}` }; + } +} + +function isPersistentCheckpointWorkspace(version) { + if (typeof version !== 'string') return false; + const parts = version.split('.').map(part => Number.parseInt(part, 10)); + if (parts.some(Number.isNaN)) return false; + const [major = 0, minor = 0, patch = 0] = parts; + if (major !== 1) return major > 1; + if (minor !== 0) return minor > 0; + return patch >= 3; +} + +function isEncryptedStreamingCheckpointWorkspace(version) { + if (typeof version !== 'string') return false; + const parts = version.split('.').map(part => Number.parseInt(part, 10)); + if (parts.some(Number.isNaN)) return false; + const [major = 0, minor = 0, patch = 0] = parts; + if (major !== 1) return major > 1; + if (minor !== 0) return minor > 0; + return patch >= 10; +} + +function getWorkspaceTransitionWaitMs(options = {}) { + const waitMs = Number(options.transitionWaitMs); + return Number.isFinite(waitMs) && waitMs >= 0 + ? waitMs + : WORKSPACE_TRANSITION_WAIT_MS; +} + +function getWorkspaceTransitionPollMs(options = {}) { + const pollMs = Number(options.transitionPollMs); + return Number.isFinite(pollMs) && pollMs >= 0 + ? pollMs + : WORKSPACE_TRANSITION_POLL_MS; +} + +function workspaceTransitionLifecycle(status) { + if (status === 'starting') { + return { phase: 'wake', message: 'Waiting for workspace to start' }; + } + return { phase: 'provision', message: 'Waiting for workspace setup' }; +} + +async function waitForWorkspaceTransition(entityId, options = {}) { + const deadline = Date.now() + getWorkspaceTransitionWaitMs(options); + const pollMs = getWorkspaceTransitionPollMs(options); + + while (Date.now() < deadline) { + await sleep(Math.min(pollMs, Math.max(deadline - Date.now(), 0))); + const entityConfig = await loadEntityConfig(entityId, { fresh: true }); + if (!entityConfig) { + return { success: false, error: 'Entity not found' }; + } + + const workspace = entityConfig.workspace; + if (workspace?.status === 'running' && workspace.url) { + return { success: true, entityConfig }; + } + if (workspace?.status === 'error') { + return { success: false, error: 'Workspace provisioning failed' }; + } + if (workspace?.status && !WORKSPACE_TRANSITION_STATUSES.has(workspace.status)) { + return { success: true, entityConfig }; + } + } + + return { success: false, error: 'Workspace is still provisioning, please retry shortly' }; +} + +async function ensureWorkspaceReady(entityId, options = {}) { + if (!isValidWorkspaceEntityId(entityId)) { + logger.warn('Workspace readiness skipped: missing entityId'); + return invalidWorkspaceEntityResult(); + } + + const onWorkspaceLifecycle = typeof options.onWorkspaceLifecycle === 'function' + ? options.onWorkspaceLifecycle + : null; + const waitForTransition = options.waitForTransition !== false; + + let entityConfig = await loadEntityConfig(entityId); + if (!entityConfig) { + return { success: false, error: 'Entity not found' }; + } + + // Recover from stale transitional states ('starting', 'provisioning'). + // If stuck for >5 min, mark as error to trigger re-provision. + const ws = entityConfig.workspace; + if (ws && WORKSPACE_TRANSITION_STATUSES.has(ws.status)) { + const transitionStartedAt = ws.provisionedAt ? new Date(ws.provisionedAt).getTime() : NaN; + const staleMs = Number.isFinite(transitionStartedAt) + ? Date.now() - transitionStartedAt + : Infinity; + if (staleMs > 5 * 60 * 1000) { + logger.warn(`Workspace for ${entityId} stuck in '${ws.status}' — marking as error`); + try { + await getEntityStore().upsertEntity({ ...entityConfig, workspace: { ...ws, status: 'error' } }); + } catch { /* best effort */ } + entityConfig = await loadEntityConfig(entityId); + } else if (waitForTransition) { + const lifecycle = workspaceTransitionLifecycle(ws.status); + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { + type: 'start', + phase: lifecycle.phase, + message: lifecycle.message, + }); + const transitionResult = await waitForWorkspaceTransition(entityId, options); + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { + type: 'finish', + phase: lifecycle.phase, + success: transitionResult.success, + error: transitionResult.error, + }); + if (!transitionResult.success) { + return transitionResult; + } + entityConfig = transitionResult.entityConfig; + } else { + return { success: false, error: `Workspace is ${ws.status}, please retry shortly` }; + } + } + + // Auto-provision if workspace not configured or in error state + if (!entityConfig.workspace || !entityConfig.workspace.url || entityConfig.workspace.status === 'error') { + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { type: 'start', phase: 'provision', message: 'Setting up workspace' }); + const provisionResult = await provisionWorkspace(entityId, entityConfig, options); + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { + type: 'finish', + phase: 'provision', + success: provisionResult.success, + error: provisionResult.error, + }); + if (!provisionResult.success) { + return { success: false, error: provisionResult.error }; + } + // Reload entity config after provisioning + entityConfig = await loadEntityConfig(entityId); + if (!entityConfig?.workspace?.url) { + return { success: false, error: 'Workspace provisioning completed but config not available' }; + } + // Seed activity timestamp so the idle reaper can see this workspace + // even if the provisioning request does not make a follow-up workspaceRequest(). + recordWorkspaceActivity(entityId); + if (provisionResult.checkpointRestored) { + entityConfig = await markWorkspaceCheckpointFresh(entityId, entityConfig); + } + } + + // Wake stopped workspace on demand — much faster than full re-provision + if (entityConfig.workspace.status === 'stopped' && entityConfig.workspace.containerId) { + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { type: 'start', phase: 'wake', message: 'Starting workspace' }); + const wakeResult = await wakeWorkspace(entityId, entityConfig); + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { + type: 'finish', + phase: 'wake', + success: wakeResult.success, + error: wakeResult.error, + }); + if (!wakeResult.success) { + return { success: false, error: wakeResult.error }; + } + entityConfig = await loadEntityConfig(entityId); + if (!entityConfig?.workspace?.url) { + return { success: false, error: 'Workspace wake completed but config not available' }; + } + recordWorkspaceActivity(entityId); + } + + // Reprovision if workspace image is outdated + const expectedVersion = config.get('workspaceImageVersion'); + if (expectedVersion && entityConfig.workspace.imageVersion && + entityConfig.workspace.imageVersion !== expectedVersion) { + logger.info(`Workspace for ${entityId} has stale image (${entityConfig.workspace.imageVersion} vs ${expectedVersion}) — reprovisioning`); + const reprovisionResult = await reprovisionStaleWorkspace(entityId, entityConfig, options, onWorkspaceLifecycle); + if (!reprovisionResult.success) { + return { success: false, error: reprovisionResult.error }; + } + entityConfig = reprovisionResult.entityConfig || await loadEntityConfig(entityId); + if (!entityConfig?.workspace?.url) { + return { success: false, error: 'Workspace re-provision completed but config not available' }; + } + recordWorkspaceActivity(entityId); + if (reprovisionResult.checkpointRestored) { + entityConfig = await markWorkspaceCheckpointFresh(entityId, entityConfig); + } + } + + return { success: true, entityConfig }; +} + +async function reprovisionStaleWorkspace(entityId, entityConfig, options, onWorkspaceLifecycle) { + if (reprovisionLocks.has(entityId)) { + await reprovisionLocks.get(entityId); + const nextEntityConfig = await loadEntityConfig(entityId, { fresh: true }); + const expectedVersion = config.get('workspaceImageVersion'); + if (!expectedVersion || nextEntityConfig?.workspace?.imageVersion === expectedVersion) { + return { success: true, entityConfig: nextEntityConfig }; + } + return { success: false, error: 'Concurrent workspace re-provision did not update the workspace image' }; + } + + const reprovisionPromise = (async () => { + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { type: 'start', phase: 'reprovision', message: 'Updating workspace' }); + const destroyResult = await destroyWorkspace(entityId, entityConfig, { onWorkspaceLifecycle }); + if (!destroyResult.success) { + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { + type: 'finish', + phase: 'reprovision', + success: false, + error: destroyResult.error, + }); + return { success: false, error: destroyResult.error }; + } + + // destroyWorkspace preserves a Blob checkpoint in entity config, so + // provisionWorkspace can restore it into a warm or fresh container. + const provisionResult = await provisionWorkspace(entityId, await loadEntityConfig(entityId), options); + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { + type: 'finish', + phase: 'reprovision', + success: provisionResult.success, + error: provisionResult.error, + }); + if (!provisionResult.success) { + return { success: false, error: provisionResult.error }; + } + return { + success: true, + entityConfig: await loadEntityConfig(entityId), + checkpointRestored: Boolean(provisionResult.checkpointRestored), + }; + })(); + + reprovisionLocks.set(entityId, reprovisionPromise); + try { + return await reprovisionPromise; + } finally { + reprovisionLocks.delete(entityId); + } +} + +async function restoreWorkspaceCheckpointToContainer(entityId, entityConfig, container, options = {}) { + const checkpointBlobPath = entityConfig?.workspace?.checkpointBlobPath; + if (!checkpointBlobPath) { + return { success: true, skipped: true, reason: 'no checkpoint blob' }; + } + + const timeoutMs = options.timeoutMs || 900000; + const onWorkspaceLifecycle = typeof options.onWorkspaceLifecycle === 'function' + ? options.onWorkspaceLifecycle + : null; + const checkpointUrl = await createWorkspaceCheckpointReadUrl(checkpointBlobPath, { + ...options, + entityId, + }); + const checkpointEncryption = buildWorkspaceCheckpointRestoreEncryption(entityConfig); + + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { type: 'start', phase: 'restore', message: 'Restoring workspace backup' }); + try { + const directResponse = await fetch(`${container.url}/restore-url`, { + method: 'POST', + headers: { + 'x-workspace-secret': container.bootstrapSecret, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + archiveUrl: checkpointUrl, + archivePath: WORKSPACE_CHECKPOINT_PATH, + ...(checkpointEncryption ? { encryption: checkpointEncryption } : {}), + }), + signal: AbortSignal.timeout(timeoutMs), + dispatcher: getLongFetchDispatcher(timeoutMs), + }); + const directBody = await directResponse.json().catch(() => ({})); + let restoreBody = directBody; + if (!directResponse.ok || directBody.error) { + const isOldWorkspaceImage = directResponse.status === 404; + if (!isOldWorkspaceImage) { + throw new Error(`/restore-url returned ${directResponse.status}: ${directBody.error || directResponse.statusText}`); + } + if (checkpointEncryption) { + throw new Error('encrypted checkpoint restore requires workspace image support for /restore-url encryption'); + } + + logger.info(`Workspace image for ${entityId} does not support /restore-url; falling back to direct shell download`); + const downloadScript = ` +const fs = require('node:fs'); +const path = require('node:path'); +const { Readable } = require('node:stream'); +const { pipeline } = require('node:stream/promises'); +(async () => { + const url = process.env.WORKSPACE_RESTORE_URL; + const out = process.env.WORKSPACE_RESTORE_PATH; + const tmp = out + '.download'; + const response = await fetch(url); + if (!response.ok || !response.body) throw new Error('download failed: ' + response.status + ' ' + response.statusText); + await fs.promises.mkdir(path.dirname(out), { recursive: true }); + await fs.promises.rm(tmp, { force: true }); + await pipeline(Readable.fromWeb(response.body), fs.createWriteStream(tmp)); + await fs.promises.rename(tmp, out); +})().catch((error) => { + console.error(error.message); + process.exit(1); +}); +`; + const downloadCommand = [ + `WORKSPACE_RESTORE_URL=${shellQuote(checkpointUrl)}`, + `WORKSPACE_RESTORE_PATH=${shellQuote(WORKSPACE_CHECKPOINT_PATH)}`, + 'node -e', + shellQuote(downloadScript), + ].join(' '); + + const shellResponse = await fetch(`${container.url}/shell`, { + method: 'POST', + headers: { + 'x-workspace-secret': container.bootstrapSecret, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ command: downloadCommand, timeout: timeoutMs }), + signal: AbortSignal.timeout(timeoutMs), + dispatcher: getLongFetchDispatcher(timeoutMs), + }); + const shellBody = await shellResponse.json().catch(() => ({})); + if (!shellResponse.ok || shellBody.error || shellBody.exitCode) { + throw new Error(`/shell checkpoint download returned ${shellResponse.status}: ${shellBody.error || shellBody.stderr || shellResponse.statusText}`); + } + + restoreBody = await restoreWorkspaceArchiveInContainer(container, WORKSPACE_CHECKPOINT_PATH, timeoutMs); + } + + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { type: 'finish', phase: 'restore', success: true }); + logger.info(`Restored workspace checkpoint for ${entityId} from ${checkpointBlobPath}`); + return { + success: true, + checkpointBlobPath, + sizeBytes: restoreBody.sizeBytes || entityConfig.workspace.checkpointSizeBytes || null, + }; + } catch (e) { + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { type: 'finish', phase: 'restore', success: false, error: e.message }); + throw e; + } +} + +async function restoreWorkspaceArchiveInContainer(container, archivePath, timeoutMs = 900000) { + const response = await fetch(`${container.url}/restore`, { + method: 'POST', + headers: { + 'x-workspace-secret': container.bootstrapSecret, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ archivePath }), + signal: AbortSignal.timeout(timeoutMs), + dispatcher: getLongFetchDispatcher(timeoutMs), + }); + const body = await response.json().catch(() => ({})); + if (!response.ok || body.error) { + throw new Error(`/restore returned ${response.status}: ${body.error || response.statusText}`); + } + return body; +} + +async function workspaceArchiveExistsInContainer(container, archivePath, timeoutMs = 30000) { + const response = await fetch(`${container.url}/shell`, { + method: 'POST', + headers: { + 'x-workspace-secret': container.bootstrapSecret, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + command: `test -f ${shellQuote(archivePath)}`, + timeout: timeoutMs, + }), + signal: AbortSignal.timeout(timeoutMs), + dispatcher: getLongFetchDispatcher(timeoutMs), + }); + const body = await response.json().catch(() => ({})); + if (!response.ok || (body.error && body.exitCode == null)) { + throw new Error(`/shell archive check returned ${response.status}: ${body.error || response.statusText}`); + } + return Number(body.exitCode || 0) === 0; +} + +async function restoreLegacyShareArchiveToContainer(entityId, entityConfig, container) { + const legacyShareName = container.legacyShareName; + if (!legacyShareName || entityConfig?.workspace?.checkpointBlobPath) { + return { success: true, skipped: true, reason: 'no legacy share archive' }; + } + + if (!await workspaceArchiveExistsInContainer(container, WORKSPACE_CHECKPOINT_PATH)) { + return { success: true, skipped: true, reason: 'legacy share has no checkpoint archive' }; + } + + const body = await restoreWorkspaceArchiveInContainer(container, WORKSPACE_CHECKPOINT_PATH); + logger.info(`Restored legacy Azure Files checkpoint for ${entityId} from share ${legacyShareName}`); + return { + success: true, + legacyShareName, + sizeBytes: body.sizeBytes || null, + }; +} + +async function checkpointLegacyShareAfterProvision(entityId, entityConfig) { + const legacyShareName = getLegacyShareName(entityConfig?.workspace); + if (legacyShareName) { + try { + const checkpointBlobPath = workspaceCheckpointBlobPath(entityId); + const checkpointUpload = await uploadWorkspaceArchiveFromLegacyShare( + entityId, + { legacyShareName }, + WORKSPACE_CHECKPOINT_PATH, + checkpointBlobPath, + ); + if (checkpointUpload?.blobPath) { + logger.info(`Copied legacy Azure Files checkpoint for ${entityId} directly to Blob after provision`); + const checkpoint = { + path: WORKSPACE_CHECKPOINT_PATH, + blobPath: checkpointUpload.blobPath, + previousBlobPath: null, + sizeBytes: checkpointUpload.sizeBytes || null, + sizeMB: checkpointUpload.sizeBytes + ? Math.round(checkpointUpload.sizeBytes / 1024 / 1024 * 100) / 100 + : null, + timestamp: new Date().toISOString(), + }; + return { + success: true, + checkpoint, + entityConfig: await persistWorkspaceCheckpoint(entityId, entityConfig, checkpoint, { + clearShareName: true, + legacyShareName, + }), + }; + } + } catch (e) { + logger.warn(`Direct legacy workspace checkpoint copy failed for ${entityId}: ${e.message}; falling back to live checkpoint`); + } + } + + const freshEntityConfig = await loadEntityConfig(entityId, { fresh: true }); + const checkpointResult = await checkpointAndPersistWorkspace(entityId, freshEntityConfig || entityConfig, { + clearShareName: true, + legacyShareName, + }); + if (!checkpointResult.success) { + logger.warn(`Legacy workspace share migration checkpoint failed for ${entityId}: ${checkpointResult.error}`); + return checkpointResult; + } + + return checkpointResult; +} + +async function setupWorkspaceContainerForEntity(entityId, entityConfig, container, backend, options = {}) { + try { + let restoreResult = await restoreWorkspaceCheckpointToContainer(entityId, entityConfig, container, options); + if (restoreResult.skipped) { + restoreResult = await restoreLegacyShareArchiveToContainer(entityId, entityConfig, container); + } + await reconfigureForEntity(entityId, entityConfig, container, backend, { + forceEnvRewrite: Boolean(restoreResult?.success && !restoreResult.skipped), + }); + return restoreResult; + } catch (e) { + try { + await backend.remove(container.containerId || container.containerName, container.containerName); + } catch { + // Best-effort cleanup. The setup error is the actionable failure. + } + throw e; + } +} + +/** + * Unified provision: claim from pool or create generic container, then reconfigure. + * + * Flow: + * 1. Restore from Blob checkpoint when present + * 2. claimContainer() — try Redis-backed warm pool (ACI only) + * 3. createGenericContainer() — create on demand, only mounting Azure Files for one-time legacy migration + * 4. reconfigureForEntity() — inject secrets, mount blob storage, rotate secret + */ +async function _doProvision(entityId, entityConfig) { + if (!isValidWorkspaceEntityId(entityId)) { + throw new Error('Workspace entityId is required'); + } + + const backend = await getBackend(); + entityConfig = await recoverExistingWorkspaceCheckpoint(entityId, entityConfig); + + // Azure Files is now legacy-only. If a Blob checkpoint exists, the entity is + // warm-pool eligible even when an old share name is still present. + const checkpointBlobPath = entityConfig?.workspace?.checkpointBlobPath || null; + const legacyShareName = getLegacyShareName(entityConfig?.workspace); + const needsLegacyShareMigration = Boolean(legacyShareName && !checkpointBlobPath); + + logger.info(`Provisioning workspace for entity ${entityId} [${backend.backendName} backend]${checkpointBlobPath ? ' (restoring Blob checkpoint)' : needsLegacyShareMigration ? ` (migrating legacy share: ${legacyShareName})` : ''}`); + + try { + // Update entity status to provisioning (preserve shareName so it's not lost) + const entityStore = getEntityStore(); + await entityStore.upsertEntity({ + ...entityConfig, + workspace: { + ...(entityConfig.workspace || {}), + status: 'provisioning', + provisionedAt: new Date(), + }, + }); + + // Step 1: Try to claim a pre-provisioned container from the warm pool. + // Legacy share-only entities need one generic ACI with the old share + // mounted so they can self-migrate to Blob checkpoints first. + let container = null; + if (!needsLegacyShareMigration && backend.backendName === 'aci' && config.get('warmPoolSize') > 0) { + const claimed = await claimContainer(entityId); + if (claimed.success) { + container = { + containerName: claimed.containerName, + url: claimed.url, + bootstrapSecret: claimed.bootstrapSecret, + containerId: claimed.containerId, + claimedFromPool: true, + imageVersion: claimed.imageVersion || null, + }; + logger.info(`[WarmPool] Claimed ${container.containerName} for entity ${entityId}`); + } else { + logger.info(`[WarmPool] No pool container available for ${entityId}, creating on demand`); + } + } + + // Step 2: If no pool container, create a generic one. + if (!container) { + container = await createGenericContainer(entityId, backend, { + shareName: needsLegacyShareMigration ? legacyShareName : null, + mountAzureFiles: needsLegacyShareMigration, + }); + } + + let migratedLegacyShare = false; + let setupResult = null; + // Step 3/4: Restore any Blob checkpoint before entity env/secrets are + // written, so restored .env files cannot win over current secrets. If + // a pool-claimed container is dead, fall back to creating a fresh one. + try { + setupResult = await setupWorkspaceContainerForEntity(entityId, entityConfig, container, backend); + } catch (provisionErr) { + if (checkpointBlobPath && legacyShareName && backend.backendName === 'aci') { + logger.warn(`Blob checkpoint restore failed for ${entityId}; falling back to legacy share migration: ${provisionErr.message}`); + const legacyEntityConfig = { + ...entityConfig, + workspace: { + ...(entityConfig.workspace || {}), + checkpointBlobPath: null, + }, + }; + container = await createGenericContainer(entityId, backend, { + shareName: legacyShareName, + mountAzureFiles: true, + }); + setupResult = await setupWorkspaceContainerForEntity(entityId, legacyEntityConfig, container, backend); + migratedLegacyShare = true; + } else if (container.claimedFromPool) { + logger.warn(`[WarmPool] Claimed container ${container.containerName} failed setup — falling back to fresh container: ${provisionErr.message}`); + container = await createGenericContainer(entityId, backend); + setupResult = await setupWorkspaceContainerForEntity(entityId, entityConfig, container, backend); + } else { + throw provisionErr; + } + } + + if (needsLegacyShareMigration || migratedLegacyShare) { + await checkpointLegacyShareAfterProvision(entityId, entityConfig); + } + + logger.info(`Workspace provisioned for entity ${entityId}: ${container.url}`); + return { success: true, checkpointRestored: Boolean(setupResult?.success && !setupResult.skipped) }; + } catch (e) { + logger.error(`Failed to provision workspace for entity ${entityId}: ${e.message}`); + + // Mark as error + try { + const entityStore = getEntityStore(); + await entityStore.upsertEntity({ + ...entityConfig, + workspace: { + ...(entityConfig.workspace || {}), + status: 'error', + }, + }); + } catch { + // Best effort + } + + return { success: false, error: `Provisioning failed: ${e.message}` }; + } +} + +/** + * Create a generic container with minimal setup (no entity secrets, no blob mount). + * Used when the warm pool is empty or disabled. + * + * @param {string} entityId - Entity UUID (used for container naming) + * @param {Object} backend - Container backend instance + * @param {Object} [options] + * @param {string} [options.shareName] - Legacy Azure Files share to mount for one-time migration + * @param {boolean} [options.mountAzureFiles] - Mount shareName as /persist for legacy migration + * @returns {Promise<{containerName: string, shareName: string, url: string, bootstrapSecret: string, containerId: string, claimedFromPool: boolean}>} + */ +async function createGenericContainer(entityId, backend, options = {}) { + if (!isValidWorkspaceEntityId(entityId)) { + throw new Error('Workspace entityId is required'); + } + + const baseContainerName = workspaceContainerNameForEntity(entityId); + const requestedShareName = options.shareName || null; + const shareName = backend.backendName === 'aci' + ? (options.mountAzureFiles ? requestedShareName : null) + : (requestedShareName || baseContainerName); + const mountAzureFiles = Boolean(options.mountAzureFiles && shareName); + const bootstrapSecret = crypto.randomBytes(32).toString('hex'); + const image = resolveWorkspaceImage(); + const cpus = parseFloat(config.get('workspaceCpus')); + const memory = config.get('workspaceMemory'); + const diskSize = config.get('workspaceDiskSize'); + const memoryMB = parseMemoryToMB(memory); + + const env = [ + `WORKSPACE_SECRET=${bootstrapSecret}`, + `PORT=3100`, + ]; + + let lastError = null; + for (let attempt = 0; attempt < 3; attempt++) { + const containerName = buildRuntimeContainerName(baseContainerName, attempt); + logger.info(`Creating generic container ${containerName} [${backend.backendName}]${mountAzureFiles ? ` (legacy share: ${shareName})` : ''}`); + + let created; + try { + created = await backend.createAndStart({ + containerName, + image, + env, + cpus, + memoryMB, + diskSize, + shareName, + mountAzureFiles, + tags: { + workspaceRole: 'entity', + entityId, + createdAt: new Date().toISOString(), + ...(mountAzureFiles ? { legacyShareName: shareName } : {}), + }, + }); + } catch (e) { + lastError = e; + if (attempt < 2 && isCrossRegionContainerNameConflict(e)) { + logger.warn(`Container name ${containerName} already exists in another Azure location; retrying with a unique runtime name`); + continue; + } + throw e; + } + + const healthOk = await waitForHealth(created.url, backend.healthTimeoutMs); + if (!healthOk) { + throw new Error('Container failed to become healthy'); + } + + return { + containerName, + shareName, + legacyShareName: mountAzureFiles ? shareName : null, + url: created.url, + bootstrapSecret, + containerId: created.containerId, + claimedFromPool: false, + imageVersion: config.get('workspaceImageVersion') || null, + }; + } + + throw lastError || new Error('Failed to create workspace container'); +} + +function buildRuntimeContainerName(baseContainerName, attempt) { + if (attempt === 0) return baseContainerName; + return `${baseContainerName}-${crypto.randomUUID().replace(/-/g, '').slice(0, 6)}`; +} + +function isCrossRegionContainerNameConflict(error) { + const message = error?.message || ''; + return message.includes('already exists in location') + && message.includes('same name cannot be created in location'); +} + +/** + * Reconfigure a container for a specific entity. + * Rotates the secret, injects entity secrets, and mounts blob storage. + * Works for both pool-claimed and freshly-created containers. + * + * @param {string} entityId - Entity UUID + * @param {Object} entityConfig - Current entity config + * @param {Object} container - Container info from claimContainer or createGenericContainer + * @param {Object} backend - Container backend instance + */ +async function reconfigureForEntity(entityId, entityConfig, container, backend, options = {}) { + const { containerName, shareName, legacyShareName, url, bootstrapSecret, containerId, claimedFromPool } = container; + const { destroyOnFailure = true, forceEnvRewrite = false } = options; + const newSecret = crypto.randomBytes(32).toString('hex'); + + try { + // Build reconfigure payload + const reconfigPayload = { secret: newSecret }; + + // Decrypt entity secrets for env injection + const plainSecrets = {}; + if (entityConfig.secrets) { + const systemKey = config.get('redisEncryptionKey'); + for (const [key, encVal] of Object.entries(entityConfig.secrets)) { + const val = decrypt(encVal, systemKey); + if (val) { + plainSecrets[key] = val; + } + } + } + if (Object.keys(plainSecrets).length > 0 || forceEnvRewrite) { + reconfigPayload.env = plainSecrets; + } + + // Add blob mount if applicable (ACI backend, private entity with single user) + if (backend.backendName === 'aci') { + const blobMount = await buildBlobMountPayload(entityConfig); + if (blobMount) { + reconfigPayload.blobMount = blobMount; + + // Also add blob env vars so the workspace knows about them + if (!reconfigPayload.env) reconfigPayload.env = {}; + reconfigPayload.env.AZURE_STORAGE_ACCOUNT_NAME = blobMount.accountName; + reconfigPayload.env.AZURE_BLOB_SAS_TOKEN = blobMount.sasToken; + reconfigPayload.env.AZURE_BLOB_CONTAINER = blobMount.containerName; + } + } + + // Call /reconfigure using the bootstrap secret + const response = await fetch(`${url}/reconfigure`, { + method: 'POST', + headers: { + 'x-workspace-secret': bootstrapSecret, + 'Content-Type': 'application/json', + }, + body: JSON.stringify(reconfigPayload), + signal: AbortSignal.timeout(30000), + }); + + if (!response.ok) { + const errBody = await response.json().catch(() => ({})); + throw new Error(`/reconfigure returned ${response.status}: ${errBody.error || response.statusText}`); + } + + // Update entity config in MongoDB + // Store bootstrapSecret so wakeWorkspace can re-authenticate after + // a container restart (the container reverts to its env-var secret). + const entityStore = getEntityStore(); + const previousWorkspace = entityConfig.workspace || {}; + const nextWorkspace = { + url, + secret: newSecret, + bootstrapSecret, + containerId: containerId || containerName, + status: 'running', + provisionedAt: new Date(), + claimedFromPool, + imageVersion: container.imageVersion || config.get('workspaceImageVersion') || null, + ...getWorkspaceCheckpointMetadata(previousWorkspace), + }; + if (shareName) { + nextWorkspace.shareName = shareName; + } + const retainedLegacyShareName = legacyShareName || previousWorkspace.legacyShareName || previousWorkspace.shareName || null; + if (retainedLegacyShareName) { + nextWorkspace.legacyShareName = retainedLegacyShareName; + } + + await entityStore.upsertEntity({ + ...entityConfig, + workspace: nextWorkspace, + }); + } catch (e) { + if (destroyOnFailure) { + // Remove the container on failure — but NEVER destroy the volume. + // The share may be pre-existing with user data (e.g. during reprovision + // or auth recovery). Only destroyWorkspace({ destroyVolume: true }) should + // delete shares, as an explicit user action. + try { + await backend.remove(containerId || containerName, containerName); + } catch { + // Best-effort cleanup + } + } + + throw e; + } +} + +/** + * Build blob mount payload for /reconfigure. + * Returns null if blob mount is not applicable (no storage config, or not a single-user entity). + * + * @param {Object} entityConfig - Entity config with assocUserIds + * @returns {Promise<{accountName: string, sasToken: string, containerName: string}|null>} + */ +async function buildBlobMountPayload(entityConfig) { + const storageAccountName = config.get('azureStorageAccountName'); + const storageAccountKey = config.get('azureStorageAccountKey'); + const blobContainerName = config.get('azureBlobContainerName'); + const assocUserIds = Array.isArray(entityConfig.assocUserIds) ? entityConfig.assocUserIds : []; + const ownerUserId = assocUserIds.length === 1 ? assocUserIds[0] : null; + + if (!storageAccountName || !storageAccountKey || !blobContainerName || !ownerUserId) { + return null; + } + + const userContainer = getUserContainerName(blobContainerName, ownerUserId); + await ensureContainer(storageAccountName, storageAccountKey, userContainer); + const sasToken = generateContainerSASToken(storageAccountName, storageAccountKey, userContainer); + + return { + accountName: storageAccountName, + sasToken, + containerName: userContainer, + }; +} + +/** + * Stop and remove a workspace container. + * When destroyVolume is false (default), a Blob checkpoint is preserved in the + * entity config so the next provision can restore it into a warm container. + */ +export async function destroyWorkspace(entityId, entityConfig, options = {}) { + const { + destroyVolume: shouldDestroyVolume = false, + skipCheckpoint = false, + lastActivityAt = null, + timeoutMs, + } = options; + const onWorkspaceLifecycle = typeof options.onWorkspaceLifecycle === 'function' + ? options.onWorkspaceLifecycle + : null; + let workspace = entityConfig?.workspace; + // Use stored containerId — pool-claimed containers have names like + // workspace-pool-{shortId}, not workspace-{entityId}. + const containerName = workspace?.containerId || `workspace-${entityId}`; + const legacyShareName = getLegacyShareName(workspace); + + try { + const backend = await getBackend(); + let checkpointResult = null; + let effectiveLastActivityAt = lastActivityAt; + if (!effectiveLastActivityAt && !shouldDestroyVolume && !skipCheckpoint && backend.backendName === 'aci' && workspace?.checkpointBlobPath) { + const latestActivity = await readLatestWorkspaceActivityTimestamp(entityId, 0); + if (latestActivity.ok) { + effectiveLastActivityAt = latestActivity.timestamp; + } + } + let checkpointAlreadyFresh = Boolean( + effectiveLastActivityAt && isWorkspaceCheckpointFresh(workspace, effectiveLastActivityAt) + ); + + if (!shouldDestroyVolume && !skipCheckpoint && backend.backendName === 'aci' && workspace?.url) { + if (!checkpointAlreadyFresh && workspace?.checkpointBlobPath) { + try { + const recoveredEntityConfig = await recoverFreshWorkspaceCheckpointFromBlob(entityId, entityConfig, effectiveLastActivityAt); + if (recoveredEntityConfig?.workspace) { + entityConfig = recoveredEntityConfig; + workspace = recoveredEntityConfig.workspace; + checkpointAlreadyFresh = true; + logger.info(`Recovered fresh workspace checkpoint metadata for ${entityId} from Blob before destroy`); + } + } catch (e) { + logger.warn(`Could not recover workspace checkpoint metadata from Blob for ${entityId}: ${e.message}`); + } + } + + if (checkpointAlreadyFresh) { + logger.info(`Skipped workspace checkpoint for ${entityId} before destroy: checkpoint is already fresh`); + } else { + checkpointResult = await checkpointWorkspace(entityId, entityConfig, { timeoutMs, onWorkspaceLifecycle }); + if (!checkpointResult.success) { + logger.warn(`Skipping destroy for ${entityId}; workspace checkpoint failed: ${checkpointResult.error}`); + return { success: false, error: checkpointResult.error }; + } + if (checkpointResult.skipped) { + logger.info(`Skipped workspace checkpoint for ${entityId} before destroy: ${checkpointResult.reason}`); + } else { + logger.info(`Checkpointed workspace for ${entityId} before destroy (${checkpointResult.checkpoint?.sizeMB ?? '?'} MB)`); + } + } + + if (effectiveLastActivityAt && !checkpointResult?.skipped) { + const latestActivity = await readLatestWorkspaceActivityTimestamp(entityId, effectiveLastActivityAt); + if (!latestActivity.ok) { + logger.warn(`Skipping destroy for ${entityId}; latest workspace activity could not be verified`); + return { success: false, error: 'Latest workspace activity could not be verified' }; + } + const checkpointedAt = checkpointResult?.checkpoint + ? parseTimestampMs(checkpointResult.checkpoint.timestamp) + : parseTimestampMs(workspace.checkpointedAt); + if (!checkpointedAt || checkpointedAt < latestActivity.timestamp) { + logger.warn(`Skipping destroy for ${entityId}; workspace changed after the latest checkpoint`); + return { success: false, error: 'Workspace checkpoint is stale' }; + } + } + } + + if (shouldDestroyVolume) { + const checkpointDeleteResult = await deleteWorkspaceCheckpointBlobs(entityId, workspace); + if (checkpointDeleteResult.attempted > 0) { + logger.info(`Deleted ${checkpointDeleteResult.deleted}/${checkpointDeleteResult.attempted} workspace checkpoint blob(s) for ${entityId}`); + } + } + + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { type: 'start', phase: 'destroy', message: 'Destroying workspace container' }); + try { + await backend.remove(containerName, containerName); + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { type: 'finish', phase: 'destroy', success: true }); + } catch (e) { + await emitWorkspaceLifecycle(onWorkspaceLifecycle, { type: 'finish', phase: 'destroy', success: false, error: e.message }); + throw e; + } + + if (shouldDestroyVolume && legacyShareName) { + await backend.destroyVolume(legacyShareName); + } + + // Update entity workspace config + const entityStore = getEntityStore(); + if (shouldDestroyVolume) { + // Volume gone — clear workspace entirely so next provision starts fresh + await entityStore.upsertEntity({ + ...entityConfig, + workspace: null, + }); + } else { + const checkpoint = checkpointResult?.checkpoint || null; + const existingCheckpointFields = getWorkspaceCheckpointMetadata(workspace); + const checkpointEntityConfig = checkpointResult?.entityConfig || checkpoint?.entityConfig || entityConfig; + const checkpointFields = checkpoint + ? workspaceCheckpointFields(checkpoint) + : (existingCheckpointFields.checkpointBlobPath ? existingCheckpointFields : null); + const stoppedWorkspace = checkpointFields + ? { + ...checkpointFields, + ...(checkpointEntityConfig?.workspace?.checkpointEncryptionKey + ? { checkpointEncryptionKey: checkpointEntityConfig.workspace.checkpointEncryptionKey } + : {}), + ...(legacyShareName ? { legacyShareName } : {}), + } + : { + // Legacy fallback for old workspace images that stored + // directly on Azure Files and could not produce a tarball. + ...(legacyShareName ? { shareName: legacyShareName } : {}), + }; + + await entityStore.upsertEntity({ + ...checkpointEntityConfig, + workspace: stoppedWorkspace, + }); + } + + logger.info(`Workspace destroyed for entity ${entityId}${shouldDestroyVolume ? ' (volume removed)' : ' (checkpoint preserved)'}`); + lastActivity.delete(entityId); + await removeWorkspaceActivityFromRedis(entityId); + return { success: true, message: `Workspace destroyed${shouldDestroyVolume ? ' (volume removed)' : ' (volume preserved)'}` }; + } catch (e) { + logger.error(`Failed to destroy workspace for entity ${entityId}: ${e.message}`); + return { success: false, error: `Destroy failed: ${e.message}` }; + } +} + +/** + * Stop a workspace container without destroying it. + * Container, volume, port bindings, and URL are all preserved for fast restart. + * + * @param {string} entityId - Entity UUID + * @param {Object} entityConfig - Current entity config + * @returns {Promise<{success: boolean, error?: string}>} + */ +export async function stopWorkspace(entityId, entityConfig) { + const workspace = entityConfig?.workspace; + if (!workspace?.containerId) { + return { success: false, error: 'No workspace container to stop' }; + } + + try { + const backend = await getBackend(); + const containerName = workspace.containerId; + await backend.stop(containerName, containerName); + } catch (e) { + logger.error(`Failed to stop workspace for entity ${entityId}: ${e.message}`); + return { success: false, error: `Stop failed: ${e.message}` }; + } + + try { + const entityStore = getEntityStore(); + await entityStore.upsertEntity({ + ...entityConfig, + workspace: { + ...workspace, + status: 'stopped', + stoppedAt: Date.now(), + }, + }); + } catch (e) { + logger.error(`Failed to update entity after stopping workspace: ${e.message}`); + return { success: false, error: `Failed to update entity: ${e.message}` }; + } + + logger.info(`Workspace stopped for entity ${entityId}`); + lastActivity.delete(entityId); + await removeWorkspaceActivityFromRedis(entityId); + return { success: true }; +} + +/** + * Wake a stopped workspace by starting its existing container. + * Much faster than full provisioning — no image pull, no container create. + * + * @param {string} entityId - Entity UUID + * @param {Object} entityConfig - Current entity config (must have workspace.containerId) + * @returns {Promise<{success: boolean, error?: string}>} + */ +async function wakeWorkspace(entityId, entityConfig) { + const workspace = entityConfig.workspace; + const backend = await getBackend(); + const containerName = workspace.containerId; + logger.info(`Waking stopped workspace for entity ${entityId}`); + + try { + const entityStore = getEntityStore(); + await entityStore.upsertEntity({ + ...entityConfig, + workspace: { ...workspace, status: 'starting' }, + }); + + const startResult = await backend.start(workspace.containerId, containerName); + const startedWorkspace = startResult?.url + ? { ...workspace, url: startResult.url } + : workspace; + + const healthOk = await waitForHealth(startedWorkspace.url, backend.wakeHealthTimeoutMs); + + if (!healthOk) { + // Container is dead — fall back to full re-provision + logger.warn(`Workspace for ${entityId} not healthy after wake — re-provisioning`); + return await provisionWorkspace(entityId, entityConfig); + } + + // Refresh from MongoDB before reconfigure/sync so wake applies the + // latest persisted secrets even if this process has a stale cache. + const freshEntityConfig = + (await loadEntityConfig(entityId, { fresh: true })) || entityConfig; + + // Real stop/start restarts the workspace process, reverting the + // in-memory secret to the bootstrap WORKSPACE_SECRET env var and + // dropping runtime mounts. Reconfigure immediately after wake so the + // next workspace operation does not have to discover this via 401. + if (workspace.bootstrapSecret) { + await reconfigureForEntity(entityId, freshEntityConfig, { + containerName, + shareName: workspace.shareName || null, + legacyShareName: workspace.legacyShareName || null, + url: startedWorkspace.url, + bootstrapSecret: workspace.bootstrapSecret, + containerId: workspace.containerId, + claimedFromPool: workspace.claimedFromPool || false, + }, backend, { destroyOnFailure: false, forceEnvRewrite: true }); + } else { + await entityStore.upsertEntity({ + ...entityConfig, + workspace: { ...startedWorkspace, status: 'running', stoppedAt: undefined }, + }); + + // Sync entity secrets to workspace — they may have been updated + // while the workspace was stopped. Always rewrite the file, even + // when the user secret set is now empty, so deletions take effect. + try { + const systemKey = config.get('redisEncryptionKey'); + const plainSecrets = {}; + for (const [key, encVal] of Object.entries(freshEntityConfig.secrets || {})) { + const val = decrypt(encVal, systemKey); + if (val === null || val === undefined) continue; + plainSecrets[key] = val; + } + + const syncResult = await syncSecretsToWorkspace( + entityId, + plainSecrets, + ); + if (!syncResult?.success) { + logger.warn( + `Failed to sync secrets on ACI wake for ${entityId}: ${syncResult.error || 'unknown error'}`, + ); + } + } catch (syncErr) { + logger.warn(`Failed to sync secrets on ACI wake for ${entityId}: ${syncErr.message}`); + } + } + + logger.info(`Workspace woken for entity ${entityId}`); + return { success: true }; + } catch (e) { + logger.error(`Failed to wake workspace for entity ${entityId}: ${e.message}`); + + // Fall back to full re-provision + logger.warn(`Falling back to re-provision for ${entityId}`); + return await provisionWorkspace(entityId, entityConfig); + } +} + +/** + * Poll a workspace's /health endpoint until it responds OK. + */ +async function waitForHealth(baseUrl, maxWaitMs) { + const start = Date.now(); + const interval = 1000; + + while (Date.now() - start < maxWaitMs) { + try { + const res = await fetch(`${baseUrl}/health`, { + signal: AbortSignal.timeout(3000), + }); + if (res.ok) return true; + } catch { + // Not ready yet + } + await new Promise(r => setTimeout(r, interval)); + } + + return false; +} + +/** + * Write entity secrets as a .env file to the workspace container. + * Used both at provision time and when secrets are updated via API. + * + * @param {string} entityId - Entity UUID + * @param {Object} secrets - { KEY: "plaintext_value", ... } + * @returns {Promise} { success: boolean, error?: string } + */ +export async function syncSecretsToWorkspace(entityId, secrets) { + const SAFE_KEY = /^[A-Za-z_][A-Za-z0-9_]*$/; + + // Start with user secrets + const allEnvVars = { ...(secrets || {}) }; + + // Also include blob storage env vars so we don't clobber them — these are + // written by reconfigureForEntity at provision time but would be lost if + // we overwrote .env with only user secrets. + try { + const entityConfig = await loadEntityConfig(entityId); + if (entityConfig) { + const blobMount = await buildBlobMountPayload(entityConfig); + if (blobMount) { + allEnvVars.AZURE_STORAGE_ACCOUNT_NAME = blobMount.accountName; + allEnvVars.AZURE_BLOB_SAS_TOKEN = blobMount.sasToken; + allEnvVars.AZURE_BLOB_CONTAINER = blobMount.containerName; + } + } + } catch (e) { + logger.warn(`syncSecretsToWorkspace: failed to load blob mount info: ${e.message}`); + } + + const envLines = Object.entries(allEnvVars) + .filter(([k]) => SAFE_KEY.test(k)) + .map(([k, v]) => { + const escaped = String(v).replace(/'/g, "'\\''"); + return `export ${k}='${escaped}'`; + }); + const envContent = envLines.length > 0 ? envLines.join('\n') + '\n' : ''; + const b64 = Buffer.from(envContent).toString('base64'); + const writeResult = await workspaceRequest(entityId, '/write', { + path: '/workspace/.env', + content: b64, + encoding: 'base64', + createDirs: false, + }, { timeoutMs: 10000 }); + if (!writeResult?.success) { + return { + success: false, + error: writeResult?.error || 'Failed to write workspace secrets', + }; + } + + // Ensure .bashrc sources .env so secrets are available in every shell + const sourceLine = '[ -f /workspace/.env ] && . /workspace/.env'; + const bashrcResult = await workspaceRequest(entityId, '/shell', { + command: `grep -qF '${sourceLine}' ~/.bashrc 2>/dev/null || echo '${sourceLine}' >> ~/.bashrc`, + }, { timeoutMs: 10000 }); + if (!bashrcResult?.success) { + return { + success: false, + error: + bashrcResult?.error || + 'Failed to register workspace secrets in shell startup', + }; + } + + // Source it now for any currently running shells + const sourceResult = await workspaceRequest(entityId, '/shell', { + command: '. /workspace/.env', + }, { timeoutMs: 10000 }); + if (!sourceResult?.success) { + return { + success: false, + error: + sourceResult?.error || + 'Failed to source workspace secrets in the running shell', + }; + } + + return { success: true }; +} + +/** + * Stream-download a file from an entity's workspace container to a local path. + * Uses the GET /download streaming endpoint instead of base64-in-JSON. + * + * @param {string} entityId - Entity UUID + * @param {string} remotePath - Path inside the container + * @param {string} localPath - Destination path on Cortex host + * @returns {Promise<{success: boolean, bytesWritten?: number, error?: string}>} + */ +export async function workspaceDownloadToFile(entityId, remotePath, localPath) { + const workspaceResult = await ensureWorkspaceReady(entityId); + if (!workspaceResult.success) { + return workspaceResult; + } + + const { entityConfig } = workspaceResult; + const { url, secret } = entityConfig.workspace; + const endpoint = `${url}/download?path=${encodeURIComponent(remotePath)}`; + recordWorkspaceActivity(entityId); + + const response = await fetch(endpoint, { + headers: { 'x-workspace-secret': secret }, + signal: AbortSignal.timeout(300000), // 5-minute timeout + }); + + if (!response.ok) { + let errMsg; + try { errMsg = (await response.json()).error; } catch { errMsg = response.statusText; } + return { success: false, error: errMsg || `Download failed: ${response.status}` }; + } + + const nodeStream = Readable.fromWeb(response.body); + const ws = fs.createWriteStream(localPath); + await pipeline(nodeStream, ws); + + const stat = fs.statSync(localPath); + recordWorkspaceActivity(entityId); + return { success: true, bytesWritten: stat.size }; +} + +/** + * Stream-upload a local file to an entity's workspace container. + * Uses the POST /upload streaming endpoint instead of base64-in-JSON. + * + * @param {string} entityId - Entity UUID + * @param {string} localPath - Source path on Cortex host + * @param {string} remotePath - Destination path inside the container + * @returns {Promise<{success: boolean, bytesWritten?: number, error?: string}>} + */ +export async function workspaceUploadFile(entityId, localPath, remotePath) { + const workspaceResult = await ensureWorkspaceReady(entityId); + if (!workspaceResult.success) { + return workspaceResult; + } + + const { entityConfig } = workspaceResult; + const { url, secret } = entityConfig.workspace; + const endpoint = `${url}/upload?path=${encodeURIComponent(remotePath)}`; + recordWorkspaceActivity(entityId); + + const fileStream = fs.createReadStream(localPath); + const response = await fetch(endpoint, { + method: 'POST', + headers: { + 'x-workspace-secret': secret, + 'Content-Type': 'application/octet-stream', + }, + body: Readable.toWeb(fileStream), + duplex: 'half', + signal: AbortSignal.timeout(300000), // 5-minute timeout + }); + + if (!response.ok) { + let errMsg; + try { errMsg = (await response.json()).error; } catch { errMsg = response.statusText; } + return { success: false, error: errMsg || `Upload failed: ${response.status}` }; + } + + const result = await response.json(); + recordWorkspaceActivity(entityId); + return { success: true, bytesWritten: result.bytesWritten }; +} + +// --------------------------------------------------------------------------- +// Idle workspace reaper — runs every 5 minutes at module scope +// --------------------------------------------------------------------------- +const REAPER_INTERVAL_MS = 5 * 60 * 1000; + +function activityAgeMs(now, timestamp) { + return timestamp ? now - timestamp : null; +} + +function workspaceIdleCheckpointMs(idleTimeoutMs) { + const configured = Number(config.get('workspaceIdleCheckpointMs')); + if (!Number.isFinite(configured) || configured <= 0) return idleTimeoutMs; + return Math.min(configured, idleTimeoutMs); +} + +function parseTimestampMs(value) { + if (!value) return 0; + const parsed = new Date(value).getTime(); + if (Number.isFinite(parsed)) return parsed; + + const checkpointFilenameTimestamp = String(value).match( + /^(\d{4}-\d{2}-\d{2})T(\d{2})-(\d{2})-(\d{2})(?:-(\d{3}))?Z$/, + ); + if (!checkpointFilenameTimestamp) return 0; + + const [, date, hour, minute, second, millis = '000'] = checkpointFilenameTimestamp; + const normalized = `${date}T${hour}:${minute}:${second}.${millis}Z`; + const normalizedParsed = new Date(normalized).getTime(); + return Number.isFinite(normalizedParsed) ? normalizedParsed : 0; +} + +function isWorkspaceCheckpointFresh(workspace, lastActivityAt) { + if (!workspace?.checkpointBlobPath || !lastActivityAt) return false; + return parseTimestampMs(workspace.checkpointedAt) >= lastActivityAt; +} + +function serializeWorkspaceForReaperLog(workspace) { + if (!workspace) return null; + + return { + containerId: workspace.containerId || null, + shareName: workspace.shareName || null, + legacyShareName: workspace.legacyShareName || null, + checkpointBlobPath: workspace.checkpointBlobPath || null, + status: workspace.status || null, + url: workspace.url || null, + hasSecret: Boolean(workspace.secret), + hasBootstrapSecret: Boolean(workspace.bootstrapSecret), + claimedFromPool: Boolean(workspace.claimedFromPool), + }; +} + +function serializeJobsCheckForReaperLog(jobsCheck) { + if (!jobsCheck) return { attempted: false }; + + return { + attempted: Boolean(jobsCheck.attempted), + ok: jobsCheck.ok ?? null, + hasRunningJobs: Boolean(jobsCheck.hasRunningJobs), + reason: jobsCheck.reason || null, + httpStatus: jobsCheck.httpStatus ?? null, + responseType: jobsCheck.responseType || null, + jobsContainer: jobsCheck.jobsContainer || null, + jobCount: Array.isArray(jobsCheck.jobs) ? jobsCheck.jobs.length : null, + runningJobCount: jobsCheck.runningJobCount ?? null, + jobStatusCounts: jobsCheck.jobStatusCounts || null, + }; +} + +function summarizeWorkspaceJobStatuses(jobs) { + const counts = {}; + + for (const job of jobs) { + const rawStatus = typeof job?.status === 'string' ? job.status : 'missing'; + const status = ['running', 'completed', 'failed', 'stopped', 'killed', 'pending'].includes(rawStatus) + ? rawStatus + : 'other'; + counts[status] = (counts[status] || 0) + 1; + } + + return counts; +} + +function logWorkspaceReaperDecision(decision) { + logger.info(`[WorkspaceReaper] ${JSON.stringify(decision)}`); +} + +async function checkpointIdleWorkspaceIfNeeded(entityId, entityConfig, lastActivityAt, decisionLog) { + const checkpointedAt = parseTimestampMs(entityConfig.workspace?.checkpointedAt); + const checkpointFresh = isWorkspaceCheckpointFresh(entityConfig.workspace, lastActivityAt); + decisionLog.checkpointedAt = checkpointedAt || null; + decisionLog.checkpointFresh = checkpointFresh; + if (checkpointFresh) { + return { success: true, entityConfig, checkpointed: false, fresh: true }; + } + + const checkpointResult = await checkpointAndPersistWorkspace(entityId, entityConfig); + decisionLog.checkpointResult = { + success: Boolean(checkpointResult.success), + skipped: Boolean(checkpointResult.skipped), + error: checkpointResult.error || null, + sizeMB: checkpointResult.checkpoint?.sizeMB ?? null, + }; + if (!checkpointResult.success) { + return { + success: false, + error: checkpointResult.error, + entityConfig, + checkpointed: false, + }; + } + if (checkpointResult.skipped) { + return { + success: true, + entityConfig, + checkpointed: false, + skipped: true, + }; + } + + return { + success: true, + entityConfig: checkpointResult.entityConfig || entityConfig, + checkpointed: true, + }; +} + +async function reapIdleWorkspaces() { + const idleTimeoutMs = config.get('workspaceIdleTimeoutMs'); + if (!idleTimeoutMs) return; // disabled when set to 0 + + const now = Date.now(); + const redis = await getActivityRedisClient(); + const backend = await getBackend(); + const checkpointIdleMs = backend.backendName === 'aci' + ? workspaceIdleCheckpointMs(idleTimeoutMs) + : idleTimeoutMs; + const maintenanceIdleMs = Math.min(idleTimeoutMs, checkpointIdleMs); + if (backend.backendName === 'aci') { + const inventoryReaped = await reapAciWorkspaceInventory({ backend, redis, now, idleTimeoutMs, checkpointIdleMs }); + if (inventoryReaped) return; + } + + const candidateEntityIds = await getWorkspaceReaperCandidates(redis, now, maintenanceIdleMs); + + for (const entityId of candidateEntityIds) { + const localLastTs = lastActivity.get(entityId) || 0; + const decisionLog = { + entityId, + now, + idleTimeoutMs, + checkpointIdleMs, + localLastActivity: localLastTs || null, + localIdleMs: activityAgeMs(now, localLastTs), + redisConfigured: isActivityRedisConfigured(), + lockAcquired: null, + workspace: null, + redisLastActivity: null, + redisIdleMs: null, + effectiveLastActivity: localLastTs || null, + effectiveIdleMs: activityAgeMs(now, localLastTs), + latestLocalActivity: null, + latestRedisActivity: null, + latestEffectiveActivity: null, + latestEffectiveIdleMs: null, + jobsCheck: { attempted: false }, + action: null, + reason: null, + reapMode: null, + reapResult: null, + checkpointedAt: null, + checkpointFresh: null, + checkpointResult: null, + }; + + try { + let entityConfig = await loadEntityConfig(entityId); + decisionLog.workspace = serializeWorkspaceForReaperLog(entityConfig?.workspace); + if (!entityConfig?.workspace || entityConfig.workspace.status !== 'running') { + lastActivity.delete(entityId); + await removeWorkspaceActivityFromRedis(entityId); + decisionLog.action = 'delete-local-activity'; + decisionLog.reason = entityConfig?.workspace + ? 'workspace-not-running' + : 'workspace-missing'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + const { acquired } = await acquireWorkspaceReaperLock(entityId); + decisionLog.lockAcquired = acquired; + if (!acquired) { + decisionLog.action = 'skip'; + decisionLog.reason = 'reaper-lock-not-acquired'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + try { + const redisActivity = await readWorkspaceActivityFromRedis(entityId, redis); + if (!redisActivity.ok) { + decisionLog.action = 'skip'; + decisionLog.reason = 'redis-activity-read-failed'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + decisionLog.redisLastActivity = redisActivity.timestamp || null; + decisionLog.redisIdleMs = activityAgeMs(now, redisActivity.timestamp); + + const effectiveLastTs = Math.max(localLastTs, redisActivity.timestamp); + decisionLog.effectiveLastActivity = effectiveLastTs; + decisionLog.effectiveIdleMs = activityAgeMs(now, effectiveLastTs); + if (now - effectiveLastTs < maintenanceIdleMs) { + if (effectiveLastTs !== localLastTs) { + lastActivity.set(entityId, effectiveLastTs); + } + decisionLog.action = 'skip'; + decisionLog.reason = 'activity-fresh'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + const jobsCheck = await getWorkspaceBackgroundJobsStatus(entityConfig); + decisionLog.jobsCheck = serializeJobsCheckForReaperLog(jobsCheck); + if (jobsCheck.hasRunningJobs) { + logger.info(`Skipping idle stop for entity ${entityId}; workspace has running background jobs`); + decisionLog.action = 'skip'; + decisionLog.reason = 'running-background-jobs'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + const latestRedisActivity = await readWorkspaceActivityFromRedis(entityId, redis); + if (!latestRedisActivity.ok) { + decisionLog.action = 'skip'; + decisionLog.reason = 'latest-redis-activity-read-failed'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + const latestLocalActivity = lastActivity.get(entityId) || 0; + const latestActivityTs = Math.max( + effectiveLastTs, + latestRedisActivity.timestamp, + latestLocalActivity, + ); + decisionLog.latestLocalActivity = latestLocalActivity || null; + decisionLog.latestRedisActivity = latestRedisActivity.timestamp || null; + decisionLog.latestEffectiveActivity = latestActivityTs; + decisionLog.latestEffectiveIdleMs = activityAgeMs(now, latestActivityTs); + if (now - latestActivityTs < maintenanceIdleMs) { + if (latestActivityTs !== localLastTs) { + lastActivity.set(entityId, latestActivityTs); + } + decisionLog.action = 'skip'; + decisionLog.reason = 'activity-fresh-after-jobs-check'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + if (backend.backendName === 'aci' && now - latestActivityTs >= checkpointIdleMs) { + const checkpointResult = await checkpointIdleWorkspaceIfNeeded( + entityId, + entityConfig, + latestActivityTs, + decisionLog, + ); + if (!checkpointResult.success) { + decisionLog.action = 'skip'; + decisionLog.reason = 'checkpoint-failed'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + entityConfig = checkpointResult.entityConfig || entityConfig; + + if (now - latestActivityTs < idleTimeoutMs) { + decisionLog.action = checkpointResult.checkpointed ? 'checkpoint' : 'skip'; + decisionLog.reason = checkpointResult.checkpointed + ? 'idle-checkpoint' + : (checkpointResult.skipped ? 'checkpoint-skipped-before-reap' : 'checkpoint-fresh-before-reap'); + logWorkspaceReaperDecision(decisionLog); + continue; + } + } + + // ACI: checkpoint and destroy the container group so it stops + // counting against the subscription's container-group quota. + // The next provision restores the Blob checkpoint into a warm + // or fresh container. + // Docker: just stop — no quota concern, and stop preserves the + // writable layer for a faster wake. + const reapMode = backend.backendName === 'aci' ? 'destroy' : 'stop'; + decisionLog.reapMode = reapMode; + const reapResult = reapMode === 'destroy' + ? await destroyWorkspace(entityId, entityConfig, { destroyVolume: false, lastActivityAt: latestActivityTs }) + : await stopWorkspace(entityId, entityConfig); + decisionLog.reapResult = { + success: Boolean(reapResult.success), + error: reapResult.error || null, + }; + if (!reapResult.success) { + decisionLog.action = 'skip'; + decisionLog.reason = `${reapMode}-failed`; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + lastActivity.delete(entityId); + await removeWorkspaceActivityFromRedis(entityId); + decisionLog.action = reapMode; + decisionLog.reason = 'idle-timeout-exceeded'; + logWorkspaceReaperDecision(decisionLog); + logger.info(`${reapMode === 'destroy' ? 'Destroyed' : 'Stopped'} idle workspace for entity ${entityId}`); + } finally { + await releaseWorkspaceReaperLock(entityId, redis); + } + } catch (e) { + decisionLog.action = 'error'; + decisionLog.reason = e.message; + logWorkspaceReaperDecision(decisionLog); + logger.error(`Idle reaper error for entity ${entityId}: ${e.message}`); + } + } +} + +function containerAgeMs(container, now) { + const raw = container.createdAt || container.tags?.createdAt || container.startedAt; + const createdMs = raw ? new Date(raw).getTime() : 0; + return Number.isFinite(createdMs) && createdMs > 0 ? now - createdMs : null; +} + +function isOldEnoughForContainerReap(container, now) { + const age = containerAgeMs(container, now); + return age !== null && age > 5 * 60 * 1000; +} + +function expectedWorkspaceImageRefs() { + const image = resolveWorkspaceImage(); + const refs = new Set([image]); + const acrServer = config.get('azureAcrServer'); + if (acrServer && !image.includes('/')) { + refs.add(`${acrServer}/${image}`); + } + return refs; +} + +function isCortexOwnedWorkspaceContainer(container) { + const tags = container.tags || {}; + if (tags.managedBy === 'cortex' && tags.workspaceContainerPrefix === workspaceContainerPrefix()) { + return true; + } + + const image = container.image; + return Boolean(image && expectedWorkspaceImageRefs().has(image)); +} + +function entityWorkspaceContainerMatches(entityConfig, containerName) { + return Boolean(entityConfig?.workspace?.containerId && entityConfig.workspace.containerId === containerName); +} + +async function lookupWorkspaceEntityForContainer(entityStore, container) { + const taggedEntityId = typeof container.tags?.entityId === 'string' && container.tags.entityId.trim() + ? container.tags.entityId.trim() + : null; + + try { + const entityConfig = await entityStore.getEntityByWorkspaceContainerId(container.name); + if (entityConfig) { + return { entityConfig, entityId: entityConfig.id, lookupFailed: false, error: null }; + } + } catch (e) { + return { + entityConfig: null, + entityId: taggedEntityId, + lookupFailed: true, + error: e, + }; + } + + if (!taggedEntityId) { + return { entityConfig: null, entityId: null, lookupFailed: false, error: null }; + } + + try { + const entityConfig = await entityStore.getEntity(taggedEntityId, { + fresh: true, + throwOnError: true, + }); + if (entityConfig && !entityWorkspaceContainerMatches(entityConfig, container.name)) { + return { + entityConfig: null, + entityId: entityConfig.id || taggedEntityId, + lookupFailed: false, + error: null, + }; + } + return { + entityConfig, + entityId: entityConfig?.id || taggedEntityId, + lookupFailed: false, + error: null, + }; + } catch (e) { + return { + entityConfig: null, + entityId: taggedEntityId, + lookupFailed: true, + error: e, + }; + } +} + +async function reapAciWorkspaceInventory({ backend, redis, now, idleTimeoutMs, checkpointIdleMs }) { + if (typeof backend.listWorkspaceContainers !== 'function') return false; + + let containers; + try { + containers = await backend.listWorkspaceContainers(); + } catch (e) { + logger.warn(`Skipping ACI workspace inventory reap; failed to list containers: ${e.message}`); + return false; + } + + let activePoolContainers; + try { + activePoolContainers = await getWarmPoolActiveContainerNames(redis); + } catch { + activePoolContainers = new Set(); + } + + const entityStore = getEntityStore(); + + for (const container of containers) { + const lookupResult = await lookupWorkspaceEntityForContainer(entityStore, container); + let entityConfig = lookupResult.entityConfig; + const entityId = lookupResult.entityId; + const inWarmPool = activePoolContainers.has(container.name); + const age = containerAgeMs(container, now); + const decisionLog = { + entityId, + containerName: container.name, + now, + idleTimeoutMs, + checkpointIdleMs, + containerAgeMs: age, + inWarmPool, + ownedByCortex: isCortexOwnedWorkspaceContainer(container), + assignedToEntity: Boolean(entityConfig), + entityLookupFailed: lookupResult.lookupFailed, + entityLookupError: lookupResult.error?.message || null, + lockAcquired: null, + workspace: serializeWorkspaceForReaperLog(entityConfig?.workspace), + redisLastActivity: null, + redisIdleMs: null, + localLastActivity: entityId ? (lastActivity.get(entityId) || null) : null, + localIdleMs: entityId ? activityAgeMs(now, lastActivity.get(entityId) || 0) : null, + latestLocalActivity: null, + latestRedisActivity: null, + latestEffectiveActivity: null, + latestEffectiveIdleMs: null, + jobsCheck: { attempted: false }, + action: null, + reason: null, + reapMode: 'destroy', + reapResult: null, + checkpointedAt: null, + checkpointFresh: null, + checkpointResult: null, + }; + + if (inWarmPool && !entityConfig) { + decisionLog.action = 'skip'; + decisionLog.reason = 'active-warm-pool'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + if (inWarmPool && entityConfig) { + try { + await removeWarmPoolEntry(container.name, redis); + decisionLog.reason = 'stale-warm-pool-entry-removed'; + } catch (e) { + logger.warn(`Failed to remove stale warm-pool entry for assigned container ${container.name}: ${e.message}`); + } + } + + if (!decisionLog.ownedByCortex) { + decisionLog.action = 'skip'; + decisionLog.reason = 'not-cortex-owned'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + if (!isOldEnoughForContainerReap(container, now)) { + decisionLog.action = 'skip'; + decisionLog.reason = 'container-too-new'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + if (lookupResult.lookupFailed) { + decisionLog.action = 'skip'; + decisionLog.reason = 'entity-lookup-failed'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + if (!entityConfig) { + const { acquired } = await acquireWorkspaceReaperLock(`container:${container.name}`); + decisionLog.lockAcquired = acquired; + if (!acquired) { + decisionLog.action = 'skip'; + decisionLog.reason = 'reaper-lock-not-acquired'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + try { + await backend.remove(container.name, container.name); + decisionLog.action = 'destroy'; + decisionLog.reason = 'orphan-container'; + decisionLog.reapResult = { success: true, error: null }; + } catch (e) { + decisionLog.action = 'skip'; + decisionLog.reason = 'orphan-destroy-failed'; + decisionLog.reapResult = { success: false, error: e.message }; + } finally { + await releaseWorkspaceReaperLock(`container:${container.name}`, redis); + } + logWorkspaceReaperDecision(decisionLog); + continue; + } + + const { acquired } = await acquireWorkspaceReaperLock(entityConfig.id); + decisionLog.lockAcquired = acquired; + if (!acquired) { + decisionLog.action = 'skip'; + decisionLog.reason = 'reaper-lock-not-acquired'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + try { + const redisActivity = await readWorkspaceActivityFromRedis(entityConfig.id, redis); + if (!redisActivity.ok) { + decisionLog.action = 'skip'; + decisionLog.reason = 'redis-activity-read-failed'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + const localActivity = lastActivity.get(entityConfig.id) || 0; + const effectiveActivity = Math.max(localActivity, redisActivity.timestamp); + decisionLog.localLastActivity = localActivity || null; + decisionLog.localIdleMs = activityAgeMs(now, localActivity); + decisionLog.redisLastActivity = redisActivity.timestamp || null; + decisionLog.redisIdleMs = activityAgeMs(now, redisActivity.timestamp); + decisionLog.latestEffectiveActivity = effectiveActivity || null; + decisionLog.latestEffectiveIdleMs = activityAgeMs(now, effectiveActivity); + + if (now - effectiveActivity < checkpointIdleMs) { + decisionLog.action = 'skip'; + decisionLog.reason = 'activity-fresh'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + if (entityConfig.workspace?.status === 'running') { + const jobsCheck = await getWorkspaceBackgroundJobsStatus(entityConfig); + decisionLog.jobsCheck = serializeJobsCheckForReaperLog(jobsCheck); + if (jobsCheck.hasRunningJobs) { + decisionLog.action = 'skip'; + decisionLog.reason = 'running-background-jobs'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + } else { + decisionLog.jobsCheck = { + attempted: false, + ok: true, + hasRunningJobs: false, + reason: 'workspace-not-running', + jobs: [], + runningJobCount: 0, + }; + } + + const latestRedisActivity = await readWorkspaceActivityFromRedis(entityConfig.id, redis); + if (!latestRedisActivity.ok) { + decisionLog.action = 'skip'; + decisionLog.reason = 'latest-redis-activity-read-failed'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + const latestLocalActivity = lastActivity.get(entityConfig.id) || 0; + const latestActivityTs = Math.max(effectiveActivity, latestLocalActivity, latestRedisActivity.timestamp); + decisionLog.latestLocalActivity = latestLocalActivity || null; + decisionLog.latestRedisActivity = latestRedisActivity.timestamp || null; + decisionLog.latestEffectiveActivity = latestActivityTs || null; + decisionLog.latestEffectiveIdleMs = activityAgeMs(now, latestActivityTs); + + if (now - latestActivityTs < checkpointIdleMs) { + decisionLog.action = 'skip'; + decisionLog.reason = 'activity-fresh-after-jobs-check'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + + const checkpointResult = await checkpointIdleWorkspaceIfNeeded( + entityConfig.id, + entityConfig, + latestActivityTs, + decisionLog, + ); + if (!checkpointResult.success) { + decisionLog.action = 'skip'; + decisionLog.reason = 'checkpoint-failed'; + logWorkspaceReaperDecision(decisionLog); + continue; + } + entityConfig = checkpointResult.entityConfig || entityConfig; + + if (now - latestActivityTs < idleTimeoutMs) { + decisionLog.action = checkpointResult.checkpointed ? 'checkpoint' : 'skip'; + decisionLog.reason = checkpointResult.checkpointed + ? 'idle-checkpoint' + : (checkpointResult.skipped ? 'checkpoint-skipped-before-reap' : 'checkpoint-fresh-before-reap'); + logWorkspaceReaperDecision(decisionLog); + continue; + } + + const reapResult = await destroyWorkspace(entityConfig.id, entityConfig, { + destroyVolume: false, + lastActivityAt: latestActivityTs, + }); + decisionLog.reapResult = { + success: Boolean(reapResult.success), + error: reapResult.error || null, + }; + decisionLog.action = reapResult.success ? 'destroy' : 'skip'; + decisionLog.reason = reapResult.success ? 'idle-timeout-exceeded' : 'destroy-failed'; + logWorkspaceReaperDecision(decisionLog); + } finally { + await releaseWorkspaceReaperLock(entityConfig.id, redis); + } + } + + return true; +} + +async function getWorkspaceBackgroundJobsStatus(entityConfig) { + const workspace = entityConfig?.workspace; + if (!workspace?.url || !workspace?.secret) { + return { + attempted: false, + ok: true, + hasRunningJobs: false, + reason: 'workspace-url-or-secret-missing', + jobs: [], + runningJobCount: 0, + }; + } + + try { + const { + response, + body: data, + authRecovered, + } = await fetchWorkspaceJsonWithAuthRecovery(entityConfig?.id, entityConfig, '/shell/jobs', { + timeoutMs: 10_000, + }); + + if (!response.ok) { + logger.warn(`Workspace background-job check failed: HTTP ${response.status}`); + return { + attempted: true, + ok: false, + hasRunningJobs: true, + reason: 'http-error', + httpStatus: response.status, + authRecovered, + jobs: null, + runningJobCount: null, + }; + } + + const responseType = Array.isArray(data) ? 'array' : typeof data; + const jobsContainer = Array.isArray(data) + ? 'response-array' + : (Array.isArray(data?.jobs) ? 'jobs-property' : null); + const jobs = Array.isArray(data) + ? data + : (Array.isArray(data?.jobs) ? data.jobs : null); + if (!jobs) { + logger.warn('Workspace background-job check failed: malformed jobs response'); + return { + attempted: true, + ok: false, + hasRunningJobs: true, + reason: 'malformed-response', + httpStatus: response.status, + authRecovered, + responseType, + jobsContainer, + jobs: null, + runningJobCount: null, + jobStatusCounts: null, + }; + } + + const runningJobs = jobs.filter(job => job?.status === 'running'); + const jobStatusCounts = summarizeWorkspaceJobStatuses(jobs); + if (runningJobs.length > 0) { + logger.info(`Workspace background-job check found ${runningJobs.length} running job(s)`); + } + return { + attempted: true, + ok: true, + hasRunningJobs: runningJobs.length > 0, + reason: runningJobs.length > 0 ? 'running-jobs-found' : 'no-running-jobs', + httpStatus: response.status, + authRecovered, + responseType, + jobsContainer, + jobs, + runningJobCount: runningJobs.length, + jobStatusCounts, + }; + } catch (e) { + logger.warn(`Workspace background-job check failed: ${e.message}`); + return { + attempted: true, + ok: false, + hasRunningJobs: true, + reason: 'request-failed', + error: e.message, + jobs: null, + runningJobCount: null, + }; + } +} + +async function hasRunningBackgroundJobs(entityConfig) { + const jobsCheck = await getWorkspaceBackgroundJobsStatus(entityConfig); + return jobsCheck.hasRunningJobs; +} + +// Combined interval: reap idle workspaces. Activity is written through to Redis +// on each successful workspace request, so the reaper does not persist activity +// into the entity document. +const _reaperTimer = setInterval(async () => { + try { + await reapIdleWorkspaces(); + } catch (e) { + logger.error(`Workspace reaper tick failed: ${e.message}`); + } +}, REAPER_INTERVAL_MS); + +// Allow the process to exit cleanly without waiting for the reaper timer +if (_reaperTimer.unref) _reaperTimer.unref(); + +// --------------------------------------------------------------------------- +// Warm pool initialization — delayed to avoid blocking startup +// --------------------------------------------------------------------------- +setTimeout(() => { + initWarmPool().catch(e => logger.error(`Warm pool initialization failed: ${e.message}`)); +}, 5000); + +function setActivityRedisClientForTest(client) { + _activityRedisClientOverride = client; + _activityRedisClient = null; +} + +function setWorkspaceCheckpointUploadForTest(fn) { + _workspaceCheckpointUploadOverride = fn; +} + +function setWorkspaceCheckpointContainerClientForTest(client) { + _workspaceCheckpointContainerClientOverride = client; +} + +function setWorkspaceLegacyShareUploadForTest(fn) { + _workspaceLegacyShareUploadOverride = fn; +} + +function resetActivityStateForTest() { + lastActivity.clear(); + _activityRedisClientOverride = undefined; + _activityRedisClient = null; + _activityRedisClientConnectPromise = null; + _workspaceCheckpointUploadOverride = undefined; + _workspaceCheckpointContainerClientOverride = undefined; + _workspaceLegacyShareUploadOverride = undefined; +} + +// Test-only: delete the Redis activity key for an entity so the reaper sees it +// as stale. Without this, integration tests can't force a reap because their +// own provisioning writes fresh activity. +async function clearRedisActivityForTest(entityId) { + const redis = await getActivityRedisClient(); + if (!redis) return false; + try { + await redis.del(workspaceActivityKey(entityId)); + await redis.zrem(workspaceActivityIndexKey(), entityId); + return true; + } catch { + return false; + } +} + +// Test-only exports for targeted unit coverage of recovery/provision paths. +export const __testables = { + checkpointWorkspace, + checkpointLegacyShareAfterProvision, + createGenericContainer, + getWorkspaceBackgroundJobsStatus, + getWorkspaceCheckpointStorageConfig, + getOrCreateWorkspaceCheckpointEncryptionKey, + hasRunningBackgroundJobs, + isEncryptedStreamingCheckpointWorkspace, + reapIdleWorkspaces, + recordWorkspaceActivity, + removeWorkspaceActivityFromRedis, + reconfigureForEntity, + restoreLegacyShareArchiveToContainer, + restoreWorkspaceCheckpointToContainer, + uploadWorkspaceCheckpoint, + setActivityRedisClientForTest, + setWorkspaceCheckpointContainerClientForTest, + setWorkspaceCheckpointUploadForTest, + setWorkspaceLegacyShareUploadForTest, + setupWorkspaceContainerForEntity, + resetActivityStateForTest, + clearRedisActivityForTest, + validateWorkspaceCheckpointMetadata, + buildWorkspaceCheckpointRestoreEncryption, + checkpointEncryptionFromMetadata, + checkpointEncryptionMetadata, + workspaceCheckpointBlobMetadata, + workspaceCheckpointBlobPath, + workspaceCheckpointBlobPathCandidates, + workspaceCheckpointEncryptionKeyId, + workspaceCheckpointIdentityHash, + lastActivity, +}; diff --git a/pathways/system/entity/tools/sys_tool_workspace_ssh.js b/pathways/system/entity/tools/sys_tool_workspace_ssh.js new file mode 100644 index 00000000..e9a4f48d --- /dev/null +++ b/pathways/system/entity/tools/sys_tool_workspace_ssh.js @@ -0,0 +1,384 @@ +// sys_tool_workspace_ssh.js +// Consolidated workspace tool — one shell interface replaces 14 individual tools. +// Built-in pseudo-commands: bg, poll, reset. +import logger from '../../../../lib/logger.js'; +import { sendToolStart, sendToolFinish } from '../../../../lib/pathwayTools.js'; +import { workspaceRequest, destroyWorkspace } from './shared/workspace_client.js'; +import { loadEntityConfig } from './shared/sys_entity_tools.js'; + +const DEFAULT_COMMAND_TIMEOUT_MS = 300000; +const RESET_DESTROY_TIMEOUT_MS = 900000; + +function timeoutSecondsToMs(timeoutSeconds, defaultMs = DEFAULT_COMMAND_TIMEOUT_MS) { + return timeoutSeconds ? timeoutSeconds * 1000 : defaultMs; +} + +export function resetDestroyTimeoutMs(timeoutSeconds) { + return Math.max(timeoutSecondsToMs(timeoutSeconds, RESET_DESTROY_TIMEOUT_MS), RESET_DESTROY_TIMEOUT_MS); +} + +/** + * Simple quoted-string-aware tokenizer. + * Splits on whitespace, but respects single and double quotes. + * Returns array of tokens with quotes stripped. + */ +export function tokenize(input) { + const tokens = []; + let current = ''; + let inQuote = null; + + for (let i = 0; i < input.length; i++) { + const ch = input[i]; + + if (inQuote) { + if (ch === inQuote) { + inQuote = null; + } else { + current += ch; + } + } else if (ch === '"' || ch === "'") { + inQuote = ch; + } else if (ch === ' ' || ch === '\t') { + if (current) { + tokens.push(current); + current = ''; + } + } else { + current += ch; + } + } + + if (current) tokens.push(current); + return tokens; +} + +/** Normalize a path so relative paths resolve under /workspace/ (matching shell cwd). */ +export function toAbsWorkspacePath(p) { + return p.startsWith('/') ? p : `/workspace/${p}`; +} + +/** + * Extract the last non-empty, non-hint line from stderr — usually the actual + * error message (e.g. "ModuleNotFoundError: No module named 'sympy'"). + */ +function lastMeaningfulLine(stderr) { + if (!stderr) return null; + const lines = stderr.split('\n') + .map(l => l.trim()) + .filter(l => l + && !l.startsWith('hint:') + && !l.startsWith('note:') + && !/^Node\.js v\d/.test(l) + ); + return lines[lines.length - 1] || null; +} + +// --- Handlers --- + +function workspaceRequestOptions(args, options = {}) { + const requestId = args._toolRequestId; + const parentCallId = args._parentToolCallId; + const lifecycleMessages = { + provision: { icon: '🏗️', message: 'Setting up workspace' }, + wake: { icon: '🚀', message: 'Starting workspace' }, + reprovision: { icon: '🔄', message: 'Updating workspace' }, + reconnect: { icon: '🔌', message: 'Reconnecting workspace' }, + checkpointBackup: { icon: '💾', message: 'Backing up workspace' }, + checkpointUpload: { icon: '☁️', message: 'Saving workspace backup' }, + restore: { icon: '📦', message: 'Restoring workspace backup' }, + destroy: { icon: '🧹', message: 'Destroying workspace container' }, + }; + + if (!requestId || !parentCallId) return options; + + return { + ...options, + async onWorkspaceLifecycle(event) { + const spec = lifecycleMessages[event.phase] || { icon: '💻', message: event.message || 'Preparing workspace' }; + const callId = `${parentCallId}:workspace-${event.phase}`; + + if (event.type === 'start') { + await sendToolStart(requestId, callId, spec.icon, event.message || spec.message); + } else if (event.type === 'finish') { + await sendToolFinish(requestId, callId, event.success !== false, event.error || null); + } + }, + }; +} + +async function handleShell(command, args, resolver) { + const { entityId, timeoutSeconds } = args; + const timeoutMs = timeoutSecondsToMs(timeoutSeconds); + const result = await workspaceRequest(entityId, '/shell', { command }, workspaceRequestOptions(args, { timeoutMs })); + + if (!result.success && !result.error) { + result.error = lastMeaningfulLine(result.stderr) + || `Command failed (exit code ${result.exitCode})`; + } + + // Hint when a failed command looks like it tried to use bg/poll as bash commands + if (!result.success && /\bbg\s+|poll\s+[0-9a-f]/i.test(command)) { + result.hint = '`bg` and `poll` are built-in commands of this tool, not bash commands. They must be the entire command string — e.g. command: "bg python train.py", not inside scripts or chained with && or ;.'; + } + + return JSON.stringify(result); +} + +async function handleBg(rawBgCommand, args, resolver) { + const { entityId } = args; + const result = await workspaceRequest(entityId, '/shell', { + command: rawBgCommand, + background: true, + }, workspaceRequestOptions(args, { timeoutMs: 15000 })); + + if (!result.success && !result.error) { + result.error = lastMeaningfulLine(result.stderr) + || `Command failed (exit code ${result.exitCode})`; + } + + return JSON.stringify(result); +} + +async function handlePoll(processId, args, resolver) { + const { entityId } = args; + const result = await workspaceRequest(entityId, `/shell/result/${encodeURIComponent(processId)}`, null, workspaceRequestOptions(args, { + method: 'GET', + timeoutMs: 10000, + })); + + if (!result.success && !result.error) { + result.error = lastMeaningfulLine(result.stderr) + || `Command failed (exit code ${result.exitCode})`; + } + + return JSON.stringify(result); +} + +async function handleJobs(args) { + const { entityId } = args; + const result = await workspaceRequest(entityId, '/shell/jobs', null, workspaceRequestOptions(args, { + method: 'GET', + timeoutMs: 10000, + })); + + return JSON.stringify(result); +} + +async function handleReset(tokens, args, resolver) { + // reset [--preserve ] [--destroy] [--destroy-volume] + const { entityId, timeoutSeconds } = args; + let destroy = false; + let destroyVolume = false; + const preservePaths = []; + + for (let i = 1; i < tokens.length; i++) { + const token = tokens[i]; + if (token === '--destroy') { + destroy = true; + } else if (token === '--destroy-volume') { + destroyVolume = true; + destroy = true; // implies destroy + } else if (token === '--preserve') { + // Collect all following non-flag tokens as preserve paths + i++; + while (i < tokens.length && !tokens[i].startsWith('--')) { + preservePaths.push(tokens[i]); + i++; + } + i--; // back up so outer loop increments correctly + } + } + + // Full container destruction + if (destroy) { + const timeoutMs = resetDestroyTimeoutMs(timeoutSeconds); + const entityConfig = await loadEntityConfig(entityId); + if (!entityConfig) { + return JSON.stringify({ success: false, error: 'Entity not found' }); + } + + const result = await destroyWorkspace(entityId, entityConfig, workspaceRequestOptions(args, { destroyVolume, timeoutMs })); + if (!result.success || destroyVolume) { + return JSON.stringify(result); + } + + const status = await workspaceRequest( + entityId, + '/status', + null, + workspaceRequestOptions(args, { method: 'GET', timeoutMs: 10000, recordActivity: false }), + ); + + return JSON.stringify({ + ...result, + reprovisioned: Boolean(status.success), + workspace: status.success ? status : undefined, + error: status.success ? undefined : status.error, + success: Boolean(status.success), + }); + } + + // Soft reset: wipe /workspace contents + const timeoutMs = timeoutSecondsToMs(timeoutSeconds); + const body = {}; + if (preservePaths.length > 0) { + body.preservePaths = preservePaths; + } + + const result = await workspaceRequest(entityId, '/reset', body, workspaceRequestOptions(args, { timeoutMs: 60000 })); + + return JSON.stringify(result); +} + +// --- Command routing --- + +/** + * Route a command string to the appropriate handler. + * Returns [handler, ...handlerArgs] or null if it's a plain shell command. + */ +function routeCommand(command) { + const tokens = tokenize(command); + if (tokens.length === 0) return null; + + const first = tokens[0].toLowerCase(); + + if (first === 'bg') { + // Preserve raw command after "bg " to avoid re-tokenizing quoted args + const rawBgCommand = command.replace(/^\s*bg\s+/, ''); + return { handler: handleBg, rawBgCommand }; + } + + if (first === 'poll') { + const processId = tokens[1]; + if (!processId) return null; // let it fall to shell (will error naturally) + return { handler: handlePoll, processId }; + } + + if (first === 'jobs') { + return { handler: handleJobs }; + } + + if (first === 'reset') { + return { handler: handleReset, tokens }; + } + + return null; // plain shell command +} + +// --- Tool definition and entry point --- + +export default { + prompt: [], + timeout: 900, + inputParameters: { + command: ``, + userMessage: ``, + icon: ``, + timeoutSeconds: 0, + entityId: ``, + contextId: ``, + fileAccessPlan: { + type: 'array', + items: { objType: 'FileAccessTargetInput' }, + default: [], + }, + chatId: ``, + userId: ``, + }, + toolDefinition: { + type: 'function', + icon: '💻', + defaultUserMessage: 'Working in the workspace', + toolCost: 1, + function: { + name: 'WorkspaceSSH', + description: `Execute commands in your persistent Linux workspace (Debian, cwd: /workspace). Use it for calculation, verification, data analysis, file processing, coding, and short-lived research notes. Don't guess when you can compute. + +Batch related checks into one shell command or small script when practical, and return concise summaries instead of dumping large files. Write full intermediate outputs to /workspace when useful. + +Pre-installed: Python 3 + pip, Node.js, bash, and common CLI tools (curl, jq, git, etc.). Install Python packages with: pip install --break-system-packages + +Persistence: files, packages, virtual environments, and project structures survive between sessions. Read /workspace/README.md if present. For multi-step research, use /workspace/research-notes.md as a concise scratchpad for facts synthesized from tool results so you do not repeat the same lookups. + +User files: /workspace/files/ syncs to the user's cloud storage. Use /workspace/files only for files the user should receive or see in their file collection. If you need cloud URL or metadata for a created file, call FileCollection with fileRef set to the workspace path. + +BUILT-IN COMMANDS — IMPORTANT: The commands below are special commands handled by this tool, NOT bash commands. They MUST be the ENTIRE command string passed to this tool. NEVER combine them with bash syntax — no "cd /workspace && bg ...", no "bg ... && echo done", no embedding them in scripts or subshells. Just pass the built-in command as the complete command string. + +• bg — run a command in the background (no timeout — suitable for servers, training, etc.). Returns a processId. The background process runs until it exits on its own or the workspace is destroyed. + CORRECT: command: "bg python train.py" + CORRECT: command: "bg node server.js" + WRONG: command: "cd /workspace && bg python train.py" (bg is not bash) + WRONG: command: "bg python train.py && echo started" (cannot chain with bg) + +• poll — check the status and output of a background process. Returns status (running/completed/failed), stdout, stderr, and exit code. + Example: command: "poll a1b2c3d4e5f6g7h8" + +• jobs — list all background processes with their processId, status, command, and duration. Use this to find processIds you may have lost, or to check what's still running. + Example: command: "jobs" + +• reset [--preserve .env] — wipe workspace contents +• reset --destroy — destroy and re-provision the workspace container while preserving files. This can take up to 15 minutes because it checkpoints first. +• reset --destroy-volume — destroy the container and persisted workspace data + +Everything else runs as a bash command. Relative and absolute paths both work.`, + parameters: { + type: 'object', + properties: { + command: { + type: 'string', + description: 'Shell command or built-in (e.g. "ls -la", "python script.py")', + }, + userMessage: { + type: 'string', + description: 'Required. A short user-facing progress string describing what this specific command is doing. Keep it specific to this call, not generic.', + }, + icon: { + type: 'string', + description: 'Required. A single emoji that visually represents the specific action this command performs — pick whatever fits best (language, file type, intent, mood). Use the generic 💻 only when truly nothing more specific applies.', + }, + timeoutSeconds: { + type: 'number', + description: 'Optional timeout in seconds for long-running commands. Defaults to 300 (5 min). reset --destroy has a 900s minimum because it checkpoints before destroying.', + }, + }, + required: ['command', 'userMessage', 'icon'], + }, + }, + }, + + executePathway: async ({ args, runAllPrompts, resolver }) => { + const { command, entityId } = args; + resolver.tool = JSON.stringify({ toolUsed: 'WorkspaceSSH' }); + + try { + if (!command || typeof command !== 'string') { + return JSON.stringify({ success: false, error: 'command is required' }); + } + + const route = routeCommand(command); + + if (!route) { + // Plain shell command + return handleShell(command, args, resolver); + } + + if (route.handler === handleBg) { + return route.handler(route.rawBgCommand, args, resolver); + } + + if (route.handler === handlePoll) { + return route.handler(route.processId, args, resolver); + } + + if (route.handler === handleJobs) { + return route.handler(args); + } + + // files and reset handlers receive tokens + return route.handler(route.tokens, args, resolver); + } catch (e) { + logger.error(`WorkspaceSSH error: ${e.message}`); + return JSON.stringify({ success: false, error: e.message }); + } + }, +}; diff --git a/tests/integration/workspace.integration.test.js b/tests/integration/workspace.integration.test.js new file mode 100644 index 00000000..fee1fcbd --- /dev/null +++ b/tests/integration/workspace.integration.test.js @@ -0,0 +1,538 @@ +/** + * Workspace Integration Tests + * + * These tests verify that the workspace container system works end-to-end. + * Self-contained: boots its own Cortex server via serverFactory(). + * + * Requirements: + * - Docker running locally (or DOCKER_HOST pointing to a remote Docker) + * - cortex-workspace:latest image available + * - Redis + MongoDB configured (via root .env) + * - Filehandler service (only for files push/pull and backup/restore tests) + * + * Run with: npm test -- cortex -- tests/integration/workspace.integration.test.js + */ + +import test from 'ava'; +import { v4 as uuidv4 } from 'uuid'; +import serverFactory from '../../index.js'; +import { stopWorkspace, destroyWorkspace, __testables as workspaceTestables } from '../../pathways/system/entity/tools/shared/workspace_client.js'; +import { loadEntityConfig } from '../../pathways/system/entity/tools/shared/sys_entity_tools.js'; +import { getEntityStore } from '../../lib/MongoEntityStore.js'; + +const TEST_ENTITY_ID = process.env.TEST_ENTITY_ID || 'jarvis'; + +let testServer; + +// Helper to call workspace SSH tool via Apollo executeOperation +async function workspaceExec(command, options = {}) { + const query = `query { + sys_tool_workspace_ssh( + command: ${JSON.stringify(command)}, + userMessage: "test", + entityId: "${options.entityId || TEST_ENTITY_ID}" + ${options.contextId ? `, contextId: "${options.contextId}"` : ''} + ${options.userId ? `, userId: "${options.userId}"` : ''} + ${options.chatId ? `, chatId: "${options.chatId}"` : ''} + ) { result } + }`; + + const response = await testServer.executeOperation({ query }); + + const errors = response.body?.singleResult?.errors; + if (errors) { + throw new Error(`GraphQL error: ${JSON.stringify(errors)}`); + } + + return JSON.parse(response.body.singleResult.data.sys_tool_workspace_ssh.result); +} + +// ============================================================================ +// Setup / Teardown +// ============================================================================ + +test.before(async (t) => { + t.timeout(300_000); // ACI provisioning can take 30-90s + try { + const { server, startServer } = await serverFactory(); + startServer && await startServer(); + testServer = server; + } catch (e) { + t.log(`Failed to start Cortex server: ${e.message}`); + t.log('Ensure MongoDB is configured and Docker is running'); + throw e; + } + + // Warm up: trigger workspace provisioning so tests don't individually timeout. + // ACI containers may need two attempts — the first provisions but may not pass + // health check in time; the second finds the container already running. + t.log('Warming up workspace (provisioning if needed)...'); + for (let attempt = 1; attempt <= 3; attempt++) { + const result = await workspaceExec('echo "workspace ready"'); + if (result.success) { + t.log('Workspace ready'); + break; + } + if (attempt === 3) { + t.log(`Workspace warm-up failed after ${attempt} attempts: ${JSON.stringify(result)}`); + throw new Error(`Workspace not ready: ${result.error}`); + } + t.log(`Workspace warm-up attempt ${attempt} failed, retrying...`); + } +}); + +test.after.always(async () => { + if (!testServer) return; + + // Destroy the workspace container so the next run starts fresh. + // The reconfigure tests rotate the container's in-memory secret; + // if the container persists across runs (or Docker restarts it), + // the env-var secret and MongoDB secret will be out of sync. + try { + await destroyWorkspace(TEST_ENTITY_ID); + } catch { + // Best-effort cleanup + } + + await testServer.stop(); +}); + +// ============================================================================ +// Basic Shell Operations +// ============================================================================ + +test.serial('shell › should execute simple commands', async (t) => { + const result = await workspaceExec('echo "hello workspace"'); + + t.true(result.success); + t.is(result.stdout.trim(), 'hello workspace'); + t.is(result.exitCode, 0); +}); + +test.serial('shell › should report command failures', async (t) => { + const result = await workspaceExec('bash -c "exit 42"'); + + // Non-zero exit code is still a "success" from execution perspective + t.is(result.exitCode, 42); +}); + +test.serial('shell › should handle complex pipelines', async (t) => { + const result = await workspaceExec('echo -e "c\\nb\\na" | sort | head -1'); + + t.true(result.success); + t.is(result.stdout.trim(), 'a'); +}); + +// ============================================================================ +// File Operations +// ============================================================================ + +test.serial('files › should create and read files', async (t) => { + // Create file + const createResult = await workspaceExec('echo "test content" > /workspace/test_file.txt'); + t.true(createResult.success); + + // Read file + const readResult = await workspaceExec('cat /workspace/test_file.txt'); + t.true(readResult.success); + t.is(readResult.stdout.trim(), 'test content'); +}); + +test.serial('files › should create directories', async (t) => { + const result = await workspaceExec('mkdir -p /workspace/test_dir/nested && ls -la /workspace/test_dir'); + + t.true(result.success); + t.true(result.stdout.includes('nested')); +}); + +test.serial('files › should handle binary files', async (t) => { + // Create a small binary file using base64 decode, verify with xxd or od + const createResult = await workspaceExec('echo "iVBORw0KGgo=" | base64 -d > /workspace/test.bin && wc -c < /workspace/test.bin'); + + t.true(createResult.success); + // Should have written some bytes + t.true(parseInt(createResult.stdout.trim()) > 0); +}); + +// ============================================================================ +// Background Jobs +// ============================================================================ + +test.serial('background › should run jobs in background', async (t) => { + // Start background job + const bgResult = await workspaceExec('bg sleep 1 && echo "bg done" > /workspace/bg_marker.txt'); + + t.true(bgResult.success); + t.truthy(bgResult.processId); + + // Wait and poll + await new Promise(resolve => setTimeout(resolve, 2000)); + + const pollResult = await workspaceExec(`poll ${bgResult.processId}`); + + t.true(pollResult.success); + t.is(pollResult.status, 'completed'); +}); + +// ============================================================================ +// Git Operations +// ============================================================================ + +test.serial('git › should initialize and commit', async (t) => { + const commands = [ + 'mkdir -p /workspace/git_test', + 'cd /workspace/git_test && git init', + 'cd /workspace/git_test && git config user.email "test@test.com"', + 'cd /workspace/git_test && git config user.name "Test"', + 'echo "# Test" > /workspace/git_test/README.md', + 'cd /workspace/git_test && git add . && git commit -m "Initial"', + ].join(' && '); + + const result = await workspaceExec(commands); + t.true(result.success); + + // Verify commit exists + const logResult = await workspaceExec('cd /workspace/git_test && git log --oneline'); + t.true(logResult.success); + t.true(logResult.stdout.includes('Initial')); +}); + +// ============================================================================ +// Network Access +// ============================================================================ + +test.serial('network › should access external URLs', async (t) => { + const result = await workspaceExec('curl -s https://httpbin.org/get | head -5'); + + t.true(result.success); + t.true(result.stdout.includes('args') || result.stdout.includes('headers')); +}); + +// ============================================================================ +// Workspace Reset +// ============================================================================ + +test.serial('reset › should clear workspace contents', async (t) => { + // Create some files first + await workspaceExec('echo "to be deleted" > /workspace/delete_me.txt'); + + // Reset + const resetResult = await workspaceExec('reset'); + + t.true(resetResult.success); + t.true(resetResult.message.includes('reset')); + + // Verify files are gone + const lsResult = await workspaceExec('ls /workspace'); + t.true(lsResult.success); + t.false(lsResult.stdout.includes('delete_me.txt')); +}); + +// ============================================================================ +// Stop / Wake-on-Demand (idle management) +// ============================================================================ + +test.serial('idle › should stop and auto-wake workspace, preserving data', async (t) => { + t.timeout(180_000); // docker: ~20s; ACI: stop (~5s) + start (~30-60s) + health check + // Create a marker file to verify data persistence across stop/start + const createResult = await workspaceExec('echo "survive stop" > /workspace/persist_test.txt'); + t.true(createResult.success); + + // Stop the workspace directly + const entityConfig = await loadEntityConfig(TEST_ENTITY_ID); + t.truthy(entityConfig?.workspace?.containerId, 'Workspace should have a containerId'); + + const stopResult = await stopWorkspace(TEST_ENTITY_ID, entityConfig); + t.true(stopResult.success); + + // Verify entity status is now 'stopped' + const stoppedConfig = await loadEntityConfig(TEST_ENTITY_ID); + t.is(stoppedConfig.workspace.status, 'stopped'); + t.truthy(stoppedConfig.workspace.stoppedAt); + + // Execute a command — this should trigger wake-on-demand automatically + const wakeResult = await workspaceExec('cat /workspace/persist_test.txt'); + t.true(wakeResult.success); + t.is(wakeResult.stdout.trim(), 'survive stop'); + + // Verify entity status is back to 'running' + const runningConfig = await loadEntityConfig(TEST_ENTITY_ID); + t.is(runningConfig.workspace.status, 'running'); +}); + +// ============================================================================ +// Reconfigure endpoint (secret rotation + env injection) +// ============================================================================ + +test.serial('reconfigure > should rotate secret', async (t) => { + t.timeout(60_000); + + // Load entity config to get current workspace URL + secret + const entityConfig = await loadEntityConfig(TEST_ENTITY_ID); + t.truthy(entityConfig?.workspace?.url, 'Workspace must be provisioned'); + t.truthy(entityConfig?.workspace?.secret, 'Workspace must have a secret'); + + const { url, secret: oldSecret } = entityConfig.workspace; + const newSecret = `rotated-${Date.now()}-${Math.random().toString(36).slice(2)}`; + + // Call /reconfigure with the old secret to rotate to the new secret + const reconfigResponse = await fetch(`${url}/reconfigure`, { + method: 'POST', + headers: { + 'x-workspace-secret': oldSecret, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ secret: newSecret }), + }); + + t.is(reconfigResponse.status, 200); + const reconfigResult = await reconfigResponse.json(); + t.true(reconfigResult.success); + + // Verify old secret is rejected + const oldSecretResponse = await fetch(`${url}/status`, { + headers: { 'x-workspace-secret': oldSecret }, + }); + t.is(oldSecretResponse.status, 401, 'Old secret should be rejected after rotation'); + + // Verify new secret is accepted + const newSecretResponse = await fetch(`${url}/status`, { + headers: { 'x-workspace-secret': newSecret }, + }); + t.is(newSecretResponse.status, 200, 'New secret should be accepted after rotation'); + + // Update entity config in MongoDB so subsequent tests use the new secret + const entityStore = getEntityStore(); + await entityStore.upsertEntity({ + ...entityConfig, + workspace: { ...entityConfig.workspace, secret: newSecret }, + }); +}); + +test.serial('reconfigure > should inject env vars', async (t) => { + t.timeout(60_000); + + // Load entity config with the (possibly rotated) secret + const entityConfig = await loadEntityConfig(TEST_ENTITY_ID); + const { url, secret } = entityConfig.workspace; + + // Call /reconfigure to inject env vars + const reconfigResponse = await fetch(`${url}/reconfigure`, { + method: 'POST', + headers: { + 'x-workspace-secret': secret, + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + env: { TEST_RECONFIG_VAR: 'hello_from_reconfigure' }, + }), + }); + + t.is(reconfigResponse.status, 200); + const reconfigResult = await reconfigResponse.json(); + t.true(reconfigResult.success); + + // Verify /workspace/.env contains the var + const catResult = await workspaceExec('cat /workspace/.env'); + t.true(catResult.success); + t.true(catResult.stdout.includes('TEST_RECONFIG_VAR'), '.env should contain TEST_RECONFIG_VAR'); + t.true(catResult.stdout.includes('hello_from_reconfigure'), '.env should contain the value'); + + // Verify the var is accessible when .env is sourced + const shellResult = await workspaceExec('. /workspace/.env && echo $TEST_RECONFIG_VAR'); + t.true(shellResult.success); + t.is(shellResult.stdout.trim(), 'hello_from_reconfigure'); +}); + +// ============================================================================ +// Per-User Blob Mount (private entity with single user association) +// ============================================================================ + +test.serial('blob mount › public entity should have no blob mount', async (t) => { + // The default test entity (jarvis) is public — no assocUserIds + // Verify /workspace/files/ is NOT a blobfuse2 mount + const result = await workspaceExec('mount | grep blobfuse2 || echo "no blobfuse2 mount"'); + + t.true(result.success); + t.true(result.stdout.includes('no blobfuse2 mount')); +}); + +test.serial('blob mount › private entity should get per-user blob mount', async (t) => { + if ((process.env.WORKSPACE_BACKEND || 'docker') !== 'aci') { + t.log('Skipping: blob mount requires ACI backend (blobfuse2 is not configured on Docker)'); + t.pass(); + return; + } + t.timeout(300_000); + + const privateEntityId = `test-private-${uuidv4().slice(0, 8)}`; + const testUserId = `test-user-${uuidv4().slice(0, 8)}`; + + // Create a private entity with a single user association + const entityStore = getEntityStore(); + await entityStore.upsertEntity({ + id: privateEntityId, + name: 'Test Private Entity', + assocUserIds: [testUserId], + tools: ['workspacessh'], + }); + + try { + // Provision workspace — should get a per-user blob mount + t.log(`Provisioning private entity ${privateEntityId} with user ${testUserId}...`); + for (let attempt = 1; attempt <= 3; attempt++) { + const result = await workspaceExec('echo "ready"', { entityId: privateEntityId }); + if (result.success) break; + if (attempt === 3) throw new Error(`Private entity workspace not ready: ${result.error}`); + t.log(`Attempt ${attempt} failed, retrying...`); + } + + // Verify blobfuse2 mount exists + const mountResult = await workspaceExec( + 'mount | grep blobfuse2 || echo "no blobfuse2 mount"', + { entityId: privateEntityId }, + ); + t.true(mountResult.success); + t.true(mountResult.stdout.includes('blobfuse2'), 'Should have a blobfuse2 mount'); + t.true(mountResult.stdout.includes('/workspace/files'), 'Mount should be at /workspace/files'); + + // Verify we can write and read through the mount + const writeResult = await workspaceExec( + 'echo "blob mount test" > /workspace/files/test_blob.txt && cat /workspace/files/test_blob.txt', + { entityId: privateEntityId }, + ); + t.true(writeResult.success); + t.is(writeResult.stdout.trim(), 'blob mount test'); + + // Verify the container env doesn't have the account key (only SAS token) + const envResult = await workspaceExec('env | grep AZURE_STORAGE_ACCOUNT_KEY || echo "no account key"', { + entityId: privateEntityId, + }); + t.true(envResult.success); + t.true(envResult.stdout.includes('no account key'), 'Account key should NOT be in env'); + } finally { + // Clean up: destroy the private entity's workspace and remove the entity + try { + const entityConfig = await loadEntityConfig(privateEntityId); + if (entityConfig?.workspace) { + await destroyWorkspace(privateEntityId, entityConfig, { destroyVolume: true }); + } + await entityStore.deleteEntity(privateEntityId); + } catch { + // Best-effort cleanup + } + } +}); + +// ============================================================================ +// Idle reaper — destroys ACI containers and rewakes via share remount +// ============================================================================ + +test.serial('reaper › idle ACI workspace is destroyed and rewakes from preserved share', async (t) => { + if ((process.env.WORKSPACE_BACKEND || 'docker') !== 'aci') { + t.log('Skipping: only relevant for ACI backend (Docker keeps stop semantics)'); + t.pass(); + return; + } + t.timeout(600_000); // provision + destroy + reprovision can run long on ACI + + const reaperEntityId = `test-reaper-${uuidv4().slice(0, 8)}`; + const entityStore = getEntityStore(); + + // Snapshot lastActivity so we don't accidentally reap any other entity. + // The reaper iterates everything in this Map, so we clear it, set only our + // test entity to a stale timestamp, then restore other entries afterwards. + const lastActivitySnapshot = new Map(workspaceTestables.lastActivity); + + await entityStore.upsertEntity({ + id: reaperEntityId, + name: 'Test Reaper Entity', + tools: ['workspacessh'], + }); + + let provisionedShareName = null; + + try { + // 1. Provision the workspace + t.log(`Provisioning ${reaperEntityId}...`); + for (let attempt = 1; attempt <= 3; attempt++) { + const result = await workspaceExec('echo "ready"', { entityId: reaperEntityId }); + if (result.success) break; + if (attempt === 3) throw new Error(`Workspace not ready: ${result.error}`); + } + + // 2. Write a marker into the Azure Files share at /workspace + const markerWrite = await workspaceExec( + 'echo "survive-reap-destroy" > /workspace/reaper_marker.txt', + { entityId: reaperEntityId }, + ); + t.true(markerWrite.success, 'should write marker file'); + + const beforeReap = await loadEntityConfig(reaperEntityId); + t.truthy(beforeReap?.workspace?.containerId, 'should have containerId before reap'); + t.is(beforeReap.workspace.status, 'running'); + provisionedShareName = beforeReap.workspace.shareName || beforeReap.workspace.containerId; + t.truthy(provisionedShareName, 'should have a shareName before reap'); + + // 3. Force the reaper to fire for ONLY this entity. + // Clear the shared lastActivity Map (snapshot restored in finally), + // then mark just this entity as ancient so the timeout check trips. + // Also clear the entity's Redis activity key — the reaper takes + // max(local, redis) and our own provisioning just wrote a fresh value. + workspaceTestables.lastActivity.clear(); + workspaceTestables.lastActivity.set(reaperEntityId, 1); + await workspaceTestables.clearRedisActivityForTest(reaperEntityId); + + t.log('Running reaper...'); + await workspaceTestables.reapIdleWorkspaces(); + + // 4. Assert the entity workspace was reduced to just shareName + const afterReap = await loadEntityConfig(reaperEntityId); + t.truthy(afterReap?.workspace, 'workspace stub should remain'); + t.is(afterReap.workspace.shareName, provisionedShareName, 'shareName preserved'); + t.falsy(afterReap.workspace.containerId, 'containerId should be cleared'); + t.falsy(afterReap.workspace.url, 'url should be cleared'); + + // 5. Trigger workspace use → must auto-reprovision and remount share + t.log('Triggering reprovision via workspace use...'); + const rewakeRead = await workspaceExec( + 'cat /workspace/reaper_marker.txt', + { entityId: reaperEntityId }, + ); + t.true(rewakeRead.success, 'reprovisioned workspace should respond'); + t.is(rewakeRead.stdout.trim(), 'survive-reap-destroy', 'marker survives via share remount'); + + // 6. Sanity: new container has a (possibly different) containerId + const afterRewake = await loadEntityConfig(reaperEntityId); + t.truthy(afterRewake?.workspace?.containerId, 'should have containerId after rewake'); + t.is(afterRewake.workspace.status, 'running'); + t.is( + afterRewake.workspace.shareName || afterRewake.workspace.containerId, + provisionedShareName, + 'rewake remounts the same share', + ); + } finally { + // Restore the lastActivity Map so we don't affect other tests' state + workspaceTestables.lastActivity.clear(); + for (const [k, v] of lastActivitySnapshot) { + workspaceTestables.lastActivity.set(k, v); + } + + // Clean up: destroy workspace AND volume, delete entity + try { + const finalConfig = await loadEntityConfig(reaperEntityId); + if (finalConfig?.workspace) { + await destroyWorkspace(reaperEntityId, finalConfig, { destroyVolume: true }); + } else if (provisionedShareName) { + // Workspace was already torn down (e.g. test failed mid-way) but + // share may linger. Best-effort: synthesize enough config to nuke it. + await destroyWorkspace(reaperEntityId, { + workspace: { shareName: provisionedShareName }, + }, { destroyVolume: true }); + } + await entityStore.deleteEntity(reaperEntityId); + } catch (e) { + t.log(`Cleanup error (best-effort): ${e.message}`); + } + } +}); diff --git a/tests/unit/tools/workspaceBootstrapSecretFix.test.js b/tests/unit/tools/workspaceBootstrapSecretFix.test.js new file mode 100644 index 00000000..c1bd66d9 --- /dev/null +++ b/tests/unit/tools/workspaceBootstrapSecretFix.test.js @@ -0,0 +1,4792 @@ +import test from 'ava'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import logger from '../../../lib/logger.js'; +import { config } from '../../../config.js'; +import { getEntityStore } from '../../../lib/MongoEntityStore.js'; + +let warmPoolModule; +let workspaceClientModule; +let ACIBackend; + +test.before(async () => { + const originalSetTimeout = global.setTimeout; + global.setTimeout = () => ({ unref() {} }); + + try { + warmPoolModule = await import('../../../pathways/system/entity/tools/shared/warmPool.js'); + workspaceClientModule = await import('../../../pathways/system/entity/tools/shared/workspace_client.js'); + ({ default: ACIBackend } = await import('../../../pathways/system/entity/tools/shared/backends/ACIBackend.js')); + } finally { + global.setTimeout = originalSetTimeout; + } +}); + +test('workspaceRequest rejects blank entity ids before provisioning', async (t) => { + const loggerStub = stubLogger(); + t.teardown(() => loggerStub.restore()); + + const result = await workspaceClientModule.workspaceRequest('', '/health'); + + t.false(result.success); + t.is(result.error, 'Workspace entityId is required'); + t.true(loggerStub.calls.some(call => call.message === 'Workspace request skipped: missing entityId')); +}); + +test('workspace checkpoints prefer workspace Azure Files storage account', (t) => { + const restoreConfig = stubConfig({ + azureStorageAccountName: 'general-blob-account', + azureStorageAccountKey: 'general-key', + workspaceAzureFilesStorageAccountName: 'workspace-files-account', + workspaceAzureFilesStorageAccountKey: 'workspace-key', + azureBlobContainerName: 'checkpoint-container', + }); + + try { + t.deepEqual(workspaceClientModule.__testables.getWorkspaceCheckpointStorageConfig(), { + accountName: 'workspace-files-account', + accountKey: 'workspace-key', + containerName: 'checkpoint-container', + }); + } finally { + restoreConfig(); + } +}); + +test('workspace checkpoint blob paths are collision-resistant for sanitized entity ids', (t) => { + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + }); + + try { + const underscorePath = workspaceClientModule.__testables.workspaceCheckpointBlobPath('entity_abc'); + const hyphenPath = workspaceClientModule.__testables.workspaceCheckpointBlobPath('entity-abc'); + const candidates = workspaceClientModule.__testables.workspaceCheckpointBlobPathCandidates('entity_abc'); + + t.not(underscorePath, hyphenPath); + t.true(underscorePath.includes(workspaceClientModule.__testables.workspaceCheckpointIdentityHash('entity_abc'))); + t.true(hyphenPath.includes(workspaceClientModule.__testables.workspaceCheckpointIdentityHash('entity-abc'))); + t.deepEqual(candidates, [ + underscorePath, + 'workspace-checkpoints/test-cortex/entity-abc/workspace.tar.gz', + ]); + } finally { + restoreConfig(); + } +}); + +test('workspace checkpoint metadata validates exact entity identity', (t) => { + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + }); + + try { + const metadata = workspaceClientModule.__testables.workspaceCheckpointBlobMetadata('entity_abc'); + + t.true(workspaceClientModule.__testables.validateWorkspaceCheckpointMetadata(metadata, 'entity_abc')); + t.throws( + () => workspaceClientModule.__testables.validateWorkspaceCheckpointMetadata(metadata, 'entity-abc'), + { message: 'Workspace checkpoint entity metadata does not match requested entity' }, + ); + t.throws( + () => workspaceClientModule.__testables.validateWorkspaceCheckpointMetadata({}, 'entity_abc'), + { message: 'Workspace checkpoint is missing entity identity metadata' }, + ); + } finally { + restoreConfig(); + } +}); + +test.serial('workspace checkpoint encryption key is stored encrypted and used for restore material', async (t) => { + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + redisEncryptionKey: 'a'.repeat(64), + }); + const store = stubMutableEntityStore({ + id: 'entity-encrypted-checkpoint', + workspace: { + status: 'running', + }, + }); + + try { + const result = await workspaceClientModule.__testables.getOrCreateWorkspaceCheckpointEncryptionKey( + 'entity-encrypted-checkpoint', + store.getEntity(), + ); + + t.is(result.algorithm, 'aes-256-gcm'); + t.is(Buffer.from(result.keyBase64, 'base64').length, 32); + t.truthy(result.keyId); + t.not(store.getEntity().workspace.checkpointEncryptionKey.encryptedKey, result.keyBase64); + t.is(store.getEntity().workspace.checkpointEncryptionKey.keyId, result.keyId); + + const restoreMaterial = workspaceClientModule.__testables.buildWorkspaceCheckpointRestoreEncryption({ + ...store.getEntity(), + workspace: { + ...store.getEntity().workspace, + checkpointEncryption: { + algorithm: 'aes-256-gcm', + keyId: result.keyId, + ivBase64: Buffer.alloc(12, 1).toString('base64'), + tagBase64: Buffer.alloc(16, 2).toString('base64'), + compression: 'zstd', + }, + }, + }); + + t.is(restoreMaterial.keyBase64, result.keyBase64); + t.is(restoreMaterial.keyId, result.keyId); + t.is(restoreMaterial.ivBase64, Buffer.alloc(12, 1).toString('base64')); + t.is(restoreMaterial.tagBase64, Buffer.alloc(16, 2).toString('base64')); + t.is(restoreMaterial.compression, 'zstd'); + } finally { + store.restore(); + restoreConfig(); + } +}); + +test.serial('workspace checkpoint encryption key creation reuses a freshly stored key', async (t) => { + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + redisEncryptionKey: 'b'.repeat(64), + }); + const store = stubMutableEntityStore({ + id: 'entity-encrypted-checkpoint-race', + workspace: { + status: 'running', + }, + }); + + try { + const existing = await workspaceClientModule.__testables.getOrCreateWorkspaceCheckpointEncryptionKey( + 'entity-encrypted-checkpoint-race', + store.getEntity(), + ); + const freshEntity = store.getEntity(); + + const result = await workspaceClientModule.__testables.getOrCreateWorkspaceCheckpointEncryptionKey( + 'entity-encrypted-checkpoint-race', + { + id: 'entity-encrypted-checkpoint-race', + workspace: { + status: 'running', + }, + }, + ); + + t.is(result.keyId, existing.keyId); + t.is(result.keyBase64, existing.keyBase64); + t.is(store.getEntity(), freshEntity); + } finally { + store.restore(); + restoreConfig(); + } +}); + +test('workspace checkpoint encryption metadata round-trips from Blob metadata', (t) => { + const metadata = workspaceClientModule.__testables.checkpointEncryptionMetadata({ + algorithm: 'aes-256-gcm', + keyId: 'key-1', + ivBase64: Buffer.alloc(12, 3).toString('base64'), + tagBase64: Buffer.alloc(16, 4).toString('base64'), + compression: 'pigz', + }); + + t.deepEqual(workspaceClientModule.__testables.checkpointEncryptionFromMetadata(metadata), { + algorithm: 'aes-256-gcm', + keyId: 'key-1', + ivBase64: Buffer.alloc(12, 3).toString('base64'), + tagBase64: Buffer.alloc(16, 4).toString('base64'), + compression: 'pigz', + }); +}); + +test('encrypted streaming checkpoint requires workspace helper 1.0.10 or newer', (t) => { + t.false(workspaceClientModule.__testables.isEncryptedStreamingCheckpointWorkspace('1.0.9')); + t.true(workspaceClientModule.__testables.isEncryptedStreamingCheckpointWorkspace('1.0.10')); + t.true(workspaceClientModule.__testables.isEncryptedStreamingCheckpointWorkspace('1.1.0')); + t.true(workspaceClientModule.__testables.isEncryptedStreamingCheckpointWorkspace('2.0.0')); +}); + +function stubConfig(stubs) { + const originalGet = config.get.bind(config); + config.get = (key) => { + if (key in stubs) return stubs[key]; + if (key === 'workspaceContainerPrefix') return 'workspace'; + return originalGet(key); + }; + return () => { + config.get = originalGet; + }; +} + +function stubLogger() { + const original = { + info: logger.info, + warn: logger.warn, + error: logger.error, + }; + const calls = []; + + logger.info = (message) => calls.push({ level: 'info', message }); + logger.warn = (message) => calls.push({ level: 'warn', message }); + logger.error = (message) => calls.push({ level: 'error', message }); + + return { + calls, + restore() { + logger.info = original.info; + logger.warn = original.warn; + logger.error = original.error; + }, + }; +} + +function stubEntityStore(entity) { + const entityStore = getEntityStore(); + const original = { + isConfigured: entityStore.isConfigured, + getEntity: entityStore.getEntity, + getEntityByWorkspaceContainerId: entityStore.getEntityByWorkspaceContainerId, + getDefaultEntity: entityStore.getDefaultEntity, + getAllEntities: entityStore.getAllEntities, + upsertEntity: entityStore.upsertEntity, + }; + let currentEntity = entity; + + entityStore.isConfigured = () => true; + entityStore.getEntity = async () => currentEntity; + entityStore.getEntityByWorkspaceContainerId = async (containerId) => + currentEntity?.workspace?.containerId === containerId ? currentEntity : null; + entityStore.getDefaultEntity = async () => null; + entityStore.getAllEntities = async () => currentEntity ? [currentEntity] : []; + entityStore.upsertEntity = async (nextEntity) => { + currentEntity = nextEntity; + return nextEntity; + }; + + return () => { + entityStore.isConfigured = original.isConfigured; + entityStore.getEntity = original.getEntity; + entityStore.getEntityByWorkspaceContainerId = original.getEntityByWorkspaceContainerId; + entityStore.getDefaultEntity = original.getDefaultEntity; + entityStore.getAllEntities = original.getAllEntities; + entityStore.upsertEntity = original.upsertEntity; + }; +} + +function stubMutableEntityStore(entity) { + const entityStore = getEntityStore(); + const original = { + isConfigured: entityStore.isConfigured, + getEntity: entityStore.getEntity, + getEntityByWorkspaceContainerId: entityStore.getEntityByWorkspaceContainerId, + getDefaultEntity: entityStore.getDefaultEntity, + getAllEntities: entityStore.getAllEntities, + upsertEntity: entityStore.upsertEntity, + }; + let currentEntity = entity; + + entityStore.isConfigured = () => true; + entityStore.getEntity = async () => currentEntity; + entityStore.getEntityByWorkspaceContainerId = async (containerId) => + currentEntity?.workspace?.containerId === containerId ? currentEntity : null; + entityStore.getDefaultEntity = async () => null; + entityStore.getAllEntities = async () => currentEntity ? [currentEntity] : []; + entityStore.upsertEntity = async (nextEntity) => { + currentEntity = nextEntity; + return nextEntity; + }; + + return { + getEntity() { + return currentEntity; + }, + setEntity(nextEntity) { + currentEntity = nextEntity; + }, + restore() { + entityStore.isConfigured = original.isConfigured; + entityStore.getEntity = original.getEntity; + entityStore.getEntityByWorkspaceContainerId = original.getEntityByWorkspaceContainerId; + entityStore.getDefaultEntity = original.getDefaultEntity; + entityStore.getAllEntities = original.getAllEntities; + entityStore.upsertEntity = original.upsertEntity; + }, + }; +} + +function createFakeRedis(options = {}) { + const { + activityTimestamp, + acquireLock = true, + activityIndex = {}, + hashes = {}, + } = options; + const values = new Map(); + const zsets = new Map(); + const calls = []; + + for (const [entityId, score] of Object.entries(activityIndex)) { + zsets.set(entityId, Number(score)); + } + + return { + calls, + async get(key) { + calls.push({ op: 'get', key }); + if (key.includes(':activity:') && activityTimestamp !== undefined) { + const value = typeof activityTimestamp === 'function' + ? activityTimestamp() + : activityTimestamp; + return String(value); + } + return values.get(key) ?? null; + }, + async set(key, value, ...args) { + calls.push({ op: 'set', key, value, args }); + if (args.includes('NX')) { + if (!acquireLock || values.has(key)) return null; + values.set(key, value); + return 'OK'; + } + values.set(key, value); + return 'OK'; + }, + async del(key) { + calls.push({ op: 'del', key }); + values.delete(key); + return 1; + }, + async zadd(key, score, member) { + calls.push({ op: 'zadd', key, score, member }); + zsets.set(member, Number(score)); + return 1; + }, + async zrangebyscore(key, min, max) { + calls.push({ op: 'zrangebyscore', key, min, max }); + const minScore = Number(min); + const maxScore = Number(max); + return Array.from(zsets.entries()) + .filter(([, score]) => score >= minScore && score <= maxScore) + .map(([member]) => member); + }, + async zrem(key, member) { + calls.push({ op: 'zrem', key, member }); + return zsets.delete(member) ? 1 : 0; + }, + async srem(key, member) { + calls.push({ op: 'srem', key, member }); + return 1; + }, + async hgetall(key) { + calls.push({ op: 'hgetall', key }); + return hashes[key] || {}; + }, + async hdel(key, field) { + calls.push({ op: 'hdel', key, field }); + if (!hashes[key] || !(field in hashes[key])) return 0; + delete hashes[key][field]; + return 1; + }, + }; +} + +test.serial('warmPool init uses WARM_POOL_ENABLED instead of bootstrap secret presence', async (t) => { + const restoreConfig = stubConfig({ + warmPoolSize: 1, + workspaceBackend: 'aci', + warmPoolEnabled: true, + warmPoolBootstrapSecret: '', + storageConnectionString: '', + cortexId: 'test-cortex', + }); + const logCapture = stubLogger(); + + try { + await warmPoolModule.initWarmPool(); + + t.true( + logCapture.calls.some(call => call.message.includes('Redis not available')), + 'expected initWarmPool to proceed past the legacy bootstrap-secret guard' + ); + t.false( + logCapture.calls.some(call => call.message.includes('WARM_POOL_BOOTSTRAP_SECRET is required')), + 'warmPool should no longer require WARM_POOL_BOOTSTRAP_SECRET' + ); + } finally { + logCapture.restore(); + restoreConfig(); + } +}); + +test.serial('warmPool pool containers get unique bootstrap secrets', async (t) => { + const restoreConfig = stubConfig({ + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '', + workspaceCpus: '1', + workspaceMemory: '512m', + workspaceDiskSize: '10g', + warmPoolBootstrapSecret: 'legacy-shared-secret', + }); + const logCapture = stubLogger(); + const originalFetch = global.fetch; + const originalCreateAndStart = ACIBackend.prototype.createAndStart; + const originalRemove = ACIBackend.prototype.remove; + const originalDestroyVolume = ACIBackend.prototype.destroyVolume; + let createCounter = 0; + + global.fetch = async () => ({ ok: true }); + ACIBackend.prototype.createAndStart = async function ({ containerName }) { + createCounter += 1; + return { + containerId: `${containerName}-id`, + url: `http://pool-${createCounter}.test:3100`, + }; + }; + ACIBackend.prototype.remove = async function () {}; + ACIBackend.prototype.destroyVolume = async function () {}; + + const fakeRedis = { + entries: new Map(), + async hset(_key, field, value) { + this.entries.set(field, JSON.parse(value)); + }, + async sadd() {}, + async srem() {}, + }; + + try { + await warmPoolModule.__testables.provisionPoolContainer(fakeRedis); + await warmPoolModule.__testables.provisionPoolContainer(fakeRedis); + + const entries = [...fakeRedis.entries.values()]; + t.is(entries.length, 2); + + const secrets = entries.map(entry => entry.bootstrapSecret); + t.not(secrets[0], secrets[1]); + t.false(secrets.includes('legacy-shared-secret')); + secrets.forEach(secret => t.regex(secret, /^[0-9a-f]{64}$/)); + } finally { + global.fetch = originalFetch; + ACIBackend.prototype.createAndStart = originalCreateAndStart; + ACIBackend.prototype.remove = originalRemove; + ACIBackend.prototype.destroyVolume = originalDestroyVolume; + logCapture.restore(); + restoreConfig(); + } +}); + +test.serial('warmPool prunes READY entries missing from ACI inventory before counting pool size', async (t) => { + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const logCapture = stubLogger(); + const removedReady = []; + const redisEntries = new Map([ + ['workspace-pool-stale123', JSON.stringify({ + status: 'READY', + containerId: 'workspace-pool-stale123', + url: 'http://stale.test:3100', + createdAt: '2026-05-09T21:09:31.615Z', + })], + ['workspace-prod-pool-live123', JSON.stringify({ + status: 'READY', + containerId: 'workspace-prod-pool-live123', + url: 'http://live.test:3100', + createdAt: '2026-05-09T21:50:21.917Z', + })], + ]); + + ACIBackend.prototype.listWorkspaceContainers = async () => [{ + name: 'workspace-prod-pool-live123', + tags: { + workspaceRole: 'pool', + workspaceContainerPrefix: 'workspace-prod', + }, + }]; + + const fakeRedis = { + async hgetall() { + return Object.fromEntries(redisEntries); + }, + async hdel(_key, field) { + const existed = redisEntries.delete(field); + return existed ? 1 : 0; + }, + async srem(_key, field) { + removedReady.push(field); + }, + }; + + try { + const pruned = await warmPoolModule.__testables.pruneStalePoolEntries(fakeRedis); + + t.is(pruned, 1); + t.false(redisEntries.has('workspace-pool-stale123')); + t.true(redisEntries.has('workspace-prod-pool-live123')); + t.deepEqual(removedReady, ['workspace-pool-stale123']); + t.true(logCapture.calls.some(call => + call.level === 'warn' && + call.message.includes('Removed stale READY entry workspace-pool-stale123') + )); + } finally { + ACIBackend.prototype.listWorkspaceContainers = originalList; + logCapture.restore(); + } +}); + +test.serial('warmPool replenishment recovers when stale PROVISIONING entries fill the target size', async (t) => { + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + warmPoolSize: 2, + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '1.0.12', + workspaceCpus: '1', + workspaceMemory: '512m', + workspaceDiskSize: '10g', + }); + const logCapture = stubLogger(); + const originalFetch = global.fetch; + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalCreateAndStart = ACIBackend.prototype.createAndStart; + const originalRemove = ACIBackend.prototype.remove; + const staleCreatedAt = new Date(Date.now() - 10 * 60 * 1000).toISOString(); + const redisEntries = new Map([ + ['workspace-pool-stuck1', JSON.stringify({ + status: 'PROVISIONING', + containerId: 'workspace-pool-stuck1', + createdAt: staleCreatedAt, + })], + ['workspace-pool-stuck2', JSON.stringify({ + status: 'PROVISIONING', + containerId: 'workspace-pool-stuck2', + createdAt: staleCreatedAt, + })], + ]); + const readyMembers = []; + const removedContainers = []; + let lockOwner = null; + let createCounter = 0; + + global.fetch = async () => ({ ok: true }); + ACIBackend.prototype.listWorkspaceContainers = async () => []; + ACIBackend.prototype.remove = async function (_containerId, containerName) { + removedContainers.push(containerName); + }; + ACIBackend.prototype.createAndStart = async function ({ containerName }) { + createCounter += 1; + return { + containerId: `${containerName}-id`, + url: `http://pool-${createCounter}.test:3100`, + }; + }; + + const fakeRedis = { + async hgetall() { + return Object.fromEntries(redisEntries); + }, + async hdel(_key, field) { + const existed = redisEntries.delete(field); + return existed ? 1 : 0; + }, + async srem() {}, + async hset(_key, field, value) { + redisEntries.set(field, value); + return 1; + }, + async sadd(_key, member) { + readyMembers.push(member); + return 1; + }, + async set(_key, value) { + lockOwner = value; + return 'OK'; + }, + async get() { + return lockOwner; + }, + async del() { + lockOwner = null; + return 1; + }, + }; + + try { + await warmPoolModule.__testables.replenish(fakeRedis); + + t.false(redisEntries.has('workspace-pool-stuck1')); + t.false(redisEntries.has('workspace-pool-stuck2')); + t.deepEqual(removedContainers.sort(), ['workspace-pool-stuck1', 'workspace-pool-stuck2']); + t.is(createCounter, 2); + t.is(readyMembers.length, 2); + t.true([...redisEntries.values()].every(raw => JSON.parse(raw).status === 'READY')); + t.true(logCapture.calls.some(call => + call.level === 'warn' && + call.message.includes('Removing stuck PROVISIONING container workspace-pool-stuck1') + )); + } finally { + global.fetch = originalFetch; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.createAndStart = originalCreateAndStart; + ACIBackend.prototype.remove = originalRemove; + logCapture.restore(); + restoreConfig(); + } +}); + +test.serial('warmPool rejects READY pool containers with stale image tags', async (t) => { + const restoreConfig = stubConfig({ + workspaceImageVersion: '1.0.9', + }); + + try { + const result = warmPoolModule.__testables.validatePoolContainerInventoryEntry( + 'workspace-local-pool-stale', + { + available: true, + containersByName: new Map([[ + 'workspace-local-pool-stale', + { + name: 'workspace-local-pool-stale', + tags: { + workspaceRole: 'pool', + imageVersion: '1.0.8', + }, + }, + ]]), + }, + { + status: 'READY', + imageVersion: '1.0.8', + }, + ); + + t.false(result.valid); + t.true(result.removeContainer); + t.is(result.reason, 'stale image (1.0.8 vs 1.0.9)'); + } finally { + restoreConfig(); + } +}); + +test.serial('createGenericContainer generates a unique bootstrap secret', async (t) => { + const restoreConfig = stubConfig({ + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '', + workspaceCpus: '1', + workspaceMemory: '512m', + workspaceDiskSize: '10g', + warmPoolBootstrapSecret: 'legacy-shared-secret', + }); + const originalFetch = global.fetch; + const captured = {}; + + global.fetch = async () => ({ ok: true }); + + const backend = { + backendName: 'docker', + healthTimeoutMs: 1, + async createAndStart(args) { + Object.assign(captured, args); + return { containerId: 'container-123', url: 'http://workspace.test:3100' }; + }, + }; + + try { + const result = await workspaceClientModule.__testables.createGenericContainer('entity-123', backend); + + t.regex(result.bootstrapSecret, /^[0-9a-f]{64}$/); + t.not(result.bootstrapSecret, 'legacy-shared-secret'); + t.true(captured.env.includes(`WORKSPACE_SECRET=${result.bootstrapSecret}`)); + t.false(captured.env.includes('WORKSPACE_SECRET=legacy-shared-secret')); + } finally { + global.fetch = originalFetch; + restoreConfig(); + } +}); + +test.serial('createGenericContainer uses configured workspace container prefix', async (t) => { + const restoreConfig = stubConfig({ + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '', + workspaceCpus: '1', + workspaceMemory: '512m', + workspaceDiskSize: '10g', + workspaceContainerPrefix: 'workspace-dev', + }); + const originalFetch = global.fetch; + const captured = {}; + + global.fetch = async () => ({ ok: true }); + + const backend = { + backendName: 'aci', + healthTimeoutMs: 1, + async createAndStart(args) { + Object.assign(captured, args); + return { containerId: args.containerName, url: 'http://workspace.test:3100' }; + }, + }; + + try { + const result = await workspaceClientModule.__testables.createGenericContainer('entity-123', backend); + + t.is(captured.containerName, 'workspace-dev-entity-123'); + t.is(captured.shareName, null); + t.false(captured.mountAzureFiles); + t.is(captured.tags.workspaceRole, 'entity'); + t.is(captured.tags.entityId, 'entity-123'); + t.is(result.containerName, 'workspace-dev-entity-123'); + } finally { + global.fetch = originalFetch; + restoreConfig(); + } +}); + +test.serial('createGenericContainer retries with unique runtime name on Azure cross-region name conflict', async (t) => { + const restoreConfig = stubConfig({ + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '', + workspaceCpus: '1', + workspaceMemory: '512m', + workspaceDiskSize: '10g', + }); + const originalFetch = global.fetch; + const attempts = []; + + global.fetch = async () => ({ ok: true }); + + const backend = { + backendName: 'aci', + healthTimeoutMs: 1, + async createAndStart(args) { + attempts.push(args); + if (attempts.length === 1) { + throw new Error( + "The resource 'workspace-entity-123' already exists in location 'qatarcentral' in resource group 'Archipelago-ML-Experimentation'. A resource with the same name cannot be created in location 'eastus'. Please select a new resource name." + ); + } + return { + containerId: args.containerName, + url: 'http://workspace.test:3100', + }; + }, + }; + + try { + const result = await workspaceClientModule.__testables.createGenericContainer( + 'entity-123', + backend, + { shareName: 'workspace-entity-123', mountAzureFiles: true }, + ); + + t.is(attempts.length, 2); + t.is(attempts[0].containerName, 'workspace-entity-123'); + t.regex(attempts[1].containerName, /^workspace-entity-123-[0-9a-f]{6}$/); + t.is(attempts[0].shareName, 'workspace-entity-123'); + t.is(attempts[1].shareName, 'workspace-entity-123'); + t.is(result.containerName, attempts[1].containerName); + t.is(result.containerId, attempts[1].containerName); + t.is(result.shareName, 'workspace-entity-123'); + } finally { + global.fetch = originalFetch; + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces destroys old orphan ACI inventory containers', async (t) => { + const now = 10_000_000; + const fakeRedis = createFakeRedis(); + const restoreConfig = stubConfig({ + azureAcrServer: '', + cortexId: 'test-cortex', + storageConnectionString: 'redis://test', + workspaceBackend: 'aci', + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '1.0.7', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + workspaceContainerPrefix: 'workspace-dev', + }); + const restoreEntityStore = stubEntityStore(null); + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalRemove = ACIBackend.prototype.remove; + const originalNow = Date.now; + const removed = []; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = async () => [{ + name: 'workspace-dev-orphan-123', + image: 'cortex-workspace:1.0.7', + tags: { createdAt: new Date(now - 10 * 60 * 1000).toISOString() }, + }]; + ACIBackend.prototype.remove = async (_containerId, containerName) => { + removed.push(containerName); + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.deepEqual(removed, ['workspace-dev-orphan-123']); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.remove = originalRemove; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces does not destroy inventory containers when entity lookup fails', async (t) => { + const now = 10_000_000; + const fakeRedis = createFakeRedis(); + const restoreConfig = stubConfig({ + azureAcrServer: '', + cortexId: 'test-cortex', + storageConnectionString: 'redis://test', + workspaceBackend: 'aci', + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '1.0.7', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + workspaceContainerPrefix: 'workspace-dev', + }); + const restoreEntityStore = stubEntityStore(null); + const entityStore = getEntityStore(); + entityStore.getEntityByWorkspaceContainerId = async () => { + throw new Error('mongo unavailable'); + }; + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalRemove = ACIBackend.prototype.remove; + const originalNow = Date.now; + const logCapture = stubLogger(); + let removeCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = async () => [{ + name: 'workspace-dev-lookup-failure', + image: 'cortex-workspace:1.0.7', + tags: { + createdAt: new Date(now - 10 * 60 * 1000).toISOString(), + entityId: 'entity-lookup-failure', + }, + }]; + ACIBackend.prototype.remove = async () => { + removeCalled = true; + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.false(removeCalled); + const reaperLog = logCapture.calls + .map(call => call.message) + .find(message => message.startsWith('[WorkspaceReaper]')); + t.truthy(reaperLog); + const decision = JSON.parse(reaperLog.slice('[WorkspaceReaper] '.length)); + t.is(decision.action, 'skip'); + t.is(decision.reason, 'entity-lookup-failed'); + t.true(decision.entityLookupFailed); + } finally { + logCapture.restore(); + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.remove = originalRemove; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces uses entityId inventory tag when containerId lookup misses', async (t) => { + const now = 10_000_000; + const entityId = 'entity-tag-fallback'; + const freshActivity = now - 1000; + const fakeRedis = createFakeRedis({ + activityTimestamp: freshActivity, + }); + const restoreConfig = stubConfig({ + azureAcrServer: '', + cortexId: 'test-cortex', + storageConnectionString: 'redis://test', + workspaceBackend: 'aci', + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '1.0.7', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + workspaceContainerPrefix: 'workspace-dev', + }); + const restoreEntityStore = stubEntityStore({ + id: entityId, + workspace: { + containerId: 'workspace-dev-current-container', + status: 'stopped', + checkpointBlobPath: workspaceClientModule.__testables.workspaceCheckpointBlobPath(entityId), + checkpointedAt: new Date(freshActivity).toISOString(), + }, + }); + const entityStore = getEntityStore(); + entityStore.getEntityByWorkspaceContainerId = async () => null; + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalRemove = ACIBackend.prototype.remove; + const originalNow = Date.now; + let removeCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = async () => [{ + name: 'workspace-dev-current-container', + image: 'cortex-workspace:1.0.7', + tags: { + createdAt: new Date(now - 10 * 60 * 1000).toISOString(), + entityId, + }, + }]; + ACIBackend.prototype.remove = async () => { + removeCalled = true; + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.false(removeCalled); + t.true(fakeRedis.calls.some(call => call.op === 'get' && call.key.endsWith(`:activity:${entityId}`))); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.remove = originalRemove; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces treats stale entityId inventory tags as orphan containers', async (t) => { + const now = 10_000_000; + const entityId = 'entity-stale-tag'; + const oldActivity = now - 60 * 60 * 1000; + const staleContainerName = 'workspace-dev-stale-container'; + const currentContainerName = 'workspace-dev-current-container'; + const fakeRedis = createFakeRedis({ + activityTimestamp: oldActivity, + }); + const restoreConfig = stubConfig({ + azureAcrServer: '', + cortexId: 'test-cortex', + storageConnectionString: 'redis://test', + workspaceBackend: 'aci', + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '1.0.7', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + workspaceContainerPrefix: 'workspace-dev', + }); + const restoreEntityStore = stubEntityStore({ + id: entityId, + workspace: { + containerId: currentContainerName, + status: 'stopped', + checkpointBlobPath: workspaceClientModule.__testables.workspaceCheckpointBlobPath(entityId), + checkpointedAt: new Date(oldActivity + 1000).toISOString(), + }, + }); + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalRemove = ACIBackend.prototype.remove; + const originalNow = Date.now; + const removed = []; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = async () => [{ + name: staleContainerName, + image: 'cortex-workspace:1.0.7', + tags: { + createdAt: new Date(now - 10 * 60 * 1000).toISOString(), + entityId, + }, + }]; + ACIBackend.prototype.remove = async (_containerId, containerName) => { + removed.push(containerName); + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.deepEqual(removed, [staleContainerName]); + t.false(fakeRedis.calls.some(call => call.op === 'get' && call.key.endsWith(`:activity:${entityId}`))); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.remove = originalRemove; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces skips active warm-pool inventory containers', async (t) => { + const now = 10_000_000; + const poolName = 'workspace-dev-pool-abc123'; + const fakeRedis = createFakeRedis({ + hashes: { + 'test-cortex-warmpool:containers': { + [poolName]: JSON.stringify({ + status: 'READY', + createdAt: new Date(now - 10 * 60 * 1000).toISOString(), + }), + }, + }, + }); + const restoreConfig = stubConfig({ + azureAcrServer: '', + cortexId: 'test-cortex', + storageConnectionString: 'redis://test', + workspaceBackend: 'aci', + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '1.0.7', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + workspaceContainerPrefix: 'workspace-dev', + }); + const restoreEntityStore = stubEntityStore(null); + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalRemove = ACIBackend.prototype.remove; + const originalNow = Date.now; + let removeCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = async () => [{ + name: poolName, + image: 'cortex-workspace:1.0.7', + tags: { createdAt: new Date(now - 10 * 60 * 1000).toISOString() }, + }]; + ACIBackend.prototype.remove = async () => { + removeCalled = true; + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.false(removeCalled); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.remove = originalRemove; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces ignores stale warm-pool registry for assigned containers', async (t) => { + const now = 10_000_000; + const oldActivity = now - 60 * 60 * 1000; + const containerName = 'workspace-dev-pool-claimed'; + const fakeRedis = createFakeRedis({ + activityTimestamp: oldActivity, + hashes: { + 'test-cortex-warmpool:containers': { + [containerName]: JSON.stringify({ + status: 'READY', + createdAt: new Date(now - 2 * 60 * 60 * 1000).toISOString(), + }), + }, + }, + }); + const entity = { + id: 'entity-assigned', + workspace: { + containerId: containerName, + status: 'stopped', + checkpointBlobPath: 'workspace-checkpoints/test/entity-assigned/workspace.tar.gz', + checkpointedAt: new Date(oldActivity + 1000).toISOString(), + }, + }; + const restoreConfig = stubConfig({ + azureAcrServer: '', + cortexId: 'test-cortex', + storageConnectionString: 'redis://test', + workspaceBackend: 'aci', + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '1.0.7', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + workspaceContainerPrefix: 'workspace-dev', + }); + const restoreEntityStore = stubEntityStore(entity); + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalRemove = ACIBackend.prototype.remove; + const originalNow = Date.now; + const removed = []; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = async () => [{ + name: containerName, + image: 'cortex-workspace:1.0.7', + tags: { createdAt: new Date(now - 2 * 60 * 60 * 1000).toISOString() }, + }]; + ACIBackend.prototype.remove = async (_containerId, name) => { + removed.push(name); + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.deepEqual(removed, [containerName]); + t.true(fakeRedis.calls.some(call => + call.op === 'hdel' && + call.key === 'test-cortex-warmpool:containers' && + call.field === containerName + )); + t.true(fakeRedis.calls.some(call => + call.op === 'srem' && + call.key === 'test-cortex-warmpool:ready' && + call.member === containerName + )); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.remove = originalRemove; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces skips prefix-matching containers without Cortex ownership proof', async (t) => { + const now = 10_000_000; + const fakeRedis = createFakeRedis(); + const restoreConfig = stubConfig({ + azureAcrServer: '', + cortexId: 'test-cortex', + storageConnectionString: 'redis://test', + workspaceBackend: 'aci', + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '1.0.7', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + workspaceContainerPrefix: 'workspace', + }); + const restoreEntityStore = stubEntityStore(null); + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalRemove = ACIBackend.prototype.remove; + const originalNow = Date.now; + let removeCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = async () => [{ + name: 'workspace-foo', + image: 'ubuntu:latest', + tags: { createdAt: new Date(now - 10 * 60 * 1000).toISOString() }, + }]; + ACIBackend.prototype.remove = async () => { + removeCalled = true; + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.false(removeCalled); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.remove = originalRemove; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('createGenericContainer does not invent ACI share names when first container name conflicts', async (t) => { + const restoreConfig = stubConfig({ + workspaceImage: 'cortex-workspace', + workspaceImageVersion: '', + workspaceCpus: '1', + workspaceMemory: '512m', + workspaceDiskSize: '10g', + }); + const originalFetch = global.fetch; + const attempts = []; + + global.fetch = async () => ({ ok: true }); + + const backend = { + backendName: 'aci', + healthTimeoutMs: 1, + async createAndStart(args) { + attempts.push(args); + if (attempts.length === 1) { + throw new Error( + "The resource 'workspace-entity-456' already exists in location 'qatarcentral' in resource group 'Archipelago-ML-Experimentation'. A resource with the same name cannot be created in location 'eastus'. Please select a new resource name." + ); + } + return { + containerId: args.containerName, + url: 'http://workspace.test:3100', + }; + }, + }; + + try { + const result = await workspaceClientModule.__testables.createGenericContainer('entity-456', backend); + + t.is(attempts.length, 2); + t.is(attempts[0].shareName, null); + t.is(attempts[1].shareName, null); + t.is(result.shareName, null); + } finally { + global.fetch = originalFetch; + restoreConfig(); + } +}); + +test.serial('workspaceRequest waits for an in-progress workspace transition before sending request', async (t) => { + const entityId = 'entity-transition-wait'; + const store = stubMutableEntityStore({ + id: entityId, + name: 'Transition Wait Entity', + workspace: { + status: 'provisioning', + provisionedAt: new Date(), + }, + }); + const originalFetch = global.fetch; + let fetchUrl = null; + const lifecycleEvents = []; + + global.fetch = async (url, options) => { + fetchUrl = url; + t.is(options.headers['x-workspace-secret'], 'secret-ready'); + return { + ok: true, + status: 200, + async json() { + return { status: 'ok' }; + }, + }; + }; + + const timer = setTimeout(() => { + store.setEntity({ + id: entityId, + name: 'Transition Wait Entity', + workspace: { + status: 'running', + url: 'http://workspace-ready.test:3100', + secret: 'secret-ready', + provisionedAt: new Date(), + }, + }); + }, 25); + + try { + const result = await workspaceClientModule.workspaceRequest(entityId, '/health', null, { + transitionWaitMs: 500, + transitionPollMs: 10, + onWorkspaceLifecycle(event) { + lifecycleEvents.push(event); + }, + }); + + t.true(result.success); + t.is(result.status, 'ok'); + t.is(fetchUrl, 'http://workspace-ready.test:3100/health'); + t.deepEqual( + lifecycleEvents.map(({ type, phase, message, success }) => ({ type, phase, message, success })), + [ + { + type: 'start', + phase: 'provision', + message: 'Waiting for workspace setup', + success: undefined, + }, + { + type: 'finish', + phase: 'provision', + message: undefined, + success: true, + }, + ], + ); + } finally { + clearTimeout(timer); + global.fetch = originalFetch; + store.restore(); + } +}); + +test.serial('workspaceRequest waits when another worker owns the provisioning lock', async (t) => { + const entityId = 'entity-peer-provision'; + const store = stubMutableEntityStore({ + id: entityId, + name: 'Peer Provision Entity', + workspace: null, + }); + const fakeRedis = createFakeRedis({ acquireLock: false }); + const originalFetch = global.fetch; + let fetchUrl = null; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + + global.fetch = async (url, options) => { + fetchUrl = url; + t.is(options.headers['x-workspace-secret'], 'peer-secret'); + return { + ok: true, + status: 200, + async json() { + return { status: 'ok' }; + }, + }; + }; + + const timer = setTimeout(() => { + store.setEntity({ + id: entityId, + name: 'Peer Provision Entity', + workspace: { + status: 'running', + url: 'http://peer-ready.test:3100', + secret: 'peer-secret', + provisionedAt: new Date(), + }, + }); + }, 25); + + try { + const result = await workspaceClientModule.workspaceRequest(entityId, '/health', null, { + transitionWaitMs: 500, + transitionPollMs: 10, + }); + + t.true(result.success); + t.is(fetchUrl, 'http://peer-ready.test:3100/health'); + t.true(fakeRedis.calls.some(call => + call.op === 'set' && + call.key.includes('workspace:provisioning-lock:entity-peer-provision') && + call.args.includes('NX') + )); + } finally { + clearTimeout(timer); + global.fetch = originalFetch; + workspaceClientModule.__testables.resetActivityStateForTest(); + store.restore(); + } +}); + +test.serial('reconfigureForEntity skips cleanup for existing workspaces when destroyOnFailure is false', async (t) => { + const originalFetch = global.fetch; + let removeCalled = false; + + global.fetch = async () => { + throw new Error('fetch failed'); + }; + + const backend = { + backendName: 'docker', + async remove() { + removeCalled = true; + }, + }; + + try { + await t.throwsAsync( + () => workspaceClientModule.__testables.reconfigureForEntity( + 'entity-123', + { secrets: null }, + { + containerName: 'workspace-entity-123', + shareName: 'workspace-entity-123', + url: 'http://wrong-host:3100', + bootstrapSecret: 'bootstrap-secret', + containerId: 'workspace-entity-123', + claimedFromPool: false, + }, + backend, + { destroyOnFailure: false }, + ), + { message: 'fetch failed' }, + ); + + t.false(removeCalled); + } finally { + global.fetch = originalFetch; + } +}); + +test.serial('reconfigureForEntity still cleans up failed disposable containers by default', async (t) => { + const originalFetch = global.fetch; + const removeCalls = []; + + global.fetch = async () => { + throw new Error('fetch failed'); + }; + + const backend = { + backendName: 'docker', + async remove(containerId, containerName) { + removeCalls.push({ containerId, containerName }); + }, + }; + + try { + await t.throwsAsync( + () => workspaceClientModule.__testables.reconfigureForEntity( + 'entity-123', + { secrets: null }, + { + containerName: 'workspace-entity-123', + shareName: 'workspace-entity-123', + url: 'http://wrong-host:3100', + bootstrapSecret: 'bootstrap-secret', + containerId: 'workspace-entity-123', + claimedFromPool: false, + }, + backend, + ), + { message: 'fetch failed' }, + ); + + t.deepEqual(removeCalls, [{ + containerId: 'workspace-entity-123', + containerName: 'workspace-entity-123', + }]); + } finally { + global.fetch = originalFetch; + } +}); + +test.serial('reconfigureForEntity records the actual claimed container image version', async (t) => { + const originalFetch = global.fetch; + const restoreConfig = stubConfig({ + workspaceImageVersion: '1.0.9', + }); + const store = stubMutableEntityStore({ + id: 'entity-claimed-stale-image', + secrets: null, + }); + + global.fetch = async (url, options = {}) => { + t.is(url, 'http://workspace.test:3100/reconfigure'); + t.is(options.headers?.['x-workspace-secret'], 'bootstrap-secret'); + return { + ok: true, + status: 200, + async json() { + return { success: true }; + }, + }; + }; + + try { + await workspaceClientModule.__testables.reconfigureForEntity( + 'entity-claimed-stale-image', + store.getEntity(), + { + containerName: 'workspace-local-pool-stale', + url: 'http://workspace.test:3100', + bootstrapSecret: 'bootstrap-secret', + containerId: 'workspace-local-pool-stale', + claimedFromPool: true, + imageVersion: '1.0.8', + }, + { backendName: 'docker' }, + ); + + t.is(store.getEntity().workspace.imageVersion, '1.0.8'); + t.true(store.getEntity().workspace.claimedFromPool); + } finally { + global.fetch = originalFetch; + store.restore(); + restoreConfig(); + } +}); + +test.serial('ACIBackend stop calls Azure container group stop', async (t) => { + const restoreConfig = stubConfig({ + azureResourceGroup: 'test-rg', + }); + const backend = new ACIBackend(); + const calls = []; + + backend._getClient = async () => ({ + containerGroups: { + async stop(resourceGroup, groupName) { + calls.push({ resourceGroup, groupName }); + }, + }, + }); + + try { + await backend.stop('container-id', 'workspace-entity-123'); + + t.deepEqual(calls, [{ + resourceGroup: 'test-rg', + groupName: 'workspace-entity-123', + }]); + } finally { + restoreConfig(); + } +}); + +test.serial('ACIBackend start waits for Azure start and returns current private-IP URL', async (t) => { + const restoreConfig = stubConfig({ + azureResourceGroup: 'test-rg', + }); + const backend = new ACIBackend(); + const calls = []; + + backend._getClient = async () => ({ + containerGroups: { + async beginStart(resourceGroup, groupName) { + calls.push({ method: 'beginStart', resourceGroup, groupName }); + return { + async pollUntilDone() { + calls.push({ method: 'pollUntilDone' }); + }, + }; + }, + async get(resourceGroup, groupName) { + calls.push({ method: 'get', resourceGroup, groupName }); + return { + ipAddress: { + ip: '10.1.2.3', + }, + }; + }, + }, + }); + + try { + const result = await backend.start('container-id', 'workspace-entity-123'); + + t.is(result.url, 'http://10.1.2.3:3100'); + t.deepEqual(calls, [ + { method: 'beginStart', resourceGroup: 'test-rg', groupName: 'workspace-entity-123' }, + { method: 'pollUntilDone' }, + { method: 'get', resourceGroup: 'test-rg', groupName: 'workspace-entity-123' }, + ]); + } finally { + restoreConfig(); + } +}); + +test.serial('ACIBackend wake timeout matches create timeout for real starts', (t) => { + const backend = new ACIBackend(); + + t.is(backend.wakeHealthTimeoutMs, backend.healthTimeoutMs); +}); + +test.serial('ACIBackend createAndStart requires private subnet outside local environments', async (t) => { + const restoreConfig = stubConfig({ + env: 'production', + aciSubnetId: '', + azureResourceGroup: 'test-rg', + }); + const backend = new ACIBackend(); + + backend._getClient = async () => { + t.fail('createAndStart should fail before creating an Azure client'); + }; + + try { + await t.throwsAsync( + () => backend.createAndStart({ + containerName: 'workspace-entity-123', + image: 'cortex-workspace:latest', + env: ['WORKSPACE_SECRET=test'], + cpus: 1, + memoryMB: 512, + diskSize: '10g', + }), + { message: 'ACI_SUBNET_ID is required for ACI workspaces outside development/test/local environments' }, + ); + } finally { + restoreConfig(); + } +}); + +test.serial('ACIBackend createAndStart allows public local debug workspaces without subnet', async (t) => { + const restoreConfig = stubConfig({ + env: 'debug', + aciSubnetId: '', + azureResourceGroup: 'test-rg', + azureLocation: 'eastus', + azureAcrServer: '', + cortexId: 'test-cortex', + workspaceContainerPrefix: 'workspace-local', + workspaceImageVersion: '1.0.9', + }); + const backend = new ACIBackend(); + let createCall = null; + + backend._getClient = async () => ({ + containerGroups: { + async beginCreateOrUpdate(resourceGroup, containerName, definition) { + createCall = { resourceGroup, containerName, definition }; + return { + async pollUntilDone() { + return { ipAddress: { ip: '1.2.3.4' } }; + }, + }; + }, + }, + }); + + try { + const result = await backend.createAndStart({ + containerName: 'workspace-local-entity-123', + image: 'cortex-workspace:1.0.9', + env: ['WORKSPACE_SECRET=test'], + cpus: 1, + memoryMB: 512, + diskSize: '10g', + }); + + t.is(result.url, 'http://1.2.3.4:3100'); + t.is(createCall.definition.restartPolicy, 'Never'); + t.is(createCall.definition.ipAddress.type, 'Public'); + t.false(Object.hasOwn(createCall.definition, 'subnetIds')); + } finally { + restoreConfig(); + } +}); + +test.serial('workspaceUploadFile wakes stopped ACI workspace before streaming upload', async (t) => { + const restoreConfig = stubConfig({ + workspaceBackend: 'aci', + workspaceImageVersion: '', + redisEncryptionKey: 'test-key', + }); + const restoreEntityStore = stubEntityStore({ + id: 'entity-123', + secrets: {}, + workspace: { + url: 'http://stopped.test:3100', + secret: 'secret-123', + bootstrapSecret: 'bootstrap-secret', + containerId: 'workspace-entity-123', + shareName: 'workspace-entity-123', + status: 'stopped', + }, + }); + const originalFetch = global.fetch; + const originalStart = ACIBackend.prototype.start; + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'workspace-upload-wake-')); + const tempFile = path.join(tempDir, 'archive.tar.gz'); + const fetchCalls = []; + const startCalls = []; + let rotatedSecret = null; + + fs.writeFileSync(tempFile, 'backup'); + + ACIBackend.prototype.start = async function (containerId, containerName) { + startCalls.push({ containerId, containerName }); + return { url: 'http://woken.test:3100' }; + }; + + global.fetch = async (url, options = {}) => { + fetchCalls.push({ url, method: options.method || 'GET' }); + const { pathname, hostname } = new URL(url); + + if (pathname === '/health') { + t.is(hostname, 'woken.test'); + return { ok: true }; + } + + if (pathname === '/reconfigure') { + t.is(hostname, 'woken.test'); + t.is(options.headers['x-workspace-secret'], 'bootstrap-secret'); + const body = JSON.parse(options.body); + t.regex(body.secret, /^[0-9a-f]{64}$/); + t.not(body.secret, 'secret-123'); + t.deepEqual(body.env, {}); + rotatedSecret = body.secret; + return { + ok: true, + status: 200, + async json() { + return { success: true }; + }, + }; + } + + if (pathname === '/upload') { + t.is(hostname, 'woken.test'); + t.truthy(rotatedSecret); + t.is(options.headers['x-workspace-secret'], rotatedSecret); + return { + ok: true, + async json() { + return { bytesWritten: 6 }; + }, + }; + } + + throw new Error(`unexpected fetch: ${url}`); + }; + + try { + const result = await workspaceClientModule.workspaceUploadFile( + 'entity-123', + tempFile, + '/tmp/workspace-restore.tar.gz', + ); + + t.deepEqual(result, { success: true, bytesWritten: 6 }); + t.deepEqual(startCalls, [{ + containerId: 'workspace-entity-123', + containerName: 'workspace-entity-123', + }]); + t.true(fetchCalls.some(call => call.url === 'http://woken.test:3100/reconfigure')); + t.true(fetchCalls.some(call => call.url === 'http://woken.test:3100/upload?path=%2Ftmp%2Fworkspace-restore.tar.gz')); + t.false(fetchCalls.some(call => call.url === 'http://woken.test:3100/write')); + t.false(fetchCalls.some(call => call.url === 'http://woken.test:3100/shell')); + t.false(fetchCalls.some(call => call.url.startsWith('http://stopped.test:3100/upload'))); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + ACIBackend.prototype.start = originalStart; + global.fetch = originalFetch; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('hasRunningBackgroundJobs detects running workspace jobs', async (t) => { + const originalFetch = global.fetch; + + global.fetch = async (url, options) => { + t.is(url, 'http://workspace.test:3100/shell/jobs'); + t.is(options.headers['x-workspace-secret'], 'secret-123'); + return { + ok: true, + async json() { + return { + jobs: [ + { processId: 'done', status: 'completed' }, + { processId: 'active', status: 'running' }, + ], + }; + }, + }; + }; + + try { + const result = await workspaceClientModule.__testables.hasRunningBackgroundJobs({ + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-123', + }, + }); + + t.true(result); + } finally { + global.fetch = originalFetch; + } +}); + +test.serial('hasRunningBackgroundJobs accepts workspace /shell/jobs array response', async (t) => { + const originalFetch = global.fetch; + + global.fetch = async (url, options) => { + t.is(url, 'http://workspace.test:3100/shell/jobs'); + t.is(options.headers['x-workspace-secret'], 'secret-123'); + return { + ok: true, + status: 200, + async json() { + return [ + { processId: 'done', status: 'completed' }, + { processId: 'active', status: 'running' }, + ]; + }, + }; + }; + + try { + const details = await workspaceClientModule.__testables.getWorkspaceBackgroundJobsStatus({ + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-123', + }, + }); + + t.true(details.hasRunningJobs); + t.true(details.ok); + t.is(details.responseType, 'array'); + t.is(details.runningJobCount, 1); + t.deepEqual(details.jobs.map(job => job.processId), ['done', 'active']); + } finally { + global.fetch = originalFetch; + } +}); + +test.serial('getWorkspaceBackgroundJobsStatus recovers bootstrap auth before checking jobs', async (t) => { + const originalFetch = global.fetch; + const entityId = 'entity-background-jobs-auth-recovery'; + const calls = []; + let freshSecret = null; + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'stale-secret', + bootstrapSecret: 'bootstrap-secret', + containerId: 'workspace-entity-background-jobs-auth-recovery', + status: 'running', + }, + }); + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push({ + url: urlString, + secret: options.headers?.['x-workspace-secret'], + hasDispatcher: Boolean(options.dispatcher), + }); + if (urlString.endsWith('/shell/jobs') && options.headers?.['x-workspace-secret'] === 'stale-secret') { + return { + ok: false, + status: 401, + statusText: 'Unauthorized', + async json() { + return { error: 'Invalid secret' }; + }, + }; + } + if (urlString.endsWith('/reconfigure')) { + t.is(options.headers?.['x-workspace-secret'], 'bootstrap-secret'); + const body = JSON.parse(options.body); + t.truthy(body.secret); + freshSecret = body.secret; + return { + ok: true, + status: 200, + async json() { + return { success: true }; + }, + }; + } + if (urlString.endsWith('/shell/jobs')) { + t.is(options.headers?.['x-workspace-secret'], freshSecret); + return { + ok: true, + status: 200, + async json() { + return []; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const details = await workspaceClientModule.__testables.getWorkspaceBackgroundJobsStatus(store.getEntity()); + + t.true(details.ok); + t.false(details.hasRunningJobs); + t.true(details.authRecovered); + t.is(details.reason, 'no-running-jobs'); + t.is(details.runningJobCount, 0); + t.truthy(freshSecret); + t.deepEqual(calls.map(call => call.url), [ + 'http://workspace.test:3100/shell/jobs', + 'http://workspace.test:3100/reconfigure', + 'http://workspace.test:3100/shell/jobs', + ]); + t.is(calls[0].secret, 'stale-secret'); + t.is(calls[1].secret, 'bootstrap-secret'); + t.is(calls[2].secret, freshSecret); + } finally { + global.fetch = originalFetch; + workspaceClientModule.__testables.resetActivityStateForTest(); + store.restore(); + } +}); + +test.serial('hasRunningBackgroundJobs allows stop when only completed jobs remain', async (t) => { + const originalFetch = global.fetch; + + global.fetch = async () => ({ + ok: true, + async json() { + return { + jobs: [ + { processId: 'done', status: 'completed' }, + { processId: 'failed', status: 'failed' }, + ], + }; + }, + }); + + try { + const result = await workspaceClientModule.__testables.hasRunningBackgroundJobs({ + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-123', + }, + }); + + t.false(result); + } finally { + global.fetch = originalFetch; + } +}); + +test.serial('hasRunningBackgroundJobs fails closed when jobs check fails', async (t) => { + const originalFetch = global.fetch; + const logCapture = stubLogger(); + + global.fetch = async () => ({ + ok: false, + status: 500, + }); + + try { + const result = await workspaceClientModule.__testables.hasRunningBackgroundJobs({ + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-123', + }, + }); + + t.true(result); + t.true(logCapture.calls.some(call => call.level === 'warn')); + } finally { + logCapture.restore(); + global.fetch = originalFetch; + } +}); + +test.serial('reapIdleWorkspaces honors newer Redis activity from another worker', async (t) => { + const now = 10_000_000; + const idleTimeoutMs = 30 * 60 * 1000; + const entityId = 'entity-123'; + const redisLastActivity = now - 60_000; + const localStaleActivity = now - idleTimeoutMs - 1; + const fakeRedis = createFakeRedis({ + activityTimestamp: redisLastActivity, + activityIndex: { [entityId]: localStaleActivity }, + }); + + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: 'redis://test', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: idleTimeoutMs, + }); + const restoreEntityStore = stubEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-123', + containerId: 'workspace-entity-123', + shareName: 'workspace-entity-123', + status: 'running', + }, + }); + const originalFetch = global.fetch; + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalStop = ACIBackend.prototype.stop; + const originalNow = Date.now; + let stopCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + workspaceClientModule.__testables.lastActivity.set(entityId, localStaleActivity); + + Date.now = () => now; + global.fetch = async () => { + t.fail('reaper should not check background jobs when Redis activity is still fresh'); + }; + ACIBackend.prototype.listWorkspaceContainers = undefined; + ACIBackend.prototype.stop = async () => { + stopCalled = true; + return {}; + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.false(stopCalled); + t.is(workspaceClientModule.__testables.lastActivity.get(entityId), redisLastActivity); + t.true(fakeRedis.calls.some(call => call.op === 'set' && call.key.endsWith(':reaper-lock:entity-123'))); + t.true(fakeRedis.calls.some(call => call.op === 'del' && call.key.endsWith(':reaper-lock:entity-123'))); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.stop = originalStop; + global.fetch = originalFetch; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces uses local activity when Redis is not configured', async (t) => { + const now = 10_000_000; + const idleTimeoutMs = 30 * 60 * 1000; + const entityId = 'entity-local-reaper'; + const staleActivity = now - idleTimeoutMs - 1; + + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: idleTimeoutMs, + }); + const restoreEntityStore = stubEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-local', + containerId: 'workspace-entity-local-reaper', + shareName: 'workspace-entity-local-reaper', + status: 'running', + }, + }); + const originalFetch = global.fetch; + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalRemove = ACIBackend.prototype.remove; + const originalNow = Date.now; + let removeCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.lastActivity.set(entityId, staleActivity); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = undefined; + global.fetch = async (url) => { + const urlString = String(url); + if (urlString.endsWith('/shell/jobs')) { + return { + ok: true, + async json() { + return { jobs: [] }; + }, + }; + } + if (urlString.endsWith('/health')) { + return { + ok: true, + status: 200, + async json() { + return { version: '1.0.2' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + status: 200, + async json() { + return { version: '1.0.2' }; + }, + }; + } + t.fail(`unexpected fetch: ${urlString}`); + }; + ACIBackend.prototype.remove = async (name) => { + removeCalled = true; + t.is(name, 'workspace-entity-local-reaper'); + return {}; + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.true(removeCalled); + t.false(workspaceClientModule.__testables.lastActivity.has(entityId)); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.remove = originalRemove; + global.fetch = originalFetch; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces checkpoints maintenance-idle ACI workspace before reap timeout', async (t) => { + const now = 10_000_000; + const idleTimeoutMs = 30 * 60 * 1000; + const checkpointIdleMs = 5 * 60 * 1000; + const entityId = 'entity-maintenance-checkpoint'; + const lastActivityAt = now - checkpointIdleMs - 1; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: idleTimeoutMs, + workspaceIdleCheckpointMs: checkpointIdleMs, + }); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-maintenance', + containerId: 'workspace-entity-maintenance-checkpoint', + status: 'running', + imageVersion: '1.0.6', + }, + }); + const originalFetch = global.fetch; + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalRemove = ACIBackend.prototype.remove; + const originalNow = Date.now; + let removeCalled = false; + let uploadCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.lastActivity.set(entityId, lastActivityAt); + workspaceClientModule.__testables.setWorkspaceCheckpointUploadForTest(async () => { + uploadCalled = true; + return { + blobPath: 'workspace-checkpoints/test-cortex/entity-maintenance-checkpoint/workspace.tar.gz', + sizeBytes: 1024, + sizeMB: 0.01, + timestamp: new Date(now).toISOString(), + }; + }); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = undefined; + ACIBackend.prototype.remove = async () => { + removeCalled = true; + }; + global.fetch = async (url) => { + const urlString = String(url); + if (urlString.endsWith('/shell/jobs')) { + return { + ok: true, + async json() { + return { jobs: [] }; + }, + }; + } + if (urlString.endsWith('/health')) { + return { + ok: true, + async json() { + return { version: '1.0.6' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + async json() { + return { uptime: 123 }; + }, + }; + } + if (urlString.endsWith('/backup')) { + return { + ok: true, + async json() { + return { + path: '/persist/workspace.tar.gz', + sizeMB: 0.01, + timestamp: new Date(now).toISOString(), + }; + }, + }; + } + t.fail(`unexpected fetch: ${urlString}`); + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.true(uploadCalled); + t.false(removeCalled); + t.is( + store.getEntity().workspace.checkpointBlobPath, + 'workspace-checkpoints/test-cortex/entity-maintenance-checkpoint/workspace.tar.gz', + ); + t.true(workspaceClientModule.__testables.lastActivity.has(entityId)); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.remove = originalRemove; + global.fetch = originalFetch; + store.restore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces checkpoints maintenance-idle ACI inventory workspace before reap timeout', async (t) => { + const now = 10_000_000; + const idleTimeoutMs = 30 * 60 * 1000; + const checkpointIdleMs = 5 * 60 * 1000; + const entityId = 'entity-inventory-checkpoint'; + const containerName = 'workspace-entity-inventory-checkpoint'; + const lastActivityAt = now - checkpointIdleMs - 1; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: idleTimeoutMs, + workspaceIdleCheckpointMs: checkpointIdleMs, + workspaceContainerPrefix: 'workspace', + }); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-inventory', + containerId: containerName, + status: 'running', + imageVersion: '1.0.6', + }, + }); + const originalFetch = global.fetch; + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalRemove = ACIBackend.prototype.remove; + const originalNow = Date.now; + let removeCalled = false; + let uploadCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.lastActivity.set(entityId, lastActivityAt); + workspaceClientModule.__testables.setWorkspaceCheckpointUploadForTest(async () => { + uploadCalled = true; + return { + blobPath: 'workspace-checkpoints/test-cortex/entity-inventory-checkpoint/workspace.tar.gz', + sizeBytes: 2048, + sizeMB: 0.02, + timestamp: new Date(now).toISOString(), + }; + }); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = async () => [{ + name: containerName, + createdAt: new Date(now - 10 * 60 * 1000).toISOString(), + tags: { + managedBy: 'cortex', + workspaceContainerPrefix: 'workspace', + entityId, + }, + }]; + ACIBackend.prototype.remove = async () => { + removeCalled = true; + }; + global.fetch = async (url) => { + const urlString = String(url); + if (urlString.endsWith('/shell/jobs')) { + return { + ok: true, + async json() { + return { jobs: [] }; + }, + }; + } + if (urlString.endsWith('/health')) { + return { + ok: true, + async json() { + return { version: '1.0.6' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + async json() { + return { uptime: 123 }; + }, + }; + } + if (urlString.endsWith('/backup')) { + return { + ok: true, + async json() { + return { + path: '/persist/workspace.tar.gz', + sizeMB: 0.02, + timestamp: new Date(now).toISOString(), + }; + }, + }; + } + t.fail(`unexpected fetch: ${urlString}`); + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.true(uploadCalled); + t.false(removeCalled); + t.is( + store.getEntity().workspace.checkpointBlobPath, + 'workspace-checkpoints/test-cortex/entity-inventory-checkpoint/workspace.tar.gz', + ); + t.true(workspaceClientModule.__testables.lastActivity.has(entityId)); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.remove = originalRemove; + global.fetch = originalFetch; + store.restore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces reaps ACI workspace without backup when checkpoint is fresh', async (t) => { + const now = 10_000_000; + const idleTimeoutMs = 30 * 60 * 1000; + const entityId = 'entity-fresh-checkpoint-reap'; + const lastActivityAt = now - idleTimeoutMs - 1; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: idleTimeoutMs, + workspaceIdleCheckpointMs: 5 * 60 * 1000, + }); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-fresh-checkpoint', + containerId: 'workspace-entity-fresh-checkpoint-reap', + status: 'running', + checkpointBlobPath: 'workspace-checkpoints/test-cortex/entity-fresh-checkpoint-reap/workspace.tar.gz', + checkpointedAt: new Date(now - 60_000).toISOString(), + }, + }); + const originalFetch = global.fetch; + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalRemove = ACIBackend.prototype.remove; + const originalNow = Date.now; + let removeCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.lastActivity.set(entityId, lastActivityAt); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = undefined; + ACIBackend.prototype.remove = async (name) => { + removeCalled = true; + t.is(name, 'workspace-entity-fresh-checkpoint-reap'); + }; + global.fetch = async (url) => { + const urlString = String(url); + if (urlString.endsWith('/shell/jobs')) { + return { + ok: true, + async json() { + return { jobs: [] }; + }, + }; + } + t.fail(`unexpected fetch: ${urlString}`); + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.true(removeCalled); + t.false(workspaceClientModule.__testables.lastActivity.has(entityId)); + t.is( + store.getEntity().workspace.checkpointBlobPath, + 'workspace-checkpoints/test-cortex/entity-fresh-checkpoint-reap/workspace.tar.gz', + ); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.remove = originalRemove; + global.fetch = originalFetch; + store.restore(); + restoreConfig(); + } +}); + +test.serial('destroyWorkspace destroyVolume deletes checkpoint blobs before clearing workspace', async (t) => { + const entityId = 'entity-destroy-checkpoint'; + const containerName = 'workspace-entity-destroy-checkpoint'; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + workspaceAzureFilesStorageAccountName: '', + workspaceAzureFilesStorageAccountKey: '', + }); + const checkpointBlobPath = workspaceClientModule.__testables.workspaceCheckpointBlobPath(entityId); + const checkpointPreviousBlobPath = workspaceClientModule.__testables.workspaceCheckpointBlobPath(entityId, 'workspace.prev.tar.gz'); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + containerId: containerName, + status: 'running', + url: 'http://workspace.test:3100', + secret: 'workspace-secret', + checkpointBlobPath, + checkpointPreviousBlobPath, + checkpointedAt: new Date().toISOString(), + }, + }); + const originalRemove = ACIBackend.prototype.remove; + const originalDestroyVolume = ACIBackend.prototype.destroyVolume; + const removedContainers = []; + const destroyedVolumes = []; + const deletedBlobs = []; + + ACIBackend.prototype.remove = async function (containerId, requestedName) { + removedContainers.push({ containerId, requestedName }); + }; + ACIBackend.prototype.destroyVolume = async function (shareName) { + destroyedVolumes.push(shareName); + }; + workspaceClientModule.__testables.setWorkspaceCheckpointContainerClientForTest({ + getBlockBlobClient(blobPath) { + return { + async deleteIfExists() { + deletedBlobs.push(blobPath); + return { succeeded: true }; + }, + }; + }, + }); + + try { + const result = await workspaceClientModule.destroyWorkspace(entityId, store.getEntity(), { + destroyVolume: true, + }); + + t.true(result.success); + t.deepEqual(removedContainers, [{ containerId: containerName, requestedName: containerName }]); + t.deepEqual(destroyedVolumes, []); + t.deepEqual(deletedBlobs.sort(), [checkpointPreviousBlobPath, checkpointBlobPath].sort()); + t.is(store.getEntity().workspace, null); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + ACIBackend.prototype.remove = originalRemove; + ACIBackend.prototype.destroyVolume = originalDestroyVolume; + store.restore(); + restoreConfig(); + } +}); + +test.serial('destroyWorkspace applies caller timeout to pre-destroy checkpoint', async (t) => { + const entityId = 'entity-checkpoint-timeout'; + const containerName = 'workspace-entity-checkpoint-timeout'; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + }); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + containerId: containerName, + status: 'running', + url: 'http://workspace.test:3100', + secret: 'workspace-secret', + }, + }); + const originalFetch = global.fetch; + const originalRemove = ACIBackend.prototype.remove; + let removeCalled = false; + + ACIBackend.prototype.remove = async () => { + removeCalled = true; + }; + global.fetch = async (_url, options = {}) => new Promise((resolve, reject) => { + options.signal?.addEventListener('abort', () => { + reject(new Error('aborted by test signal')); + }); + }); + + try { + const startedAt = Date.now(); + const result = await workspaceClientModule.destroyWorkspace(entityId, store.getEntity(), { + timeoutMs: 20, + }); + + t.false(result.success); + t.regex(result.error, /Checkpoint failed: \/health fetch failed: aborted by test signal/); + t.true(Date.now() - startedAt < 1000); + t.false(removeCalled); + } finally { + global.fetch = originalFetch; + ACIBackend.prototype.remove = originalRemove; + store.restore(); + restoreConfig(); + } +}); + +test.serial('destroyWorkspace skips checkpoint when stored checkpoint is fresh against tracked activity', async (t) => { + const entityId = 'entity-fresh-manual-destroy'; + const containerName = 'workspace-entity-fresh-manual-destroy'; + const now = Date.now(); + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + workspaceAzureFilesStorageAccountName: '', + workspaceAzureFilesStorageAccountKey: '', + }); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + containerId: containerName, + status: 'running', + url: 'http://workspace.test:3100', + secret: 'workspace-secret', + checkpointBlobPath: 'workspace-checkpoints/test-cortex/entity-fresh-manual-destroy/workspace.tar.gz', + checkpointedAt: new Date(now).toISOString(), + }, + }); + const originalFetch = global.fetch; + const originalRemove = ACIBackend.prototype.remove; + const removedContainers = []; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.lastActivity.set(entityId, now - 1000); + ACIBackend.prototype.remove = async function (containerId, requestedName) { + removedContainers.push({ containerId, requestedName }); + }; + global.fetch = async (url) => { + t.fail(`unexpected checkpoint fetch: ${url}`); + }; + + try { + const result = await workspaceClientModule.destroyWorkspace(entityId, store.getEntity()); + + t.true(result.success); + t.deepEqual(removedContainers, [{ containerId: containerName, requestedName: containerName }]); + t.is( + store.getEntity().workspace.checkpointBlobPath, + 'workspace-checkpoints/test-cortex/entity-fresh-manual-destroy/workspace.tar.gz', + ); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + ACIBackend.prototype.remove = originalRemove; + global.fetch = originalFetch; + store.restore(); + restoreConfig(); + } +}); + +test.serial('destroyWorkspace recovers fresh checkpoint metadata from Blob before checkpointing', async (t) => { + const entityId = 'entity-fresh-blob-manual-destroy'; + const containerName = 'workspace-entity-fresh-blob-manual-destroy'; + const lastActivityAt = Date.parse('2026-05-24T02:38:19.000Z'); + const blobCheckpointedAt = '2026-05-24T02:44:03.690Z'; + const checkpointBlobPath = 'workspace-checkpoints/test-cortex/entity-fresh-blob-manual-destroy/workspace.tar.gz'; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + workspaceAzureFilesStorageAccountName: '', + workspaceAzureFilesStorageAccountKey: '', + }); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + containerId: containerName, + status: 'running', + url: 'http://workspace.test:3100', + secret: 'workspace-secret', + checkpointBlobPath, + checkpointSizeBytes: 123, + checkpointSizeMB: 0.01, + checkpointedAt: '2026-05-24T02:33:22.803Z', + }, + }); + const originalFetch = global.fetch; + const originalRemove = ACIBackend.prototype.remove; + const removedContainers = []; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.lastActivity.set(entityId, lastActivityAt); + workspaceClientModule.__testables.setWorkspaceCheckpointContainerClientForTest({ + getBlockBlobClient(blobPath) { + t.is(blobPath, checkpointBlobPath); + return { + async getProperties() { + return { + contentLength: 334673497, + lastModified: new Date(blobCheckpointedAt), + metadata: workspaceClientModule.__testables.workspaceCheckpointBlobMetadata(entityId, { + checkpointedAt: blobCheckpointedAt, + }), + }; + }, + }; + }, + }); + ACIBackend.prototype.remove = async function (containerId, requestedName) { + removedContainers.push({ containerId, requestedName }); + }; + global.fetch = async (url) => { + t.fail(`unexpected checkpoint fetch: ${url}`); + }; + + try { + const result = await workspaceClientModule.destroyWorkspace(entityId, store.getEntity()); + + t.true(result.success); + t.deepEqual(removedContainers, [{ containerId: containerName, requestedName: containerName }]); + t.is(store.getEntity().workspace.checkpointBlobPath, checkpointBlobPath); + t.is(store.getEntity().workspace.checkpointSizeBytes, 334673497); + t.is(store.getEntity().workspace.checkpointSizeMB, 319.17); + t.is(store.getEntity().workspace.checkpointedAt, blobCheckpointedAt); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + ACIBackend.prototype.remove = originalRemove; + global.fetch = originalFetch; + store.restore(); + restoreConfig(); + } +}); + +test.serial('destroyWorkspace trusts valid checkpoint Blob when no activity is tracked', async (t) => { + const entityId = 'entity-no-activity-blob-manual-destroy'; + const containerName = 'workspace-entity-no-activity-blob-manual-destroy'; + const blobCheckpointedAt = '2026-05-24T02:44:03.690Z'; + const checkpointBlobPath = 'workspace-checkpoints/test-cortex/entity-no-activity-blob-manual-destroy/workspace.tar.gz'; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + workspaceAzureFilesStorageAccountName: '', + workspaceAzureFilesStorageAccountKey: '', + }); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + containerId: containerName, + status: 'running', + url: 'http://workspace.test:3100', + secret: 'workspace-secret', + checkpointBlobPath, + checkpointedAt: '2026-05-24T02:33:22.803Z', + }, + }); + const originalFetch = global.fetch; + const originalRemove = ACIBackend.prototype.remove; + const removedContainers = []; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setWorkspaceCheckpointContainerClientForTest({ + getBlockBlobClient(blobPath) { + t.is(blobPath, checkpointBlobPath); + return { + async getProperties() { + return { + contentLength: 334673497, + lastModified: new Date(blobCheckpointedAt), + metadata: workspaceClientModule.__testables.workspaceCheckpointBlobMetadata(entityId, { + checkpointedAt: blobCheckpointedAt, + }), + }; + }, + }; + }, + }); + ACIBackend.prototype.remove = async function (containerId, requestedName) { + removedContainers.push({ containerId, requestedName }); + }; + global.fetch = async (url) => { + t.fail(`unexpected checkpoint fetch: ${url}`); + }; + + try { + const result = await workspaceClientModule.destroyWorkspace(entityId, store.getEntity()); + + t.true(result.success); + t.deepEqual(removedContainers, [{ containerId: containerName, requestedName: containerName }]); + t.is(store.getEntity().workspace.checkpointBlobPath, checkpointBlobPath); + t.is(store.getEntity().workspace.checkpointedAt, blobCheckpointedAt); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + ACIBackend.prototype.remove = originalRemove; + global.fetch = originalFetch; + store.restore(); + restoreConfig(); + } +}); + +test.serial('destroyWorkspace accepts checkpoint timestamps from 1.0.9 workspace helpers', async (t) => { + const entityId = 'entity-sanitized-checkpoint-timestamp'; + const containerName = 'workspace-entity-sanitized-checkpoint-timestamp'; + const lastActivityAt = Date.parse('2026-05-24T02:42:26.000Z'); + const checkpointTimestamp = '2026-05-24T02-44-03-690Z'; + const checkpointBlobPath = 'workspace-checkpoints/test-cortex/entity-sanitized-checkpoint-timestamp/workspace.tar.gz'; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + workspaceAzureFilesStorageAccountName: '', + workspaceAzureFilesStorageAccountKey: '', + }); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + containerId: containerName, + status: 'running', + url: 'http://workspace.test:3100', + secret: 'workspace-secret', + checkpointBlobPath, + checkpointedAt: '2026-05-24T02:33:22.803Z', + }, + }); + const originalFetch = global.fetch; + const originalRemove = ACIBackend.prototype.remove; + const removedContainers = []; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.lastActivity.set(entityId, lastActivityAt); + workspaceClientModule.__testables.setWorkspaceCheckpointUploadForTest(async (_entityId, _workspace, backupBody) => ({ + blobPath: checkpointBlobPath, + sizeBytes: 334673497, + sizeMB: 319.18, + timestamp: backupBody.timestamp, + })); + ACIBackend.prototype.remove = async function (containerId, requestedName) { + removedContainers.push({ containerId, requestedName }); + }; + global.fetch = async (url) => { + const urlString = String(url); + if (urlString.endsWith('/health')) { + return { + ok: true, + status: 200, + async json() { + return { version: '1.0.9' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + status: 200, + async json() { + return { version: '1.0.9' }; + }, + }; + } + if (urlString.endsWith('/backup')) { + return { + ok: true, + status: 200, + async json() { + return { + path: '/persist/workspace.tar.gz', + sizeBytes: 334673497, + sizeMB: 319.18, + timestamp: checkpointTimestamp, + }; + }, + }; + } + t.fail(`unexpected checkpoint fetch: ${urlString}`); + }; + + try { + const result = await workspaceClientModule.destroyWorkspace(entityId, store.getEntity()); + + t.true(result.success); + t.deepEqual(removedContainers, [{ containerId: containerName, requestedName: containerName }]); + t.is(store.getEntity().workspace.checkpointedAt, '2026-05-24T02:44:03.690Z'); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + ACIBackend.prototype.remove = originalRemove; + global.fetch = originalFetch; + store.restore(); + restoreConfig(); + } +}); + +test.serial('destroyWorkspace preserves existing checkpoint metadata without a fresh checkpoint', async (t) => { + const entityId = 'entity-preserve-existing-checkpoint'; + const containerName = 'workspace-entity-preserve-existing-checkpoint'; + const checkpointBlobPath = 'workspace-checkpoints/test-cortex/entity-preserve-existing-checkpoint/workspace.tar.gz'; + const checkpointPreviousBlobPath = 'workspace-checkpoints/test-cortex/entity-preserve-existing-checkpoint/workspace.prev.tar.gz'; + const checkpointedAt = new Date().toISOString(); + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + }); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + containerId: containerName, + status: 'provisioning', + shareName: 'legacy-share', + checkpointBlobPath, + checkpointPreviousBlobPath, + checkpointSizeBytes: 1234, + checkpointSizeMB: 0.01, + checkpointedAt, + }, + }); + const originalRemove = ACIBackend.prototype.remove; + const removedContainers = []; + + ACIBackend.prototype.remove = async function (containerId, requestedName) { + removedContainers.push({ containerId, requestedName }); + }; + + try { + const result = await workspaceClientModule.destroyWorkspace(entityId, store.getEntity()); + + t.true(result.success); + t.deepEqual(removedContainers, [{ containerId: containerName, requestedName: containerName }]); + t.deepEqual(store.getEntity().workspace, { + checkpointBlobPath, + checkpointPreviousBlobPath, + checkpointSizeBytes: 1234, + checkpointSizeMB: 0.01, + checkpointedAt, + legacyShareName: 'legacy-share', + }); + } finally { + ACIBackend.prototype.remove = originalRemove; + store.restore(); + restoreConfig(); + } +}); + +test.serial('destroyWorkspace preserves checkpoint encryption key returned on checkpoint entity config', async (t) => { + const entityId = 'entity-destroy-encrypted-checkpoint'; + const containerName = 'workspace-entity-destroy-encrypted-checkpoint'; + const checkpointBlobPath = 'workspace-checkpoints/test-cortex/entity-destroy-encrypted-checkpoint/workspace.tar.gz'; + const checkpointEncryptionKey = { + algorithm: 'aes-256-gcm', + keyId: 'checkpoint-key-id', + encryptedKey: 'encrypted-checkpoint-key', + createdAt: '2026-05-24T17:51:40.000Z', + }; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + }); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + containerId: containerName, + status: 'running', + url: 'http://workspace.test:3100', + secret: 'workspace-secret', + }, + }); + const originalFetch = global.fetch; + const originalRemove = ACIBackend.prototype.remove; + const removedContainers = []; + + workspaceClientModule.__testables.setWorkspaceCheckpointUploadForTest(async (_entityId, _workspace, backupBody) => ({ + blobPath: checkpointBlobPath, + sizeBytes: 1234, + timestamp: backupBody.timestamp, + encryption: { + algorithm: 'aes-256-gcm', + keyId: checkpointEncryptionKey.keyId, + ivBase64: Buffer.alloc(12, 5).toString('base64'), + tagBase64: Buffer.alloc(16, 6).toString('base64'), + compression: 'gzip', + }, + entityConfig: { + ...store.getEntity(), + workspace: { + ...store.getEntity().workspace, + checkpointEncryptionKey, + }, + }, + })); + ACIBackend.prototype.remove = async function (containerId, requestedName) { + removedContainers.push({ containerId, requestedName }); + }; + global.fetch = async (url) => { + const urlString = String(url); + if (urlString.endsWith('/health')) { + return { + ok: true, + status: 200, + async json() { + return { version: '1.0.9' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + status: 200, + async json() { + return { version: '1.0.9' }; + }, + }; + } + if (urlString.endsWith('/backup')) { + return { + ok: true, + status: 200, + async json() { + return { + path: '/persist/workspace.tar.gz', + sizeBytes: 1234, + timestamp: '2026-05-24T17:51:40.000Z', + }; + }, + }; + } + t.fail(`unexpected checkpoint fetch: ${urlString}`); + }; + + try { + const result = await workspaceClientModule.destroyWorkspace(entityId, store.getEntity()); + + t.true(result.success); + t.deepEqual(removedContainers, [{ containerId: containerName, requestedName: containerName }]); + t.deepEqual(store.getEntity().workspace.checkpointEncryptionKey, checkpointEncryptionKey); + t.is(store.getEntity().workspace.checkpointEncryption.keyId, checkpointEncryptionKey.keyId); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + ACIBackend.prototype.remove = originalRemove; + global.fetch = originalFetch; + store.restore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces skips stop when live workspace reports running background jobs', async (t) => { + const now = 10_000_000; + const idleTimeoutMs = 30 * 60 * 1000; + const entityId = 'entity-456'; + const staleActivity = now - idleTimeoutMs - 1; + const fakeRedis = createFakeRedis({ + activityTimestamp: staleActivity, + activityIndex: { [entityId]: staleActivity }, + }); + + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: 'redis://test', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: idleTimeoutMs, + }); + const restoreEntityStore = stubEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-456', + containerId: 'workspace-entity-456', + shareName: 'workspace-entity-456', + status: 'running', + }, + }); + const originalFetch = global.fetch; + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalStop = ACIBackend.prototype.stop; + const originalNow = Date.now; + const logCapture = stubLogger(); + let stopCalled = false; + let jobsChecked = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + workspaceClientModule.__testables.lastActivity.set(entityId, staleActivity); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = undefined; + global.fetch = async (url, options) => { + jobsChecked = true; + t.is(url, 'http://workspace.test:3100/shell/jobs'); + t.is(options.headers['x-workspace-secret'], 'secret-456'); + return { + ok: true, + async json() { + return { + jobs: [{ + processId: 'bg-1', + status: 'running', + command: 'curl -H "Authorization: Bearer secret-token" https://example.test', + }], + }; + }, + }; + }; + ACIBackend.prototype.stop = async () => { + stopCalled = true; + return {}; + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.true(jobsChecked); + t.false(stopCalled); + t.is(workspaceClientModule.__testables.lastActivity.get(entityId), staleActivity); + const reaperLog = logCapture.calls + .map(call => call.message) + .find(message => message.startsWith('[WorkspaceReaper]')); + t.truthy(reaperLog); + + const decision = JSON.parse(reaperLog.slice('[WorkspaceReaper] '.length)); + t.is(decision.entityId, entityId); + t.is(decision.action, 'skip'); + t.is(decision.reason, 'running-background-jobs'); + t.is(decision.localLastActivity, staleActivity); + t.is(decision.redisLastActivity, staleActivity); + t.is(decision.effectiveLastActivity, staleActivity); + t.is(decision.workspace.containerId, 'workspace-entity-456'); + t.true(decision.workspace.hasSecret); + t.true(decision.jobsCheck.attempted); + t.true(decision.jobsCheck.hasRunningJobs); + t.is(decision.jobsCheck.responseType, 'object'); + t.is(decision.jobsCheck.jobsContainer, 'jobs-property'); + t.is(decision.jobsCheck.jobCount, 1); + t.is(decision.jobsCheck.runningJobCount, 1); + t.deepEqual(decision.jobsCheck.jobStatusCounts, { running: 1 }); + t.false(Object.hasOwn(decision.jobsCheck, 'jobs')); + t.false(reaperLog.includes('secret-token')); + t.false(reaperLog.includes('Authorization')); + } finally { + logCapture.restore(); + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.stop = originalStop; + global.fetch = originalFetch; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces rechecks Redis activity after jobs check before stopping', async (t) => { + const now = 10_000_000; + const idleTimeoutMs = 30 * 60 * 1000; + const entityId = 'entity-654'; + const staleActivity = now - idleTimeoutMs - 1; + const freshActivity = now - 1_000; + let redisActivity = staleActivity; + const fakeRedis = createFakeRedis({ + activityTimestamp: () => redisActivity, + activityIndex: { [entityId]: staleActivity }, + }); + + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: 'redis://test', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: idleTimeoutMs, + }); + const restoreEntityStore = stubEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-654', + containerId: 'workspace-entity-654', + shareName: 'workspace-entity-654', + status: 'running', + }, + }); + const originalFetch = global.fetch; + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalStop = ACIBackend.prototype.stop; + const originalNow = Date.now; + let stopCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + workspaceClientModule.__testables.lastActivity.set(entityId, staleActivity); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = undefined; + global.fetch = async () => { + redisActivity = freshActivity; + return { + ok: true, + async json() { + return { jobs: [] }; + }, + }; + }; + ACIBackend.prototype.stop = async () => { + stopCalled = true; + return {}; + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.false(stopCalled); + t.is(workspaceClientModule.__testables.lastActivity.get(entityId), freshActivity); + t.true(fakeRedis.calls.filter(call => call.op === 'get' && call.key.endsWith(':activity:entity-654')).length >= 2); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.stop = originalStop; + global.fetch = originalFetch; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces skips stop when another host holds the Redis reaper lock', async (t) => { + const now = 10_000_000; + const idleTimeoutMs = 30 * 60 * 1000; + const entityId = 'entity-789'; + const staleActivity = now - idleTimeoutMs - 1; + const fakeRedis = createFakeRedis({ + activityTimestamp: staleActivity, + activityIndex: { [entityId]: staleActivity }, + acquireLock: false, + }); + + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: 'redis://test', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: idleTimeoutMs, + }); + const restoreEntityStore = stubEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-789', + containerId: 'workspace-entity-789', + shareName: 'workspace-entity-789', + status: 'running', + }, + }); + const originalFetch = global.fetch; + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalStop = ACIBackend.prototype.stop; + const originalNow = Date.now; + let stopCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + workspaceClientModule.__testables.lastActivity.set(entityId, staleActivity); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = undefined; + global.fetch = async () => { + t.fail('reaper should not check background jobs without the Redis lock'); + }; + ACIBackend.prototype.stop = async () => { + stopCalled = true; + return {}; + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.false(stopCalled); + t.true(fakeRedis.calls.some(call => call.op === 'set' && call.key.endsWith(':reaper-lock:entity-789'))); + t.false(fakeRedis.calls.some(call => call.op === 'get' && call.key.endsWith(':activity:entity-789'))); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.stop = originalStop; + global.fetch = originalFetch; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('reapIdleWorkspaces uses Redis activity index after local restart', async (t) => { + const now = 10_000_000; + const idleTimeoutMs = 30 * 60 * 1000; + const entityId = 'entity-restarted'; + const staleActivity = now - idleTimeoutMs - 1; + const fakeRedis = createFakeRedis({ + activityTimestamp: staleActivity, + activityIndex: { [entityId]: staleActivity }, + }); + + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: 'redis://test', + workspaceBackend: 'aci', + workspaceIdleTimeoutMs: idleTimeoutMs, + }); + const restoreEntityStore = stubEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-restarted', + containerId: 'workspace-entity-restarted', + shareName: 'workspace-entity-restarted', + status: 'running', + }, + }); + const originalFetch = global.fetch; + const originalList = ACIBackend.prototype.listWorkspaceContainers; + const originalRemove = ACIBackend.prototype.remove; + const originalNow = Date.now; + let removeCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + workspaceClientModule.__testables.setActivityRedisClientForTest(fakeRedis); + + Date.now = () => now; + ACIBackend.prototype.listWorkspaceContainers = undefined; + global.fetch = async (url) => { + const urlString = String(url); + if (urlString.endsWith('/shell/jobs')) { + return { + ok: true, + async json() { + return { jobs: [] }; + }, + }; + } + if (urlString.endsWith('/health')) { + return { + ok: true, + async json() { + return { version: '1.0.2' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + async json() { + return { uptime: 123 }; + }, + }; + } + if (urlString.endsWith('/backup')) { + return { + ok: true, + async json() { + return { success: true, checkpoint: { sizeMB: 1.25 } }; + }, + }; + } + t.fail(`unexpected fetch: ${urlString}`); + }; + ACIBackend.prototype.remove = async (name) => { + removeCalled = true; + t.is(name, 'workspace-entity-restarted'); + return {}; + }; + + try { + await workspaceClientModule.__testables.reapIdleWorkspaces(); + + t.true(removeCalled); + t.false(workspaceClientModule.__testables.lastActivity.has(entityId)); + t.true(fakeRedis.calls.some(call => call.op === 'zrangebyscore' && call.key === 'test-cortex-workspace:activity-index')); + t.true(fakeRedis.calls.some(call => call.op === 'zrem' && call.member === entityId)); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + Date.now = originalNow; + ACIBackend.prototype.listWorkspaceContainers = originalList; + ACIBackend.prototype.remove = originalRemove; + global.fetch = originalFetch; + restoreEntityStore(); + restoreConfig(); + } +}); + +test.serial('checkpointWorkspace uses /health version before calling /backup', async (t) => { + const originalFetch = global.fetch; + const calls = []; + + global.fetch = async (url) => { + calls.push(String(url)); + if (String(url).endsWith('/health')) { + return { + ok: true, + status: 200, + async json() { + return { status: 'ok', version: '1.0.6' }; + }, + }; + } + if (String(url).endsWith('/status')) { + return { + ok: true, + status: 200, + async json() { + return { uptime: 123 }; + }, + }; + } + if (String(url).endsWith('/backup')) { + return { + ok: true, + status: 200, + async json() { + return { path: '/persist/workspace.tar.gz', sizeMB: 12.3 }; + }, + }; + } + t.fail(`unexpected fetch url ${url}`); + }; + + try { + const result = await workspaceClientModule.__testables.checkpointWorkspace('entity-checkpoint', { + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-checkpoint', + }, + }, { + uploadCheckpoint: async (_entityId, _workspace, body) => ({ + blobPath: 'workspace-checkpoints/test/entity-checkpoint/workspace.tar.gz', + sizeBytes: body.sizeBytes || null, + sizeMB: body.sizeMB || null, + timestamp: body.timestamp || '2026-05-22T00:00:00.000Z', + }), + }); + + t.true(result.success); + t.falsy(result.skipped); + t.is(result.checkpoint.blobPath, 'workspace-checkpoints/test/entity-checkpoint/workspace.tar.gz'); + t.deepEqual(calls, [ + 'http://workspace.test:3100/health', + 'http://workspace.test:3100/status', + 'http://workspace.test:3100/backup', + ]); + } finally { + global.fetch = originalFetch; + } +}); + +test.serial('checkpointWorkspace uploads checkpoint directly to Blob URL', async (t) => { + const originalFetch = global.fetch; + const calls = []; + const lifecycleEvents = []; + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push({ + url: urlString, + method: options.method, + secret: options.headers?.['x-workspace-secret'], + body: options.body, + }); + if (urlString.endsWith('/health')) { + return { + ok: true, + status: 200, + async json() { + return { status: 'ok', version: '1.0.7' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + status: 200, + async json() { + return { uptime: 123 }; + }, + }; + } + if (urlString.endsWith('/backup')) { + return { + ok: true, + status: 200, + async json() { + return { + path: '/persist/workspace.tar.gz', + sizeBytes: 16, + sizeMB: 0.01, + timestamp: '2026-05-22T00:00:00.000Z', + }; + }, + }; + } + if (urlString.endsWith('/upload-url')) { + const body = JSON.parse(options.body); + t.is(options.method, 'POST'); + t.is(options.headers['x-workspace-secret'], 'secret-checkpoint'); + t.is(body.archiveUrl, 'https://storage.test/workspace.tar.gz?sas=write'); + t.is(body.archivePath, '/persist/workspace.tar.gz'); + t.is(body.metadata.entityId, 'entity-checkpoint'); + return { + ok: true, + status: 200, + async json() { + return { message: 'uploaded', sizeBytes: 16 }; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const result = await workspaceClientModule.__testables.checkpointWorkspace('entity-checkpoint', { + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-checkpoint', + }, + }, { + checkpointWriteSasUrl: 'https://storage.test/workspace.tar.gz?sas=write', + onWorkspaceLifecycle: async (event) => lifecycleEvents.push(event), + }); + + t.true(result.success); + t.is( + result.checkpoint.blobPath, + workspaceClientModule.__testables.workspaceCheckpointBlobPath('entity-checkpoint'), + ); + t.deepEqual(calls.map(call => call.url), [ + 'http://workspace.test:3100/health', + 'http://workspace.test:3100/status', + 'http://workspace.test:3100/backup', + 'http://workspace.test:3100/upload-url', + ]); + t.deepEqual(lifecycleEvents.map(event => [event.type, event.phase, event.success]), [ + ['start', 'checkpointBackup', undefined], + ['finish', 'checkpointBackup', true], + ['start', 'checkpointUpload', undefined], + ['finish', 'checkpointUpload', true], + ]); + } finally { + global.fetch = originalFetch; + } +}); + +test.serial('checkpointWorkspace streams encrypted checkpoints on helper 1.0.10+', async (t) => { + const originalFetch = global.fetch; + const calls = []; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + redisEncryptionKey: 'b'.repeat(64), + }); + const store = stubMutableEntityStore({ + id: 'entity-encrypted-streaming-checkpoint', + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-checkpoint', + }, + }); + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push(urlString); + if (urlString.endsWith('/health')) { + return { + ok: true, + status: 200, + async json() { + return { status: 'ok', version: '1.0.10' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + status: 200, + async json() { + return { uptime: 123 }; + }, + }; + } + if (urlString.endsWith('/backup-upload-url')) { + const body = JSON.parse(options.body); + t.is(body.archiveUrl, 'https://storage.test/workspace.tar.gz?sas=write'); + t.is(body.metadata.entityId, 'entity-encrypted-streaming-checkpoint'); + t.is(body.metadata.checkpointEncrypted, 'true'); + t.is(body.encryption.algorithm, 'aes-256-gcm'); + t.is(Buffer.from(body.encryption.keyBase64, 'base64').length, 32); + t.truthy(body.encryption.keyId); + return { + ok: true, + status: 200, + async json() { + return { + encrypted: true, + sizeBytes: 1234, + durationMs: 56, + encryption: { + algorithm: 'aes-256-gcm', + keyId: body.encryption.keyId, + ivBase64: Buffer.alloc(12, 5).toString('base64'), + tagBase64: Buffer.alloc(16, 6).toString('base64'), + compression: 'zstd', + }, + }; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const result = await workspaceClientModule.__testables.checkpointWorkspace( + 'entity-encrypted-streaming-checkpoint', + store.getEntity(), + { checkpointWriteSasUrl: 'https://storage.test/workspace.tar.gz?sas=write' }, + ); + + t.true(result.success); + t.is(result.checkpoint.sizeBytes, 1234); + t.is(result.checkpoint.encryption.algorithm, 'aes-256-gcm'); + t.is(result.checkpoint.encryption.compression, 'zstd'); + t.is(result.checkpoint.compression, 'zstd'); + t.is(store.getEntity().workspace.checkpointEncryptionKey.keyId, result.checkpoint.encryption.keyId); + t.deepEqual(calls, [ + 'http://workspace.test:3100/health', + 'http://workspace.test:3100/status', + 'http://workspace.test:3100/backup-upload-url', + ]); + } finally { + global.fetch = originalFetch; + store.restore(); + restoreConfig(); + } +}); + +test.serial('checkpointWorkspace does not reconfigure after encrypted upload timeout', async (t) => { + const originalFetch = global.fetch; + const calls = []; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + redisEncryptionKey: 'b'.repeat(64), + }); + const store = stubMutableEntityStore({ + id: 'entity-encrypted-streaming-timeout', + workspace: { + url: 'http://workspace.test:3100', + secret: 'rotated-secret', + bootstrapSecret: 'bootstrap-secret', + containerId: 'workspace-entity-encrypted-streaming-timeout', + status: 'running', + }, + }); + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push({ + url: urlString, + secret: options.headers?.['x-workspace-secret'], + hasDispatcher: Boolean(options.dispatcher), + }); + if (urlString.endsWith('/health')) { + return { + ok: true, + status: 200, + async json() { + return { status: 'ok', version: '1.0.10' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + status: 200, + async json() { + return { uptime: 123 }; + }, + }; + } + if (urlString.endsWith('/backup-upload-url')) { + const error = new Error('The operation was aborted due to timeout'); + error.name = 'TimeoutError'; + throw error; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const result = await workspaceClientModule.__testables.checkpointWorkspace( + 'entity-encrypted-streaming-timeout', + store.getEntity(), + { checkpointWriteSasUrl: 'https://storage.test/workspace.tar.gz?sas=write' }, + ); + + t.false(result.success); + t.regex(result.error, /\/backup-upload-url timed out after 900s/); + t.deepEqual(calls.map(call => call.url), [ + 'http://workspace.test:3100/health', + 'http://workspace.test:3100/status', + 'http://workspace.test:3100/backup-upload-url', + ]); + t.is(calls[2].secret, 'rotated-secret'); + t.true(calls[2].hasDispatcher); + } finally { + global.fetch = originalFetch; + store.restore(); + restoreConfig(); + } +}); + +test.serial('checkpointWorkspace retries upload-url fetch failure for Blob-only workspaces', async (t) => { + const originalFetch = global.fetch; + const calls = []; + let uploadAttempts = 0; + const entity = { + id: 'entity-checkpoint-upload-retry', + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-checkpoint', + }, + }; + const restoreEntityStore = stubEntityStore(entity); + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push(urlString); + if (urlString.endsWith('/health')) { + return { + ok: true, + status: 200, + async json() { + return { status: 'ok', version: '1.0.8' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + status: 200, + async json() { + return { uptime: 123 }; + }, + }; + } + if (urlString.endsWith('/backup')) { + return { + ok: true, + status: 200, + async json() { + return { + path: '/persist/workspace.tar.gz', + sizeBytes: 16, + sizeMB: 0.01, + timestamp: '2026-05-22T00:00:00.000Z', + }; + }, + }; + } + if (urlString.endsWith('/upload-url')) { + uploadAttempts += 1; + t.is(options.headers['x-workspace-secret'], 'secret-checkpoint'); + if (uploadAttempts === 1) { + throw new TypeError('fetch failed'); + } + return { + ok: true, + status: 200, + async json() { + return { message: 'uploaded', sizeBytes: 16 }; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const result = await workspaceClientModule.__testables.checkpointWorkspace(entity.id, entity, { + checkpointWriteSasUrl: 'https://storage.test/workspace.tar.gz?sas=write', + }); + + t.true(result.success); + t.is(result.checkpoint.sizeBytes, 16); + t.is(uploadAttempts, 2); + t.deepEqual(calls, [ + 'http://workspace.test:3100/health', + 'http://workspace.test:3100/status', + 'http://workspace.test:3100/backup', + 'http://workspace.test:3100/upload-url', + 'http://workspace.test:3100/upload-url', + ]); + } finally { + global.fetch = originalFetch; + restoreEntityStore(); + } +}); + +test.serial('checkpointWorkspace recovers bootstrap auth after upload-url 401', async (t) => { + const originalFetch = global.fetch; + const entityId = 'entity-checkpoint-upload-auth-recovery'; + const calls = []; + let uploadAttempts = 0; + let freshSecret = null; + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'stale-secret', + bootstrapSecret: 'bootstrap-secret', + containerId: 'workspace-entity-checkpoint-upload-auth-recovery', + status: 'running', + }, + }); + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push({ + url: urlString, + secret: options.headers?.['x-workspace-secret'], + }); + if (urlString.endsWith('/health')) { + return { + ok: true, + status: 200, + async json() { + return { status: 'ok', version: '1.0.8' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + status: 200, + async json() { + return { uptime: 123 }; + }, + }; + } + if (urlString.endsWith('/backup')) { + return { + ok: true, + status: 200, + async json() { + return { + path: '/persist/workspace.tar.gz', + sizeBytes: 16, + sizeMB: 0.01, + timestamp: '2026-05-22T00:00:00.000Z', + }; + }, + }; + } + if (urlString.endsWith('/reconfigure')) { + t.is(options.headers?.['x-workspace-secret'], 'bootstrap-secret'); + const body = JSON.parse(options.body); + t.truthy(body.secret); + freshSecret = body.secret; + return { + ok: true, + status: 200, + async json() { + return { success: true }; + }, + }; + } + if (urlString.endsWith('/upload-url')) { + uploadAttempts += 1; + if (uploadAttempts === 1) { + t.is(options.headers?.['x-workspace-secret'], 'stale-secret'); + return { + ok: false, + status: 401, + statusText: 'Unauthorized', + async json() { + return { error: 'Invalid secret' }; + }, + }; + } + t.is(options.headers?.['x-workspace-secret'], freshSecret); + return { + ok: true, + status: 200, + async json() { + return { message: 'uploaded', sizeBytes: 16 }; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const result = await workspaceClientModule.__testables.checkpointWorkspace(entityId, store.getEntity(), { + checkpointWriteSasUrl: 'https://storage.test/workspace.tar.gz?sas=write', + }); + + t.true(result.success); + t.is(result.checkpoint.sizeBytes, 16); + t.is(uploadAttempts, 2); + t.truthy(freshSecret); + t.deepEqual(calls.map(call => call.url), [ + 'http://workspace.test:3100/health', + 'http://workspace.test:3100/status', + 'http://workspace.test:3100/backup', + 'http://workspace.test:3100/upload-url', + 'http://workspace.test:3100/reconfigure', + 'http://workspace.test:3100/upload-url', + ]); + } finally { + global.fetch = originalFetch; + workspaceClientModule.__testables.resetActivityStateForTest(); + store.restore(); + } +}); + +test.serial('checkpointWorkspace recovers auth with bootstrap secret before backup', async (t) => { + const entityId = 'entity-checkpoint-auth-recovery'; + const originalFetch = global.fetch; + const calls = []; + let freshSecret = null; + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'stale-secret', + bootstrapSecret: 'bootstrap-secret', + containerId: 'workspace-entity-checkpoint-auth-recovery', + status: 'running', + legacyShareName: 'workspace-legacy-share', + }, + }); + + workspaceClientModule.__testables.setWorkspaceCheckpointUploadForTest(async (_entityId, workspace, backupBody) => { + t.is(_entityId, entityId); + t.is(workspace.secret, freshSecret); + t.is(backupBody.path, '/persist/workspace.tar.gz'); + return { + blobPath: workspaceClientModule.__testables.workspaceCheckpointBlobPath(entityId), + sizeBytes: 128, + }; + }); + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push({ + url: urlString, + secret: options.headers?.['x-workspace-secret'], + }); + if (urlString.endsWith('/health')) { + return { + ok: true, + status: 200, + async json() { + return { status: 'ok', version: '1.0.8' }; + }, + }; + } + if (urlString.endsWith('/status') && options.headers?.['x-workspace-secret'] === 'stale-secret') { + return { + ok: false, + status: 401, + statusText: 'Unauthorized', + async json() { + return { error: 'Invalid secret' }; + }, + }; + } + if (urlString.endsWith('/reconfigure')) { + t.is(options.headers?.['x-workspace-secret'], 'bootstrap-secret'); + const body = JSON.parse(options.body); + t.truthy(body.secret); + freshSecret = body.secret; + return { + ok: true, + status: 200, + async json() { + return { success: true }; + }, + }; + } + if (urlString.endsWith('/status')) { + t.is(options.headers?.['x-workspace-secret'], freshSecret); + return { + ok: true, + status: 200, + async json() { + return { uptime: 123, version: '1.0.8' }; + }, + }; + } + if (urlString.endsWith('/backup')) { + t.is(options.headers?.['x-workspace-secret'], freshSecret); + return { + ok: true, + status: 200, + async json() { + return { + path: '/persist/workspace.tar.gz', + sizeBytes: 128, + sizeMB: 0.01, + timestamp: '2026-05-22T00:00:00.000Z', + }; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const result = await workspaceClientModule.__testables.checkpointWorkspace(entityId, store.getEntity()); + + t.true(result.success); + t.is(result.checkpoint.sizeBytes, 128); + t.truthy(freshSecret); + t.deepEqual(calls.map(call => call.url), [ + 'http://workspace.test:3100/health', + 'http://workspace.test:3100/status', + 'http://workspace.test:3100/reconfigure', + 'http://workspace.test:3100/status', + 'http://workspace.test:3100/backup', + ]); + t.is(calls[0].secret, 'stale-secret'); + t.is(calls[1].secret, 'stale-secret'); + t.is(calls[2].secret, 'bootstrap-secret'); + t.is(calls[3].secret, freshSecret); + t.is(calls[4].secret, freshSecret); + } finally { + global.fetch = originalFetch; + workspaceClientModule.__testables.resetActivityStateForTest(); + store.restore(); + } +}); + +test.serial('uploadWorkspaceCheckpoint marks streaming checkpoints at upload start', async (t) => { + const entityId = 'entity-streaming-checkpoint-start'; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + redisEncryptionKey: 'd'.repeat(64), + }); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + status: 'running', + }, + }); + await workspaceClientModule.__testables.getOrCreateWorkspaceCheckpointEncryptionKey(entityId, store.getEntity()); + + const originalFetch = global.fetch; + let uploadBody = null; + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + if (urlString.endsWith('/backup-upload-url')) { + uploadBody = JSON.parse(options.body); + return { + ok: true, + status: 200, + async json() { + return { + encrypted: true, + sizeBytes: 128, + durationMs: 5000, + encryption: { + algorithm: 'aes-256-gcm', + keyId: uploadBody.encryption.keyId, + ivBase64: Buffer.alloc(12, 1).toString('base64'), + tagBase64: Buffer.alloc(16, 2).toString('base64'), + compression: 'gzip', + }, + }; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const result = await workspaceClientModule.__testables.uploadWorkspaceCheckpoint( + entityId, + { + url: 'http://workspace.test:3100', + secret: 'workspace-secret', + }, + null, + 900000, + { + checkpointWriteSasUrl: 'https://storage.test/workspace.tar.gz?sas=write', + entityConfig: store.getEntity(), + }, + ); + + t.truthy(uploadBody.metadata.checkpointedAt); + t.is(result.timestamp, uploadBody.metadata.checkpointedAt); + } finally { + global.fetch = originalFetch; + workspaceClientModule.__testables.resetActivityStateForTest(); + store.restore(); + restoreConfig(); + } +}); + +test.serial('checkpointWorkspace copies legacy Azure Files checkpoint when workspace image lacks upload-url', async (t) => { + const originalFetch = global.fetch; + const calls = []; + let legacyCopyCall = null; + + workspaceClientModule.__testables.setWorkspaceLegacyShareUploadForTest(async (call) => { + legacyCopyCall = call; + return { + blobPath: call.blobPath, + sizeBytes: 32, + }; + }); + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push(urlString); + if (urlString.endsWith('/health')) { + return { + ok: true, + status: 200, + async json() { + return { status: 'ok', version: '1.0.7' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + status: 200, + async json() { + return { uptime: 123 }; + }, + }; + } + if (urlString.endsWith('/backup')) { + return { + ok: true, + status: 200, + async json() { + return { + path: '/persist/workspace.tar.gz', + sizeBytes: 32, + sizeMB: 0.01, + timestamp: '2026-05-22T00:00:00.000Z', + }; + }, + }; + } + if (urlString.endsWith('/upload-url')) { + return { + ok: false, + status: 404, + statusText: 'Not Found', + async json() { + return { error: 'not found' }; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const result = await workspaceClientModule.__testables.checkpointWorkspace('entity-legacy', { + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-checkpoint', + legacyShareName: 'workspace-legacy-share', + }, + }, { + checkpointWriteSasUrl: 'https://storage.test/workspace.tar.gz?sas=write', + }); + + t.true(result.success); + t.is(result.checkpoint.sizeBytes, 32); + t.is(legacyCopyCall.entityId, 'entity-legacy'); + t.is(legacyCopyCall.archivePath, '/persist/workspace.tar.gz'); + t.is(legacyCopyCall.shareName, 'workspace-legacy-share'); + t.is( + legacyCopyCall.blobPath, + workspaceClientModule.__testables.workspaceCheckpointBlobPath('entity-legacy'), + ); + t.deepEqual(calls, [ + 'http://workspace.test:3100/health', + 'http://workspace.test:3100/status', + 'http://workspace.test:3100/backup', + 'http://workspace.test:3100/upload-url', + ]); + } finally { + global.fetch = originalFetch; + workspaceClientModule.__testables.resetActivityStateForTest(); + } +}); + +test.serial('checkpointWorkspace copies legacy Azure Files checkpoint when upload-url fetch fails', async (t) => { + const originalFetch = global.fetch; + const calls = []; + let legacyCopyCall = null; + + workspaceClientModule.__testables.setWorkspaceLegacyShareUploadForTest(async (call) => { + legacyCopyCall = call; + return { + blobPath: call.blobPath, + sizeBytes: 64, + }; + }); + + global.fetch = async (url) => { + const urlString = String(url); + calls.push(urlString); + if (urlString.endsWith('/health')) { + return { + ok: true, + status: 200, + async json() { + return { status: 'ok', version: '1.0.8' }; + }, + }; + } + if (urlString.endsWith('/status')) { + return { + ok: true, + status: 200, + async json() { + return { uptime: 123 }; + }, + }; + } + if (urlString.endsWith('/backup')) { + return { + ok: true, + status: 200, + async json() { + return { + path: '/persist/workspace.tar.gz', + sizeBytes: 64, + sizeMB: 0.01, + timestamp: '2026-05-22T00:00:00.000Z', + }; + }, + }; + } + if (urlString.endsWith('/upload-url')) { + throw new TypeError('fetch failed'); + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const result = await workspaceClientModule.__testables.checkpointWorkspace('entity-legacy-fetch-fail', { + workspace: { + url: 'http://workspace.test:3100', + secret: 'secret-checkpoint', + legacyShareName: 'workspace-legacy-share', + }, + }, { + checkpointWriteSasUrl: 'https://storage.test/workspace.tar.gz?sas=write', + }); + + t.true(result.success); + t.is(result.checkpoint.sizeBytes, 64); + t.is(legacyCopyCall.entityId, 'entity-legacy-fetch-fail'); + t.is(legacyCopyCall.archivePath, '/persist/workspace.tar.gz'); + t.is(legacyCopyCall.shareName, 'workspace-legacy-share'); + t.deepEqual(calls, [ + 'http://workspace.test:3100/health', + 'http://workspace.test:3100/status', + 'http://workspace.test:3100/backup', + 'http://workspace.test:3100/upload-url', + ]); + } finally { + global.fetch = originalFetch; + workspaceClientModule.__testables.resetActivityStateForTest(); + } +}); + +test.serial('checkpointLegacyShareAfterProvision copies legacy archive directly to Blob', async (t) => { + const entityId = 'entity-legacy-after-provision'; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + }); + const store = stubMutableEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'workspace-secret', + containerId: 'workspace-entity-legacy-after-provision', + status: 'running', + shareName: 'legacy-share-after-provision', + legacyShareName: 'legacy-share-after-provision', + }, + }); + const originalFetch = global.fetch; + const uploads = []; + const checkpointBlobPath = workspaceClientModule.__testables.workspaceCheckpointBlobPath(entityId); + + workspaceClientModule.__testables.setWorkspaceLegacyShareUploadForTest(async (call) => { + uploads.push(call); + return { + blobPath: call.blobPath, + sizeBytes: 4096, + }; + }); + global.fetch = async (url) => { + t.fail(`unexpected fetch: ${String(url)}`); + }; + + try { + const result = await workspaceClientModule.__testables.checkpointLegacyShareAfterProvision( + entityId, + store.getEntity(), + ); + + t.true(result.success); + t.is(uploads.length, 1); + t.deepEqual(uploads[0], { + entityId, + workspace: { + legacyShareName: 'legacy-share-after-provision', + }, + shareName: 'legacy-share-after-provision', + archivePath: '/persist/workspace.tar.gz', + blobPath: checkpointBlobPath, + }); + t.is( + store.getEntity().workspace.checkpointBlobPath, + checkpointBlobPath, + ); + t.is(store.getEntity().workspace.legacyShareName, 'legacy-share-after-provision'); + t.false(Object.hasOwn(store.getEntity().workspace, 'shareName')); + } finally { + global.fetch = originalFetch; + workspaceClientModule.__testables.resetActivityStateForTest(); + store.restore(); + restoreConfig(); + } +}); + +test.serial('restoreWorkspaceCheckpointToContainer restores checkpoint directly from Blob URL once', async (t) => { + const originalFetch = global.fetch; + const calls = []; + const lifecycleEvents = []; + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push({ + url: urlString, + method: options.method, + secret: options.headers?.['x-workspace-secret'], + }); + if (urlString.endsWith('/restore-url')) { + t.is(options.method, 'POST'); + t.is(options.headers['x-workspace-secret'], 'bootstrap-secret'); + t.deepEqual(JSON.parse(options.body), { + archiveUrl: 'https://storage.test/workspace.tar.gz?sas=1', + archivePath: '/persist/workspace.tar.gz', + }); + return { + ok: true, + status: 200, + async json() { + return { message: 'downloaded', sizeBytes: 16 }; + }, + }; + } + if (urlString.endsWith('/restore')) { + t.fail('/restore should not be called after /restore-url succeeds'); + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const result = await workspaceClientModule.__testables.restoreWorkspaceCheckpointToContainer( + 'entity-restore', + { + workspace: { + checkpointBlobPath: 'workspace-checkpoints/test/entity-restore/workspace.tar.gz', + checkpointSizeBytes: 16, + }, + }, + { + url: 'http://workspace.test:3100', + bootstrapSecret: 'bootstrap-secret', + }, + { + checkpointSasUrl: 'https://storage.test/workspace.tar.gz?sas=1', + onWorkspaceLifecycle: async (event) => lifecycleEvents.push(event), + }, + ); + + t.true(result.success); + t.deepEqual(calls.map(call => call.url), [ + 'http://workspace.test:3100/restore-url', + ]); + t.deepEqual(lifecycleEvents.map(event => [event.type, event.phase, event.success]), [ + ['start', 'restore', undefined], + ['finish', 'restore', true], + ]); + } finally { + global.fetch = originalFetch; + } +}); + +test.serial('restoreWorkspaceCheckpointToContainer passes encrypted checkpoint material to helper', async (t) => { + const originalFetch = global.fetch; + const restoreConfig = stubConfig({ + redisEncryptionKey: 'c'.repeat(64), + }); + const store = stubMutableEntityStore({ + id: 'entity-restore-encrypted', + workspace: { + checkpointBlobPath: 'workspace-checkpoints/test/entity-restore-encrypted/workspace.tar.gz', + }, + }); + const key = await workspaceClientModule.__testables.getOrCreateWorkspaceCheckpointEncryptionKey( + 'entity-restore-encrypted', + store.getEntity(), + ); + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + if (urlString.endsWith('/restore-url')) { + const body = JSON.parse(options.body); + t.is(body.archiveUrl, 'https://storage.test/workspace.tar.gz?sas=encrypted'); + t.is(body.archivePath, '/persist/workspace.tar.gz'); + t.is(body.encryption.algorithm, 'aes-256-gcm'); + t.is(body.encryption.keyBase64, key.keyBase64); + t.is(body.encryption.keyId, key.keyId); + t.is(body.encryption.ivBase64, Buffer.alloc(12, 7).toString('base64')); + t.is(body.encryption.tagBase64, Buffer.alloc(16, 8).toString('base64')); + t.is(body.encryption.compression, 'zstd'); + return { + ok: true, + status: 200, + async json() { + return { message: 'restored encrypted', sizeBytes: 16 }; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const result = await workspaceClientModule.__testables.restoreWorkspaceCheckpointToContainer( + 'entity-restore-encrypted', + { + ...store.getEntity(), + workspace: { + ...store.getEntity().workspace, + checkpointEncryption: { + algorithm: 'aes-256-gcm', + keyId: key.keyId, + ivBase64: Buffer.alloc(12, 7).toString('base64'), + tagBase64: Buffer.alloc(16, 8).toString('base64'), + compression: 'zstd', + }, + }, + }, + { + url: 'http://workspace.test:3100', + bootstrapSecret: 'bootstrap-secret', + }, + { checkpointSasUrl: 'https://storage.test/workspace.tar.gz?sas=encrypted' }, + ); + + t.true(result.success); + } finally { + global.fetch = originalFetch; + store.restore(); + restoreConfig(); + } +}); + +test.serial('restoreWorkspaceCheckpointToContainer uses /restore only for old workspace images', async (t) => { + const originalFetch = global.fetch; + const calls = []; + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push({ + url: urlString, + method: options.method, + secret: options.headers?.['x-workspace-secret'], + }); + if (urlString.endsWith('/restore-url')) { + return { + ok: false, + status: 404, + statusText: 'Not Found', + async json() { + return {}; + }, + }; + } + if (urlString.endsWith('/shell')) { + t.is(options.method, 'POST'); + t.is(options.headers['x-workspace-secret'], 'bootstrap-secret'); + t.regex(JSON.parse(options.body).command, /WORKSPACE_RESTORE_URL=/); + return { + ok: true, + status: 200, + async json() { + return { exitCode: 0 }; + }, + }; + } + if (urlString.endsWith('/restore')) { + t.is(options.method, 'POST'); + t.is(options.headers['x-workspace-secret'], 'bootstrap-secret'); + t.deepEqual(JSON.parse(options.body), { archivePath: '/persist/workspace.tar.gz' }); + return { + ok: true, + status: 200, + async json() { + return { message: 'restored' }; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + try { + const result = await workspaceClientModule.__testables.restoreWorkspaceCheckpointToContainer( + 'entity-restore', + { + workspace: { + checkpointBlobPath: 'workspace-checkpoints/test/entity-restore/workspace.tar.gz', + checkpointSizeBytes: 16, + }, + }, + { + url: 'http://workspace.test:3100', + bootstrapSecret: 'bootstrap-secret', + }, + { checkpointSasUrl: 'https://storage.test/workspace.tar.gz?sas=1' }, + ); + + t.true(result.success); + t.deepEqual(calls.map(call => call.url), [ + 'http://workspace.test:3100/restore-url', + 'http://workspace.test:3100/shell', + 'http://workspace.test:3100/restore', + ]); + } finally { + global.fetch = originalFetch; + } +}); + +test.serial('setupWorkspaceContainerForEntity rewrites env after restored checkpoint', async (t) => { + const originalFetch = global.fetch; + const calls = []; + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push({ url: urlString, body: options.body ? JSON.parse(options.body) : null }); + if (urlString.endsWith('/restore-url')) { + return { + ok: true, + status: 200, + async json() { + return { message: 'restored' }; + }, + }; + } + if (urlString.endsWith('/reconfigure')) { + t.deepEqual(JSON.parse(options.body).env, {}); + return { + ok: true, + status: 200, + async json() { + return { success: true }; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + const backend = { + backendName: 'aci', + async remove() { + t.fail('setup should not remove a successfully configured container'); + }, + }; + const store = stubMutableEntityStore({ + id: 'entity-restore-env', + secrets: {}, + assocUserIds: [], + workspace: { + checkpointBlobPath: 'workspace-checkpoints/test/entity-restore-env/workspace.tar.gz', + }, + }); + + try { + const result = await workspaceClientModule.__testables.setupWorkspaceContainerForEntity( + 'entity-restore-env', + store.getEntity(), + { + url: 'http://workspace.test:3100', + bootstrapSecret: 'bootstrap-secret', + containerId: 'workspace-entity-restore-env', + containerName: 'workspace-entity-restore-env', + }, + backend, + { checkpointSasUrl: 'https://storage.test/workspace.tar.gz?sas=1' }, + ); + + t.true(result.success); + t.deepEqual(calls.map(call => call.url), [ + 'http://workspace.test:3100/restore-url', + 'http://workspace.test:3100/reconfigure', + ]); + } finally { + global.fetch = originalFetch; + store.restore(); + } +}); + +test.serial('setupWorkspaceContainerForEntity restores mounted legacy share before reconfigure', async (t) => { + const originalFetch = global.fetch; + const calls = []; + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push({ url: urlString, body: options.body ? JSON.parse(options.body) : null }); + if (urlString.endsWith('/shell')) { + t.is(options.headers['x-workspace-secret'], 'bootstrap-secret'); + t.is(JSON.parse(options.body).command, "test -f '/persist/workspace.tar.gz'"); + return { + ok: true, + status: 200, + async json() { + return { exitCode: 0 }; + }, + }; + } + if (urlString.endsWith('/restore')) { + t.is(options.headers['x-workspace-secret'], 'bootstrap-secret'); + t.deepEqual(JSON.parse(options.body), { archivePath: '/persist/workspace.tar.gz' }); + return { + ok: true, + status: 200, + async json() { + return { message: 'restored legacy archive' }; + }, + }; + } + if (urlString.endsWith('/reconfigure')) { + t.deepEqual(JSON.parse(options.body).env, {}); + return { + ok: true, + status: 200, + async json() { + return { success: true }; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + const backend = { + backendName: 'aci', + async remove() { + t.fail('setup should not remove a successfully configured container'); + }, + }; + const store = stubMutableEntityStore({ + id: 'entity-legacy-restore', + secrets: {}, + assocUserIds: [], + workspace: { + shareName: 'legacy-share', + legacyShareName: 'legacy-share', + }, + }); + + try { + const result = await workspaceClientModule.__testables.setupWorkspaceContainerForEntity( + 'entity-legacy-restore', + store.getEntity(), + { + url: 'http://workspace.test:3100', + bootstrapSecret: 'bootstrap-secret', + containerId: 'workspace-entity-legacy-restore', + containerName: 'workspace-entity-legacy-restore', + legacyShareName: 'legacy-share', + }, + backend, + ); + + t.true(result.success); + t.is(result.legacyShareName, 'legacy-share'); + t.deepEqual(calls.map(call => call.url), [ + 'http://workspace.test:3100/shell', + 'http://workspace.test:3100/restore', + 'http://workspace.test:3100/reconfigure', + ]); + } finally { + global.fetch = originalFetch; + store.restore(); + } +}); + +test.serial('setupWorkspaceContainerForEntity preserves raw legacy shares without a tarball', async (t) => { + const originalFetch = global.fetch; + const calls = []; + + global.fetch = async (url, options = {}) => { + const urlString = String(url); + calls.push({ url: urlString, body: options.body ? JSON.parse(options.body) : null }); + if (urlString.endsWith('/shell')) { + t.is(options.headers['x-workspace-secret'], 'bootstrap-secret'); + t.is(JSON.parse(options.body).command, "test -f '/persist/workspace.tar.gz'"); + return { + ok: true, + status: 200, + async json() { + return { exitCode: 1 }; + }, + }; + } + if (urlString.endsWith('/restore')) { + t.fail('/restore should not be called when the legacy share has no tarball'); + } + if (urlString.endsWith('/reconfigure')) { + t.false(Object.hasOwn(JSON.parse(options.body), 'env')); + return { + ok: true, + status: 200, + async json() { + return { success: true }; + }, + }; + } + t.fail(`unexpected fetch url ${urlString}`); + }; + + const backend = { + backendName: 'aci', + async remove() { + t.fail('setup should not remove a raw legacy-share container'); + }, + }; + const store = stubMutableEntityStore({ + id: 'entity-legacy-raw', + secrets: {}, + assocUserIds: [], + workspace: { + shareName: 'legacy-raw-share', + legacyShareName: 'legacy-raw-share', + }, + }); + + try { + const result = await workspaceClientModule.__testables.setupWorkspaceContainerForEntity( + 'entity-legacy-raw', + store.getEntity(), + { + url: 'http://workspace.test:3100', + bootstrapSecret: 'bootstrap-secret', + containerId: 'workspace-entity-legacy-raw', + containerName: 'workspace-entity-legacy-raw', + legacyShareName: 'legacy-raw-share', + }, + backend, + ); + + t.true(result.success); + t.true(result.skipped); + t.is(result.reason, 'legacy share has no checkpoint archive'); + t.deepEqual(calls.map(call => call.url), [ + 'http://workspace.test:3100/shell', + 'http://workspace.test:3100/reconfigure', + ]); + } finally { + global.fetch = originalFetch; + store.restore(); + } +}); + +test.serial('workspaceRequest aborts stale image reprovision when checkpoint fails', async (t) => { + const entityId = 'entity-stale-checkpoint-failure'; + const restoreConfig = stubConfig({ + cortexId: 'test-cortex', + storageConnectionString: '', + workspaceBackend: 'aci', + workspaceImageVersion: '1.0.7', + workspaceIdleTimeoutMs: 30 * 60 * 1000, + warmPoolSize: 0, + }); + const restoreEntityStore = stubEntityStore({ + id: entityId, + workspace: { + url: 'http://workspace.test:3100', + secret: 'stale-secret', + containerId: 'workspace-stale-checkpoint-failure', + shareName: 'workspace-stale-checkpoint-failure', + status: 'running', + imageVersion: '1.0.6', + }, + }); + const originalFetch = global.fetch; + const originalRemove = ACIBackend.prototype.remove; + const originalCreateAndStart = ACIBackend.prototype.createAndStart; + const lifecycleEvents = []; + let createCalled = false; + let removeCalled = false; + + workspaceClientModule.__testables.resetActivityStateForTest(); + global.fetch = async (url) => { + const urlString = String(url); + if (urlString.endsWith('/health')) { + return { + ok: false, + status: 401, + statusText: 'Unauthorized', + async json() { + return { error: 'invalid workspace secret' }; + }, + }; + } + t.fail(`unexpected fetch after checkpoint failure: ${urlString}`); + }; + ACIBackend.prototype.remove = async () => { + removeCalled = true; + return {}; + }; + ACIBackend.prototype.createAndStart = async () => { + createCalled = true; + throw new Error('provision should not run after checkpoint failure'); + }; + + try { + const result = await workspaceClientModule.workspaceRequest(entityId, '/health', null, { + onWorkspaceLifecycle(event) { + lifecycleEvents.push(event); + }, + }); + + t.false(result.success); + t.true(result.error.includes('/health returned 401')); + t.false(removeCalled); + t.false(createCalled); + t.deepEqual( + lifecycleEvents.map(({ type, phase, success }) => ({ type, phase, success })), + [ + { type: 'start', phase: 'reprovision', success: undefined }, + { type: 'finish', phase: 'reprovision', success: false }, + ], + ); + } finally { + workspaceClientModule.__testables.resetActivityStateForTest(); + ACIBackend.prototype.remove = originalRemove; + ACIBackend.prototype.createAndStart = originalCreateAndStart; + global.fetch = originalFetch; + restoreEntityStore(); + restoreConfig(); + } +}); diff --git a/tests/unit/tools/workspaceReconfigure.test.js b/tests/unit/tools/workspaceReconfigure.test.js new file mode 100644 index 00000000..c6e5ff96 --- /dev/null +++ b/tests/unit/tools/workspaceReconfigure.test.js @@ -0,0 +1,119 @@ +/** + * Unit tests for workspace secret rotation (auth.js getSecret/setSecret). + * + * These tests run in isolation — no Docker or server needed. + * Uses test.serial because tests share module-level secret state. + * + * Run with: npm test -- cortex -- tests/unit/tools/workspaceReconfigure.test.js + */ + +import test from 'ava'; + +// Set the initial secret before importing the auth module +const INITIAL_SECRET = 'test-secret-initial'; +process.env.WORKSPACE_SECRET = INITIAL_SECRET; + +// Dynamic import so the module picks up process.env.WORKSPACE_SECRET +const { getSecret, setSecret, requireAuth } = await import( + '../../../helper-apps/cortex-workspace/lib/auth.js' +); + +// Helper: create mock req/res/next for Express middleware testing +function mockReqRes(secretHeader) { + const req = { + headers: secretHeader !== undefined + ? { 'x-workspace-secret': secretHeader } + : {}, + }; + let statusCode = null; + let jsonBody = null; + let nextCalled = false; + + const res = { + status(code) { + statusCode = code; + return res; + }, + json(body) { + jsonBody = body; + return res; + }, + }; + + const next = () => { nextCalled = true; }; + + return { req, res, next, getStatus: () => statusCode, getBody: () => jsonBody, wasNextCalled: () => nextCalled }; +} + +// ============================================================================ +// getSecret / setSecret +// ============================================================================ + +test.serial('getSecret › returns initial value from process.env', (t) => { + t.is(getSecret(), INITIAL_SECRET); +}); + +test.serial('setSecret › changes the secret', (t) => { + setSecret('new-secret'); + t.is(getSecret(), 'new-secret'); + + // Restore for subsequent tests + setSecret(INITIAL_SECRET); + t.is(getSecret(), INITIAL_SECRET); +}); + +// ============================================================================ +// requireAuth middleware +// ============================================================================ + +test.serial('requireAuth › accepts correct secret', (t) => { + const { req, res, next, wasNextCalled, getStatus } = mockReqRes(INITIAL_SECRET); + requireAuth(req, res, next); + + t.true(wasNextCalled()); + t.is(getStatus(), null); // no error status set +}); + +test.serial('requireAuth › rejects wrong secret', (t) => { + const { req, res, next, wasNextCalled, getStatus, getBody } = mockReqRes('wrong-secret'); + requireAuth(req, res, next); + + t.false(wasNextCalled()); + t.is(getStatus(), 401); + t.is(getBody().error, 'Invalid secret'); +}); + +test.serial('requireAuth › rejects missing header', (t) => { + const { req, res, next, wasNextCalled, getStatus, getBody } = mockReqRes(undefined); + requireAuth(req, res, next); + + t.false(wasNextCalled()); + t.is(getStatus(), 401); + t.is(getBody().error, 'Missing x-workspace-secret header'); +}); + +// ============================================================================ +// Secret rotation flow (simulates /reconfigure) +// ============================================================================ + +test.serial('rotation › old secret rejected after setSecret', (t) => { + const oldSecret = getSecret(); + const newSecret = 'rotated-secret-' + Date.now(); + + setSecret(newSecret); + + // Old secret should be rejected + const { req: req1, res: res1, next: next1, wasNextCalled: n1, getStatus: s1 } = mockReqRes(oldSecret); + requireAuth(req1, res1, next1); + t.false(n1()); + t.is(s1(), 401); + + // New secret should be accepted + const { req: req2, res: res2, next: next2, wasNextCalled: n2, getStatus: s2 } = mockReqRes(newSecret); + requireAuth(req2, res2, next2); + t.true(n2()); + t.is(s2(), null); + + // Restore + setSecret(INITIAL_SECRET); +}); diff --git a/tests/unit/tools/workspaceSSH.test.js b/tests/unit/tools/workspaceSSH.test.js new file mode 100644 index 00000000..f342d76f --- /dev/null +++ b/tests/unit/tools/workspaceSSH.test.js @@ -0,0 +1,56 @@ +import test from 'ava'; +import { resetDestroyTimeoutMs, tokenize, toAbsWorkspacePath } from '../../../pathways/system/entity/tools/sys_tool_workspace_ssh.js'; + +// ============================================================================ +// Tokenizer tests +// ============================================================================ + +test('tokenize › should split simple command', (t) => { + const tokens = tokenize('ls -la'); + t.deepEqual(tokens, ['ls', '-la']); +}); + +test('tokenize › should handle double quotes', (t) => { + const tokens = tokenize('echo "hello world"'); + t.deepEqual(tokens, ['echo', 'hello world']); +}); + +test('tokenize › should handle single quotes', (t) => { + const tokens = tokenize("echo 'hello world'"); + t.deepEqual(tokens, ['echo', 'hello world']); +}); + +test('tokenize › should handle mixed quotes', (t) => { + const tokens = tokenize('cp "my file.txt" \'another file.pdf\''); + t.deepEqual(tokens, ['cp', 'my file.txt', 'another file.pdf']); +}); + +test('tokenize › should handle empty input', (t) => { + const tokens = tokenize(''); + t.deepEqual(tokens, []); +}); + +test('tokenize › should handle multiple spaces', (t) => { + const tokens = tokenize('ls -la /workspace'); + t.deepEqual(tokens, ['ls', '-la', '/workspace']); +}); + +// ============================================================================ +// Path normalization tests +// ============================================================================ + +test('toAbsWorkspacePath › should preserve absolute paths', (t) => { + t.is(toAbsWorkspacePath('/workspace/foo.txt'), '/workspace/foo.txt'); + t.is(toAbsWorkspacePath('/tmp/file.txt'), '/tmp/file.txt'); +}); + +test('toAbsWorkspacePath › should normalize relative paths', (t) => { + t.is(toAbsWorkspacePath('foo.txt'), '/workspace/foo.txt'); + t.is(toAbsWorkspacePath('subdir/bar.txt'), '/workspace/subdir/bar.txt'); +}); + +test('resetDestroyTimeoutMs › should use a 15 minute minimum for reprovision', (t) => { + t.is(resetDestroyTimeoutMs(undefined), 900000); + t.is(resetDestroyTimeoutMs(60), 900000); + t.is(resetDestroyTimeoutMs(1200), 1200000); +}); diff --git a/tests/unit/tools/workspaceShareName.test.js b/tests/unit/tools/workspaceShareName.test.js new file mode 100644 index 00000000..1f2f2b15 --- /dev/null +++ b/tests/unit/tools/workspaceShareName.test.js @@ -0,0 +1,322 @@ +/** + * Unit tests for workspace shareName preservation logic. + * + * Verifies that: + * - ACIBackend.createAndStart() uses explicit shareName when provided + * - ACIBackend.destroyVolume() uses shareName directly (not derived from containerName) + * - DockerBackend.createAndStart() uses explicit shareName for volume naming + * - DockerBackend.destroyVolume() uses shareName directly + * + * These are "structural" tests — they verify the parameter plumbing without + * actually calling Azure or Docker APIs. We achieve this by subclassing the + * backends and capturing the arguments passed to the underlying operations. + * + * Run with: npm test -- cortex tests/unit/tools/workspaceShareName.test.js + */ + +import test from 'ava'; + +// ============================================================================ +// ACIBackend: shareName parameter plumbing +// ============================================================================ + +// We can't easily unit-test ACIBackend without Azure credentials, but we +// can verify the DockerBackend plumbing since it uses local HTTP calls. +// For ACIBackend, we test the shareName logic by importing and inspecting +// the source behavior indirectly. + +test('ACIBackend › createAndStart mounts Azure Files only for explicit legacy migration', async (t) => { + // Import ACIBackend and override the Azure-dependent methods + const { default: ACIBackend } = await import( + '../../../pathways/system/entity/tools/shared/backends/ACIBackend.js' + ); + + class TestACIBackend extends ACIBackend { + constructor() { + super(); + this.capturedShareName = null; + this.capturedContainerGroupDef = null; + } + + async _getClient() { + // Return a mock client that captures the container group definition + return { + containerGroups: { + beginCreateOrUpdate: async (_rg, _name, def) => { + this.capturedContainerGroupDef = def; + return { + pollUntilDone: async () => ({ + ipAddress: { fqdn: 'test.eastus.azurecontainer.io', ip: '1.2.3.4' }, + }), + }; + }, + }, + }; + } + + async _ensureFileShare(shareName) { + this.capturedShareName = shareName; + } + } + + const backend = new TestACIBackend(); + + // Stub config values needed by createAndStart + const originalGet = (await import('../../../config.js')).config.get; + const configStubs = { + azureResourceGroup: 'test-rg', + azureLocation: 'eastus', + azureAcrServer: null, + azureAcrUsername: null, + azureAcrPassword: null, + azureStorageAccountName: 'testaccount', + azureStorageAccountKey: 'dGVzdGtleQ==', // base64 "testkey" + }; + + const { config } = await import('../../../config.js'); + const origGet = config.get.bind(config); + config.get = (key) => { + if (key in configStubs) return configStubs[key]; + return origGet(key); + }; + + try { + const created = await backend.createAndStart({ + containerName: 'workspace-entity-123', + image: 'cortex-workspace:latest', + env: ['WORKSPACE_SECRET=test', 'PORT=3100'], + cpus: 1, + memoryMB: 512, + diskSize: '10g', + shareName: 'workspace-pool-abc123', + mountAzureFiles: true, + }); + + // The share name should be the explicit shareName, NOT the containerName + t.is(backend.capturedShareName, 'workspace-pool-abc123'); + + // The persistent Azure Files mount should use the explicit share name. + const persistVolumeDef = backend.capturedContainerGroupDef.volumes.find(v => v.name === 'persist-vol'); + t.is(persistVolumeDef.azureFile.shareName, 'workspace-pool-abc123'); + + // /workspace itself should be local ephemeral storage so symlinks work. + const workspaceVolumeDef = backend.capturedContainerGroupDef.volumes.find(v => v.name === 'workspace-vol'); + t.deepEqual(workspaceVolumeDef.emptyDir, {}); + + // Prefer the assigned IP over the DNS label because ACI can briefly + // serve stale DNS after deleting/recreating a group with the same name. + t.is(created.url, 'http://1.2.3.4:3100'); + } finally { + config.get = origGet; + } +}); + +test('ACIBackend › createAndStart uses local persist volume by default', async (t) => { + const { default: ACIBackend } = await import( + '../../../pathways/system/entity/tools/shared/backends/ACIBackend.js' + ); + + class TestACIBackend extends ACIBackend { + constructor() { + super(); + this.capturedShareName = null; + } + + async _getClient() { + return { + containerGroups: { + beginCreateOrUpdate: async (_rg, _name, def) => { + this.capturedContainerGroupDef = def; + return { + pollUntilDone: async () => ({ + ipAddress: { fqdn: 'test.eastus.azurecontainer.io' }, + }), + }; + }, + }, + }; + } + + async _ensureFileShare(shareName) { + this.capturedShareName = shareName; + } + } + + const backend = new TestACIBackend(); + + const { config } = await import('../../../config.js'); + const origGet = config.get.bind(config); + config.get = (key) => { + const stubs = { + azureResourceGroup: 'test-rg', + azureLocation: 'eastus', + azureAcrServer: null, + azureAcrUsername: null, + azureAcrPassword: null, + azureStorageAccountName: 'testaccount', + azureStorageAccountKey: 'dGVzdGtleQ==', + }; + if (key in stubs) return stubs[key]; + return origGet(key); + }; + + try { + await backend.createAndStart({ + containerName: 'workspace-entity-456', + image: 'cortex-workspace:latest', + env: ['WORKSPACE_SECRET=test', 'PORT=3100'], + cpus: 1, + memoryMB: 512, + diskSize: '10g', + // No shareName or mountAzureFiles: Blob checkpoints are restored by Cortex. + }); + + t.is(backend.capturedShareName, null); + const persistVolumeDef = backend.capturedContainerGroupDef.volumes.find(v => v.name === 'persist-vol'); + t.deepEqual(persistVolumeDef.emptyDir, {}); + t.falsy(persistVolumeDef.azureFile); + } finally { + config.get = origGet; + } +}); + +// ============================================================================ +// DockerBackend: shareName parameter plumbing +// ============================================================================ + +test('DockerBackend › createAndStart should use shareName for volume naming', async (t) => { + const { default: DockerBackend } = await import( + '../../../pathways/system/entity/tools/shared/backends/DockerBackend.js' + ); + + let capturedCreateBody = null; + + class TestDockerBackend extends DockerBackend { + async _api(method, path, body) { + if (method === 'POST' && path.startsWith('/containers/create')) { + capturedCreateBody = body; + return { Id: 'test-container-id' }; + } + if (method === 'POST' && path.includes('/start')) { + return {}; + } + if (method === 'DELETE') { + return {}; + } + if (method === 'GET' && path.includes('/json')) { + return { NetworkSettings: { Ports: { '3100/tcp': [{ HostPort: '12345' }] } } }; + } + return {}; + } + } + + const backend = new TestDockerBackend(); + + await backend.createAndStart({ + containerName: 'workspace-entity-789', + image: 'cortex-workspace:latest', + env: ['WORKSPACE_SECRET=test', 'PORT=3100'], + cpus: 1, + memoryMB: 512, + diskSize: '10g', + shareName: 'workspace-pool-xyz789', + }); + + // Volume name should be derived from shareName, not containerName + const binds = capturedCreateBody.HostConfig.Binds; + t.true(binds[0].startsWith('workspace-pool-xyz789-data:'), `Expected volume "workspace-pool-xyz789-data" but got: ${binds[0]}`); +}); + +test('DockerBackend › createAndStart should fall back to containerName when no shareName', async (t) => { + const { default: DockerBackend } = await import( + '../../../pathways/system/entity/tools/shared/backends/DockerBackend.js' + ); + + let capturedCreateBody = null; + + class TestDockerBackend extends DockerBackend { + async _api(method, path, body) { + if (method === 'POST' && path.startsWith('/containers/create')) { + capturedCreateBody = body; + return { Id: 'test-container-id' }; + } + if (method === 'POST' && path.includes('/start')) { + return {}; + } + if (method === 'DELETE') { + return {}; + } + return {}; + } + } + + const backend = new TestDockerBackend(); + + await backend.createAndStart({ + containerName: 'workspace-entity-000', + image: 'cortex-workspace:latest', + env: ['WORKSPACE_SECRET=test', 'PORT=3100'], + cpus: 1, + memoryMB: 512, + diskSize: '10g', + // No shareName + }); + + const binds = capturedCreateBody.HostConfig.Binds; + t.true(binds[0].startsWith('workspace-entity-000-data:'), `Expected volume "workspace-entity-000-data" but got: ${binds[0]}`); +}); + +// ============================================================================ +// DockerBackend: destroyVolume uses shareName directly +// ============================================================================ + +test('DockerBackend › destroyVolume should use shareName param directly', async (t) => { + const { default: DockerBackend } = await import( + '../../../pathways/system/entity/tools/shared/backends/DockerBackend.js' + ); + + let deletedVolume = null; + + class TestDockerBackend extends DockerBackend { + async _api(method, path) { + if (method === 'DELETE' && path.startsWith('/volumes/')) { + deletedVolume = path.replace('/volumes/', ''); + return {}; + } + return {}; + } + } + + const backend = new TestDockerBackend(); + + await backend.destroyVolume('workspace-pool-custom-share'); + + t.is(deletedVolume, 'workspace-pool-custom-share-data'); +}); + +// ============================================================================ +// warmPool: claimContainer returns an Azure-Files-free claim +// ============================================================================ + +test('warmPool › claimContainer return shape does not require shareName', async (t) => { + // We can't easily test the full Redis-backed pool without Redis, + // but we can verify the module exports and return type documentation. + // The actual integration test covers this end-to-end. + + // Import to verify it doesn't throw + const { claimContainer } = await import( + '../../../pathways/system/entity/tools/shared/warmPool.js' + ); + + t.is(typeof claimContainer, 'function'); + + // Without Redis, claimContainer returns { success: false }. In an + // environment with Redis/pool state, validate that successful claims no + // longer carry an Azure Files share placeholder. + const result = await claimContainer(); + if (result.success) { + t.is(result.shareName, undefined); + } else { + t.false(result.success); + } +});