diff --git a/bin/lloom.mjs b/bin/lloom.mjs index 443e4cb..6bad78c 100755 --- a/bin/lloom.mjs +++ b/bin/lloom.mjs @@ -67,6 +67,12 @@ import { loadRecipeById, loadRecipes, planRecipe } from '../src/recipes.mjs'; import { RuntimeManager } from '../src/runtime-manager.mjs'; import { runCommand } from '../src/process-control.mjs'; import { applyRuntimePolicyPlan, createRuntimePolicyPlan } from '../src/runtime-policy.mjs'; +import { + applyRuntimePolicyConfig, + createRuntimePolicyConfigReport, + parseNumericMemoryFlags, + NUMERIC_MEMORY_FLAGS +} from '../src/runtime-policy-config.mjs'; import { createLloomServer } from '../src/server.mjs'; import { applySetup, createSetupPlan, syncClusterSetupMembers } from '../src/setup.mjs'; import { createSetupStatus } from '../src/setup-status.mjs'; @@ -110,7 +116,8 @@ const COMMAND_REGISTRY = [ { name: 'backend-install', aliases: [], tier: 'advanced', needsInstalledConfig: false }, { name: 'runtimes', aliases: ['runtime-status'], tier: 'advanced', needsInstalledConfig: true }, { name: 'cluster', aliases: ['cluster-status'], tier: 'advanced', needsInstalledConfig: true }, - { name: 'runtime-plan', aliases: ['runtime-policy'], tier: 'advanced', needsInstalledConfig: true }, + { name: 'runtime-plan', aliases: [], tier: 'advanced', needsInstalledConfig: true }, + { name: 'runtime-policy', aliases: [], tier: 'advanced', needsInstalledConfig: true }, { name: 'runtime-admit', aliases: [], tier: 'advanced', needsInstalledConfig: true }, { name: 'runtime-start', aliases: [], tier: 'advanced', needsInstalledConfig: true }, { name: 'runtime-warmup', aliases: [], tier: 'advanced', needsInstalledConfig: true }, @@ -205,6 +212,7 @@ Backends and runtimes: lloom backend-install [--apply --yes] [--step step-id] lloom runtimes [runtime-id|all] lloom runtime-plan + lloom runtime-policy [--max-memory-utilization f] [--reserve-memory-gb n] [--apply --yes] lloom suspend [--apply --yes] [--drain-timeout-ms 300000] lloom resume [--apply --yes] lloom runtime-admit [--apply --yes] @@ -2632,6 +2640,16 @@ async function main() { console.log(JSON.stringify(response, null, 2)); }, 'runtime-plan': async ({ args, config, command: _command }) => { + if (args.some((arg) => NUMERIC_MEMORY_FLAGS.some((flag) => arg === flag || arg.startsWith(`${flag}=`)))) { + throw new Error( + 'runtime-plan is read-only and does not accept numeric memory flags; use `lloom runtime-policy --max-memory-utilization --reserve-memory-gb --apply --yes`' + ); + } + if (hasFlag(args, '--apply') || hasFlag(args, '--yes')) { + throw new Error( + 'runtime-plan is read-only and does not accept --apply/--yes; use `lloom runtime-policy --apply --yes` to write memory-policy changes' + ); + } const requestedRuntimeId = positional(args)[1]; const livePlan = clusterConfigured(config) ? await gatewayRequest( @@ -2850,7 +2868,30 @@ async function main() { handlers['routing'] = handlers['route']; handlers['runtime-status'] = handlers['runtimes']; handlers['cluster-status'] = handlers['cluster']; - handlers['runtime-policy'] = handlers['runtime-plan']; + handlers['runtime-policy'] = async ({ args, config, command: _command }) => { + const flags = parseNumericMemoryFlags(args); + const apply = hasFlag(args, '--apply'); + const yes = hasFlag(args, '--yes'); + if (!flags) { + if (apply || yes) { + throw new Error('runtime-policy --apply/--yes requires --max-memory-utilization and/or --reserve-memory-gb'); + } + // Compatibility: a bare `runtime-policy` with no numeric flags keeps the + // historical read-only alias behavior and delegates to the runtime plan. + return handlers['runtime-plan']({ args, config, command: 'runtime-plan' }); + } + if (yes && !apply) { + throw new Error('runtime-policy --yes requires --apply'); + } + if (!apply) { + console.log(JSON.stringify(createRuntimePolicyConfigReport(config, flags), null, 2)); + return; + } + if (!yes) { + throw new Error('applying a memory-policy change requires both --apply and --yes'); + } + console.log(JSON.stringify(await applyRuntimePolicyConfig(config, flags, { apply, yes }), null, 2)); + }; handlers['voice-list'] = handlers['voices']; handlers['install-voice'] = handlers['voice-install']; handlers['voice-rm'] = handlers['voice-remove']; diff --git a/docs/runtime-policy.md b/docs/runtime-policy.md index 006471b..92196a5 100644 --- a/docs/runtime-policy.md +++ b/docs/runtime-policy.md @@ -21,6 +21,35 @@ an alias, or the config root. The legacy `evictable` field (top-level or `policy.evictable`) still normalizes to `keepWarm: true` when set to `false`, unchanged from earlier releases. +## Explicit memory limits + +Preview a numeric memory-policy change before applying it: + +```bash +lloom runtime-policy --max-memory-utilization 0.90 --reserve-memory-gb 8 +lloom runtime-policy --max-memory-utilization 0.90 --reserve-memory-gb 8 --apply --yes +``` + +These example values are not a model-specific capacity recommendation. The +utilization must be greater than zero and less than one; the reserve must be +positive. Either option can be supplied alone. The command reports the selected +before/after fields and any per-node overrides, without printing the config. + +Utilization updates both predictive admission and the memory guard's utilization +limit. Reserve updates both admission's reserve and the guard's minimum available +memory. The existing guard mode is preserved, including an explicitly selected +`yolo` mode; this command does not turn enforcement on or off. Percentage and +absolute-reserve limits both apply, so the stricter limit can determine admission. + +Changes affect the resolved local config file. Configure each managed member's +local policy as needed. Cluster node resource overrides are preserved and shown +in the preview; they can take precedence over global values. A global absolute +`runtimePolicy.memoryBudgetGb` must be resolved explicitly before using these +numeric options. Unrelated settings, routes and credentials are preserved. + +`runtime-plan` remains read-only. Without numeric options, `runtime-policy` +retains its historical read-only runtime-plan behavior. + ## Idle restoration The preferred-restore pass starts after all owned, enabled keep-warm pins are diff --git a/package.json b/package.json index 4b5a53e..6470aed 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "check": "npm run check:cluster && npm run check:js && npm run check:entity-stagger && npm run check:python && npm run check:shell && npm run check:spark-deploy && npm run check:workers", "check:cluster": "node --check src/cluster.mjs && node --check test/cluster.test.mjs", "check:entity-stagger": "node --check scripts/entity-stagger-benchmark.mjs && node --check test/entity-stagger-benchmark.test.mjs", - "check:js": "node --check bin/lloom.mjs && node --check bin/lloom-host.mjs && node --check src/backend-catalog.mjs && node --check src/benchmarks.mjs && node --check src/bootstrap.mjs && node --check src/client-integrations.mjs && node --check src/community-client.mjs && node --check src/community-site.mjs && node --check src/config.mjs && node --check src/dashboard.mjs && node --check src/doctor.mjs && node --check src/host-memory.mjs && node --check src/host-server.mjs && node --check src/init.mjs && node --check src/installer.mjs && node --check src/interchange.mjs && node --check src/machine-profile.mjs && node --check src/managed-environment.mjs && node --check src/model-acquisition.mjs && node --check src/model-intake.mjs && node --check src/model-removal.mjs && node --check src/onboarding.mjs && node --check src/recipe-index.mjs && node --check src/recipe-pack-export.mjs && node --check src/recipe-pack.mjs && node --check src/registry.mjs && node --check src/route-control.mjs && node --check src/recipes.mjs && node --check src/resource-fit.mjs && node --check src/runtime-manager.mjs && node --check src/runtime-policy.mjs && node --check src/server.mjs && node --check src/setup.mjs && node --check src/setup-status.mjs && node --check src/process-control.mjs && node --check src/security.mjs && node --check src/tts-catalog.mjs && node --check src/voice-profiles.mjs && node --check src/protocol/text.mjs && node --check src/protocol/reasoning-normalize.mjs && node --check src/protocol/reasoning-effort.mjs && node --check src/protocol/responses.mjs && node --check src/protocol/anthropic.mjs && node --check src/protocol/sse.mjs && node --check src/protocol/stream-anthropic.mjs && node --check src/protocol/stream-responses.mjs && node --check src/protocol/index.mjs && node --check scripts/apply-spark-route-catalog.mjs && node --check scripts/capture-live-metrics.mjs && node --check scripts/merge-live-metrics.mjs && node --check scripts/run-qualitative-chat-benchmark.mjs && node --check scripts/run-local-decision-benchmark.mjs && node --check scripts/check-community-deploy.mjs && node --check scripts/generate-client-configs.mjs && node --check scripts/check-interchange.mjs && node --check scripts/check-package.mjs && node --check scripts/release-lib.mjs && node --check scripts/build-release.mjs && node --check scripts/deploy-spark.mjs && node --check test/audio-generations.test.mjs && node --check test/community-host.test.mjs && node --check test/dashboard-status.test.mjs && node --check test/ds4fv-recipe.test.mjs && node --check test/dspark-hotfixes.test.mjs && node --check test/glm53-exl3-recipe.test.mjs && node --check test/host-memory.test.mjs && node --check test/metrics-persistence.test.mjs && node --check test/model-acquisition.test.mjs && node --check test/model-failover.test.mjs && node --check test/qwen38-vllm-recipe.test.mjs && node --check test/reasoning-normalize.test.mjs && node --check test/reasoning-effort.test.mjs && node --check test/resource-fit.test.mjs && node --check test/route-control.test.mjs && node --check test/spark-route-catalog.test.mjs && node --check test/runtime-watchdog.test.mjs && node --check test/tts-catalog.test.mjs && node --check test/voice-profiles.test.mjs && node --check test/chatterbox-recipe.test.mjs && node --check test/hear-recipe.test.mjs && node --check test/smoke.mjs && node --check src/runtime-supervisor.mjs && node --check test/runtime-supervisor.test.mjs && node --check test/comfyui-media.test.mjs && node --check src/runtime-residency.mjs && node --check src/dashboard-scene.mjs && node --check src/dashboard-presence.mjs && node --check src/dashboard-presence-client.mjs && node --check src/first-run.mjs && node --check src/first-run-page.mjs && node --check src/browser-setup.mjs && node --check src/runtime-preferences.mjs && node --check src/installation-jobs.mjs && node --check src/model-installation.mjs && node --check src/dashboard-installation.mjs && node --check src/runtime-memory-safety.mjs && node --check src/dashboard-memory.mjs && node --check src/runtime-memory-usage.mjs && node --check src/config-profiles.mjs && node --check test/config-profiles.test.mjs", + "check:js": "node --check bin/lloom.mjs && node --check bin/lloom-host.mjs && node --check src/backend-catalog.mjs && node --check src/benchmarks.mjs && node --check src/bootstrap.mjs && node --check src/client-integrations.mjs && node --check src/community-client.mjs && node --check src/community-site.mjs && node --check src/config.mjs && node --check src/dashboard.mjs && node --check src/doctor.mjs && node --check src/host-memory.mjs && node --check src/host-server.mjs && node --check src/init.mjs && node --check src/installer.mjs && node --check src/interchange.mjs && node --check src/machine-profile.mjs && node --check src/managed-environment.mjs && node --check src/model-acquisition.mjs && node --check src/model-intake.mjs && node --check src/model-removal.mjs && node --check src/onboarding.mjs && node --check src/recipe-index.mjs && node --check src/recipe-pack-export.mjs && node --check src/recipe-pack.mjs && node --check src/registry.mjs && node --check src/route-control.mjs && node --check src/recipes.mjs && node --check src/resource-fit.mjs && node --check src/runtime-manager.mjs && node --check src/runtime-policy.mjs && node --check src/server.mjs && node --check src/setup.mjs && node --check src/setup-status.mjs && node --check src/process-control.mjs && node --check src/security.mjs && node --check src/tts-catalog.mjs && node --check src/voice-profiles.mjs && node --check src/protocol/text.mjs && node --check src/protocol/reasoning-normalize.mjs && node --check src/protocol/reasoning-effort.mjs && node --check src/protocol/responses.mjs && node --check src/protocol/anthropic.mjs && node --check src/protocol/sse.mjs && node --check src/protocol/stream-anthropic.mjs && node --check src/protocol/stream-responses.mjs && node --check src/protocol/index.mjs && node --check scripts/apply-spark-route-catalog.mjs && node --check scripts/capture-live-metrics.mjs && node --check scripts/merge-live-metrics.mjs && node --check scripts/run-qualitative-chat-benchmark.mjs && node --check scripts/run-local-decision-benchmark.mjs && node --check scripts/check-community-deploy.mjs && node --check scripts/generate-client-configs.mjs && node --check scripts/check-interchange.mjs && node --check scripts/check-package.mjs && node --check scripts/release-lib.mjs && node --check scripts/build-release.mjs && node --check scripts/deploy-spark.mjs && node --check test/audio-generations.test.mjs && node --check test/community-host.test.mjs && node --check test/dashboard-status.test.mjs && node --check test/ds4fv-recipe.test.mjs && node --check test/dspark-hotfixes.test.mjs && node --check test/glm53-exl3-recipe.test.mjs && node --check test/host-memory.test.mjs && node --check test/metrics-persistence.test.mjs && node --check test/model-acquisition.test.mjs && node --check test/model-failover.test.mjs && node --check test/qwen38-vllm-recipe.test.mjs && node --check test/reasoning-normalize.test.mjs && node --check test/reasoning-effort.test.mjs && node --check test/resource-fit.test.mjs && node --check test/route-control.test.mjs && node --check test/spark-route-catalog.test.mjs && node --check test/runtime-watchdog.test.mjs && node --check test/tts-catalog.test.mjs && node --check test/voice-profiles.test.mjs && node --check test/chatterbox-recipe.test.mjs && node --check test/hear-recipe.test.mjs && node --check test/smoke.mjs && node --check src/runtime-supervisor.mjs && node --check test/runtime-supervisor.test.mjs && node --check test/comfyui-media.test.mjs && node --check src/runtime-residency.mjs && node --check src/dashboard-scene.mjs && node --check src/dashboard-presence.mjs && node --check src/dashboard-presence-client.mjs && node --check src/first-run.mjs && node --check src/first-run-page.mjs && node --check src/browser-setup.mjs && node --check src/runtime-preferences.mjs && node --check src/installation-jobs.mjs && node --check src/model-installation.mjs && node --check src/dashboard-installation.mjs && node --check src/runtime-memory-safety.mjs && node --check src/dashboard-memory.mjs && node --check src/runtime-memory-usage.mjs && node --check src/config-profiles.mjs && node --check test/config-profiles.test.mjs && node --check src/runtime-policy-config.mjs && node --check test/runtime-policy-config.test.mjs", "check:python": "python3 -m py_compile backends/dspark-vllm/apply-patch-pack.py backends/dspark-vllm/packs/miaai-dsv4flash-d1b76251-defaults/patches/*.py backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/*.py backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/vision_exp/*.py backends/qwen38-vllm/apply-ple-fp8-patch.py backends/qwen38-sglang/*.py backends/glm53-exl3/*.py backends/mlx-audio/lloom_audio_server.py backends/hear/lloom_hear_server.py backends/chatterbox/lloom_chatterbox_server.py patches/apply_mtplx_longctx_fix.py backends/comfyui-media/bridge/*.py backends/comfyui-media/graphs/*.py backends/comfyui-media/build/*.py backends/comfyui-media/install.py backends/qwen-image-diffusers/*.py backends/hear/test/test_hear_security.py src/darwin-memory-usage.py", "check:shell": "bash -n backends/dspark-vllm/entrypoint.sh backends/dspark-vllm/runtime/build-hardened-image.sh backends/dspark-vllm/packs/miaai-dsv4flash-d1b76251-defaults/patches/*.sh backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/apply-runtime.sh backends/dspark-vllm/packs/miaai-ds4fv-f5665e8/patches/*.sh backends/qwen38-vllm/entrypoint.sh backends/qwen38-sglang/entrypoint.sh backends/glm53-exl3/entrypoint.sh backends/mlx-audio/install.sh backends/hear/install.sh backends/chatterbox/install.sh clients/examples/lloom-claude clients/examples/lloom-codex clients/examples/lloom-hermes clients/examples/lloom-opencode clients/examples/lloom-zero scripts/remote-install-spark.sh scripts/remote-stage-spark-worker.sh", "check:spark-deploy": "node --check test/spark-deploy.test.mjs && node test/spark-deploy.test.mjs", @@ -86,7 +86,7 @@ "test:comfyui": "python3 scripts/test-comfyui-media.py", "test:hear": "python3 -m unittest discover -s backends/hear/test -v", "test:ux": "node --test test/first-run.test.mjs test/runtime-preferences.test.mjs test/admin-browser-security.test.mjs test/installation-jobs.test.mjs", - "test:memory-safety": "node --test test/runtime-memory-safety.test.mjs" + "test:memory-safety": "node --test test/runtime-memory-safety.test.mjs test/runtime-policy-config.test.mjs" }, "engines": { "node": ">=22.19.0" diff --git a/src/config.mjs b/src/config.mjs index 9ed7402..26a7c2a 100644 --- a/src/config.mjs +++ b/src/config.mjs @@ -370,6 +370,41 @@ function validateConfig(config, sourcePath, env) { } } + // Top-level numeric memory-policy fields. These mirror the ranges used by the + // runtime admission and memory-safety layers so a hand-edited or CLI-written + // config fails loudly instead of silently falling through to defaults. + const topLevelMaxMemoryUtilization = config.runtimePolicy?.maxMemoryUtilization; + if (topLevelMaxMemoryUtilization != null) { + if ( + typeof topLevelMaxMemoryUtilization !== 'number' || + !Number.isFinite(topLevelMaxMemoryUtilization) || + topLevelMaxMemoryUtilization <= 0 || + topLevelMaxMemoryUtilization >= 1 + ) { + errors.push('runtimePolicy.maxMemoryUtilization must be a finite number greater than 0 and less than 1'); + } + } + const topLevelReserveMemoryGb = config.runtimePolicy?.reserveMemoryGb; + if (topLevelReserveMemoryGb != null) { + if ( + typeof topLevelReserveMemoryGb !== 'number' || + !Number.isFinite(topLevelReserveMemoryGb) || + topLevelReserveMemoryGb <= 0 + ) { + errors.push('runtimePolicy.reserveMemoryGb must be a positive finite number'); + } + } + const topLevelMemoryBudgetGb = config.runtimePolicy?.memoryBudgetGb; + if (topLevelMemoryBudgetGb != null) { + if ( + typeof topLevelMemoryBudgetGb !== 'number' || + !Number.isFinite(topLevelMemoryBudgetGb) || + topLevelMemoryBudgetGb <= 0 + ) { + errors.push('runtimePolicy.memoryBudgetGb must be a positive finite number'); + } + } + const memorySafety = config.runtimePolicy?.memorySafety; if (memorySafety != null) { if (typeof memorySafety !== 'object' || Array.isArray(memorySafety)) { diff --git a/src/installer.mjs b/src/installer.mjs index b35a0b5..893e5a5 100644 --- a/src/installer.mjs +++ b/src/installer.mjs @@ -3,7 +3,12 @@ import fs from 'node:fs/promises'; import path from 'node:path'; import { planBackend } from './backend-catalog.mjs'; import { defaultLloomHome, defaultUserModelRoot } from './config.mjs'; -import { finalizeModelAcquisition, modelAcquisitionStatus, prepareModelAcquisition } from './model-acquisition.mjs'; +import { + finalizeModelAcquisition, + modelAcquisitionStatus, + prepareModelAcquisition, + recoverModelAcquisitionDestination +} from './model-acquisition.mjs'; import { runCommand } from './process-control.mjs'; import { planRecipe } from './recipes.mjs'; @@ -225,20 +230,45 @@ async function executeDownloadModel(step, { env = process.env, stdio } = {}) { ); const workCommand = template.map((arg, index) => (template[index - 1] === '--local-dir' ? prepared.workPath : arg)); commands.push(workCommand); - execution = await executeCommand(workCommand, { env, stdio }); - if (!execution.ok) return { ...execution, commands, status: 'failed', partialDestination: prepared.workPath }; + try { + execution = await executeCommand(workCommand, { env, stdio }); + } catch (error) { + execution = { + ok: false, + status: 'failed', + command: workCommand, + code: null, + stdout: '', + stderr: error?.message ?? String(error) + }; + } + if (!execution.ok) { + const recovery = await recoverModelAcquisitionDestination(prepared); + return { + ...execution, + commands, + status: 'failed', + ...(!recovery.restored ? { partialDestination: prepared.workPath } : {}), + ...(recovery.restored ? { destinationRestored: true } : {}), + ...(recovery.recoveryError + ? { stderr: [execution.stderr, recovery.recoveryError].filter(Boolean).join('\n') } + : {}) + }; + } } execution.commands = commands; try { const acquisition = await finalizeModelAcquisition(step, prepared); return { ...execution, status: 'completed', acquisition }; } catch (error) { + const recovery = await recoverModelAcquisitionDestination(prepared); return { ...execution, ok: false, status: 'failed', - stderr: [execution.stderr, error?.message ?? String(error)].filter(Boolean).join('\n'), - partialDestination: prepared.workPath + stderr: [execution.stderr, error?.message ?? String(error), recovery.recoveryError].filter(Boolean).join('\n'), + ...(!recovery.restored ? { partialDestination: prepared.workPath } : {}), + ...(recovery.restored ? { destinationRestored: true } : {}) }; } } diff --git a/src/model-acquisition.mjs b/src/model-acquisition.mjs index b0696b2..e4ac08c 100644 --- a/src/model-acquisition.mjs +++ b/src/model-acquisition.mjs @@ -5,12 +5,67 @@ import { matchIncludedFiles, modelDirectoryComplete, validModelFilePattern } fro export const MODEL_ACQUISITION_MANIFEST = '.lloom-acquisition.json'; -async function pathExists(filePath) { +async function pathState(filePath) { try { - await fs.access(filePath); + const stat = await fs.lstat(filePath); + return { exists: true, stat }; + } catch (error) { + if (error?.code === 'ENOENT') return { exists: false }; + return { exists: null, error }; + } +} + +function samePathIdentity(left, right) { + return Boolean( + left?.exists && right?.exists && left.stat?.dev === right.stat?.dev && left.stat?.ino === right.stat?.ino + ); +} + +async function directoryWritableByWrite(directory) { + let stat; + try { + stat = await fs.stat(directory); + } catch { + return false; + } + if (!stat.isDirectory()) return false; + let scratch; + try { + // mkdtemp owns the generated name. Never unlink a fixed probe path that + // could have belonged to the model or a concurrent downloader. + scratch = await fs.mkdtemp(path.join(directory, '.lloom-write-probe-')); return true; } catch { return false; + } finally { + if (scratch) await fs.rm(scratch, { recursive: true, force: true }).catch(() => {}); + } +} + +// Downloaders write below `/.cache`. The checkpoint directory may +// already exist with a nested cache the current user cannot write; moving the +// destination aside would then hide the only copy of the checkpoint and the +// download would fail into an unusable work path. Probe the cache directories we +// know about before any rename and fail with a remediation message instead. +async function assertCacheWritable(destination, model) { + const candidates = [ + destination, + path.join(destination, '.cache'), + path.join(destination, '.cache', 'huggingface'), + path.join(destination, '.cache', 'huggingface', 'download') + ]; + for (const candidate of candidates) { + const state = await pathState(candidate); + if (state.error) { + throw new Error( + `cannot inspect ${candidate} for ${model}: ${state.error.message}; the existing model directory or its download cache may not be accessible to the current user. Fix ownership or permissions for that directory (no automatic chmod/chown is performed), or move it aside manually, then retry.` + ); + } + if (!state.exists) continue; + if (await directoryWritableByWrite(candidate)) continue; + throw new Error( + `cannot write to ${candidate} for ${model}; the existing model directory or its download cache is not writable by the current user. Fix ownership or permissions for that directory (no automatic chmod/chown is performed), or move it aside manually, then retry.` + ); } } @@ -108,7 +163,66 @@ async function readManifest(destination) { } } -export async function modelAcquisitionStatus(step = {}) { +async function stagePreviousManifest(destination) { + const manifestPath = path.join(destination, MODEL_ACQUISITION_MANIFEST); + let raw; + try { + raw = await fs.readFile(manifestPath); + } catch (error) { + if (error?.code === 'ENOENT') return null; + throw error; + } + + const backupDirectory = await fs.mkdtemp( + path.join(path.dirname(destination), `.lloom-acquisition-previous-${path.basename(destination)}-`) + ); + const backupPath = path.join(backupDirectory, MODEL_ACQUISITION_MANIFEST); + try { + // Move the marker out while the canonical directory is still in place. If + // this fails, no rename has happened and the completed destination stays + // visible with its original provenance. + await fs.rename(manifestPath, backupPath); + } catch (error) { + await fs.rm(backupDirectory, { recursive: true, force: true }).catch(() => {}); + throw error; + } + + let manifest = null; + try { + manifest = JSON.parse(raw.toString('utf8')); + } catch { + // Preserve malformed bytes for inspection, but do not reuse them as + // acquisition metadata during this attempt. + } + return { backupDirectory, backupPath, manifest }; +} + +async function restorePreviousManifest(previous, destination, { expectedIdentity } = {}) { + if (!previous?.backupPath) return { restored: true }; + const manifestPath = path.join(destination, MODEL_ACQUISITION_MANIFEST); + const destinationState = await pathState(destination); + if (destinationState.error) return { restored: false, error: destinationState.error }; + if (expectedIdentity && !samePathIdentity(destinationState, expectedIdentity)) { + return { restored: false, error: new Error(`${destination} changed while acquisition was being prepared`) }; + } + const state = await pathState(manifestPath); + if (state.error) return { restored: false, error: state.error }; + if (state.exists) return { restored: false, error: new Error(`${manifestPath} already exists`) }; + try { + await fs.rename(previous.backupPath, manifestPath); + return { restored: true }; + } catch (error) { + return { restored: false, error }; + } +} + +async function cleanupPreviousManifest(previous) { + if (previous?.backupDirectory) { + await fs.rm(previous.backupDirectory, { recursive: true, force: true }).catch(() => {}); + } +} + +export async function modelAcquisitionStatus(step = {}, { manifestOverride } = {}) { const destination = step.destination; const spec = acquisitionSpec(step); const payloadComplete = Boolean( @@ -120,7 +234,7 @@ export async function modelAcquisitionStatus(step = {}) { if (!payloadComplete) return { complete: false, payloadComplete: false, verified: false, reason: 'payload-missing' }; const constrained = Boolean(spec.revision || spec.files.length || spec.include.length); if (!constrained) return { complete: true, payloadComplete: true, verified: false, reason: 'payload-present' }; - const manifest = await readManifest(destination); + const manifest = manifestOverride ?? (await readManifest(destination)); if (spec.revision && manifest?.revision !== spec.revision) { return { complete: false, payloadComplete: true, verified: false, reason: 'revision-unverified', manifest }; } @@ -200,11 +314,25 @@ export async function prepareModelAcquisition(step = {}) { const destination = step.destination; const incomplete = `${destination}.incomplete`; await fs.mkdir(path.dirname(destination), { recursive: true }); - const destinationExists = await pathExists(destination); - const incompleteExists = await pathExists(incomplete); + const destinationState = await pathState(destination); + const incompleteState = await pathState(incomplete); + if (destinationState.error) { + throw new Error(`cannot inspect existing model destination ${destination}: ${destinationState.error.message}`); + } + if (incompleteState.error) { + throw new Error(`cannot inspect partial model destination ${incomplete}: ${incompleteState.error.message}`); + } + const destinationExists = destinationState.exists; + const incompleteExists = incompleteState.exists; if (destinationExists && incompleteExists) { throw new Error(`both partial model paths exist; reconcile ${destination} and ${incomplete} before retrying`); } + // Probe the destinations' download caches before touching anything. If the + // existing checkpoint (or a cache under it) is not writable, fail here while + // the canonical destination is still visible instead of renaming a checkpoint + // we cannot actually reuse. + if (destinationExists) await assertCacheWritable(destination, step.model); + if (incompleteExists) await assertCacheWritable(incomplete, step.model); const spec = acquisitionSpec(step); if (spec.downloadSizeBytes != null) { const stats = await fs.statfs(path.dirname(destination)); @@ -216,14 +344,54 @@ export async function prepareModelAcquisition(step = {}) { ); } } - if (destinationExists) await fs.rename(destination, incomplete); - await fs.mkdir(incomplete, { recursive: true }); - return { destination, workPath: incomplete, spec }; + const previousManifest = destinationExists ? await stagePreviousManifest(destination) : null; + const originalDestinationIdentity = destinationState; + let priorDestinationMoved = false; + try { + if (destinationExists) { + await fs.rename(destination, incomplete); + priorDestinationMoved = true; + } + await fs.mkdir(incomplete, { recursive: true }); + } catch (error) { + if (priorDestinationMoved) { + const destinationStateAfterFailure = await pathState(destination); + const incompleteStateAfterFailure = await pathState(incomplete); + if ( + !destinationStateAfterFailure.error && + !destinationStateAfterFailure.exists && + samePathIdentity(incompleteStateAfterFailure, originalDestinationIdentity) + ) { + try { + await fs.rename(incomplete, destination); + } catch { + // Keep the staged directory and previous marker backup visible for + // manual recovery when a concurrent writer won the destination. + } + } + } + if (previousManifest) { + const restored = await restorePreviousManifest(previousManifest, destination, { + expectedIdentity: originalDestinationIdentity + }); + if (!restored.restored) { + throw new Error( + `${error?.message ?? String(error)}; could not restore previous acquisition manifest: ${ + restored.error?.message ?? String(restored.error) + }; preserved bytes remain at ${previousManifest.backupPath}`, + { cause: error } + ); + } + await cleanupPreviousManifest(previousManifest); + } + throw error; + } + return { destination, workPath: incomplete, spec, priorDestinationMoved, previousManifest }; } export async function finalizeModelAcquisition(step, prepared) { const workStep = { ...step, destination: prepared.workPath }; - const previous = await readManifest(prepared.workPath); + const previous = prepared.previousManifest?.manifest ?? (await readManifest(prepared.workPath)); const reusable = previous?.provider === prepared.spec.provider && previous?.model === prepared.spec.model && @@ -238,12 +406,74 @@ export async function finalizeModelAcquisition(step, prepared) { files: prepared.spec.files, completedAt: new Date().toISOString() }; + const status = await modelAcquisitionStatus(workStep, { manifestOverride: manifest }); + if (!status.complete) throw new Error(`download verification failed for ${step.model}: ${status.reason}`); + // Do not publish acquisition provenance until the payload has passed every + // requested check. A failed verification must never leave a completion + // manifest behind when the staged directory is restored. await fs.writeFile( path.join(prepared.workPath, MODEL_ACQUISITION_MANIFEST), `${JSON.stringify(manifest, null, 2)}\n` ); - const status = await modelAcquisitionStatus(workStep); - if (!status.complete) throw new Error(`download verification failed for ${step.model}: ${status.reason}`); + // A concurrent installer may have published a destination between our rename + // aside and now. Refuse to clobber it; keep the staged copy for manual review. + const destinationState = await pathState(prepared.destination); + if (destinationState.error) { + throw new Error( + `could not inspect ${prepared.destination}: ${destinationState.error.message}; refusing to publish while the destination state is unknown` + ); + } + if (destinationState.exists) { + throw new Error( + `refusing to overwrite ${prepared.destination}: another destination appeared while ${step.model} was downloading; reconcile the staged copy at ${prepared.workPath} before retrying` + ); + } await fs.rename(prepared.workPath, prepared.destination); + await cleanupPreviousManifest(prepared.previousManifest); return { ...status, manifest, destination: prepared.destination }; } + +// Restores the checkpoint that prepareModelAcquisition moved aside when a later +// download/verify/finalize step fails. Only restores when the canonical +// destination is absent, so a concurrent writer is never overwritten. A fresh +// failed download (no prior destination) stays staged under `.incomplete` +// so its partial payload can be resumed. +export async function recoverModelAcquisitionDestination(prepared = {}) { + if (!prepared?.priorDestinationMoved) return { restored: false }; + const destination = prepared.destination; + const workPath = prepared.workPath; + const destinationState = destination ? await pathState(destination) : { exists: false }; + if (destinationState.error) { + return { + restored: false, + recoveryError: `could not inspect ${destination}: ${destinationState.error.message}; the staged checkpoint remains at ${workPath}` + }; + } + if (!destinationState.exists) { + const workState = workPath ? await pathState(workPath) : { exists: false }; + if (workState.error) { + return { + restored: false, + recoveryError: `could not inspect staged checkpoint ${workPath}: ${workState.error.message}; it was left untouched` + }; + } + const stagedPresent = workState.exists; + if (stagedPresent) { + try { + await fs.rename(workPath, destination); + return { restored: true }; + } catch (error) { + return { + restored: false, + recoveryError: `could not restore ${destination}: ${error?.message ?? String(error)}; the staged checkpoint remains at ${workPath}` + }; + } + } + } + return { + restored: false, + recoveryError: destinationState.exists + ? `could not restore ${destination}: another model directory appeared during download; the previous checkpoint remains staged at ${workPath}` + : `could not restore ${destination}: the staged checkpoint at ${workPath} is missing` + }; +} diff --git a/src/model-installation.mjs b/src/model-installation.mjs index 39108fa..fabcd94 100644 --- a/src/model-installation.mjs +++ b/src/model-installation.mjs @@ -1,7 +1,12 @@ import { defaultBackendVariables, getBackend, loadBackendCatalog } from './backend-catalog.mjs'; import { applyBackend } from './installer.mjs'; import { runCommand } from './process-control.mjs'; -import { modelAcquisitionStatus, prepareModelAcquisition, finalizeModelAcquisition } from './model-acquisition.mjs'; +import { + modelAcquisitionStatus, + prepareModelAcquisition, + finalizeModelAcquisition, + recoverModelAcquisitionDestination +} from './model-acquisition.mjs'; export async function installImportedModelAssets(plan, { onProgress, backendCatalog, env = process.env } = {}) { const variables = defaultBackendVariables(env); @@ -34,8 +39,39 @@ export async function installImportedModelAssets(plan, { onProgress, backendCata const [command, ...args] = prepared ? planned.map((arg, index) => (planned[index - 1] === '--local-dir' ? prepared.workPath : arg)) : planned; - const result = await runCommand(command, args, { allowFailure: true, env: installEnv, stdio: 'inherit' }); - if (result.code !== 0) + let result; + try { + result = await runCommand(command, args, { allowFailure: true, env: installEnv, stdio: 'inherit' }); + } catch (error) { + const recovery = prepared ? await recoverModelAcquisitionDestination(prepared) : { restored: false }; + throw new Error([error?.message ?? String(error), recovery.recoveryError].filter(Boolean).join('\n'), { + cause: error + }); + } + if (result.code !== 0) { + if (prepared) { + const recovery = await recoverModelAcquisitionDestination(prepared); + if (recovery.recoveryError) { + throw new Error( + [ + result.stderr || 'Model download failed. Check the vendor requirements and credentials.', + recovery.recoveryError + ] + .filter(Boolean) + .join('\n') + ); + } + } throw new Error(result.stderr || 'Model download failed. Check the vendor requirements and credentials.'); - if (prepared) await finalizeModelAcquisition(acquisition, prepared); + } + if (prepared) { + try { + await finalizeModelAcquisition(acquisition, prepared); + } catch (error) { + const recovery = await recoverModelAcquisitionDestination(prepared); + throw new Error([error?.message ?? String(error), recovery.recoveryError].filter(Boolean).join('\n'), { + cause: error + }); + } + } } diff --git a/src/protocol/responses.mjs b/src/protocol/responses.mjs index 3ed51cb..62827a8 100644 --- a/src/protocol/responses.mjs +++ b/src/protocol/responses.mjs @@ -11,6 +11,21 @@ import { import { normalizeOpenAIChatCompletionBody, normalizeOpenAIChatRequestBody } from './reasoning-normalize.mjs'; import { translateReasoningEffortForBackend } from './reasoning-effort.mjs'; +function openAIUrlDescriptor(value) { + if (typeof value === 'string' && value) return { url: value }; + if (value && typeof value === 'object' && typeof value.url === 'string' && value.url) { + return { ...value }; + } + return null; +} + +function contentMediaPart(type, value, fallback = null, detail) { + const descriptor = openAIUrlDescriptor(value) ?? openAIUrlDescriptor(fallback); + if (!descriptor) return null; + if (detail !== undefined && descriptor.detail === undefined) descriptor.detail = detail; + return { type, [type]: descriptor }; +} + export function responsesContentPartToOpenAI(part) { if (typeof part === 'string') return { type: 'text', text: part }; if (!part || typeof part !== 'object') return { type: 'text', text: String(part ?? '') }; @@ -18,17 +33,14 @@ export function responsesContentPartToOpenAI(part) { return { type: 'text', text: part.text ?? '' }; } if (part.type === 'input_image') { - const imageUrl = part.image_url ?? part.url; - if (imageUrl) { - return { - type: 'image_url', - image_url: { - url: imageUrl - } - }; - } + const image = contentMediaPart('image_url', part.image_url, part.url, part.detail); + if (image) return image; } if (part.type === 'image_url') return part; + if (part.type === 'input_video' || part.type === 'video_url' || part.type === 'video') { + const video = contentMediaPart('video_url', part.video_url, part.url ?? part.video, part.detail); + if (video) return video; + } return { type: 'text', text: part.text ?? JSON.stringify(part) }; } diff --git a/src/protocol/structured-output.mjs b/src/protocol/structured-output.mjs index 2caf765..02ef432 100644 --- a/src/protocol/structured-output.mjs +++ b/src/protocol/structured-output.mjs @@ -49,7 +49,85 @@ function prepareToolChoiceForBackend(body, resolved) { return { ...body, thinking: { ...(isObject(body.thinking) ? body.thinking : {}), type: 'disabled' } }; } +// JSON Schema's `type` keyword. A caller that writes `type: "invalid"` is +// asking for a constraint that cannot be compiled, and a backend that ignores +// it answers as if the request were unconstrained — the caller gets a 200 and +// silently loses the guarantee they asked for. +const JSON_SCHEMA_TYPES = new Set(['null', 'boolean', 'object', 'array', 'number', 'string', 'integer']); + +function childSchemas(node) { + const children = []; + for (const key of ['items', 'additionalProperties', 'not', 'if', 'then', 'else', 'contains', 'propertyNames']) { + if (isObject(node[key])) children.push([`${key}`, node[key]]); + } + for (const key of ['properties', 'patternProperties', 'definitions', '$defs', 'dependentSchemas']) { + if (isObject(node[key])) { + for (const [name, child] of Object.entries(node[key])) { + if (isObject(child)) children.push([`${key}.${name}`, child]); + } + } + } + for (const key of ['allOf', 'anyOf', 'oneOf', 'prefixItems']) { + if (Array.isArray(node[key])) { + node[key].forEach((child, index) => { + if (isObject(child)) children.push([`${key}[${index}]`, child]); + }); + } + } + return children; +} + +/** + * Reject a schema no JSON Schema implementation could compile. + * + * Deliberately conservative: `$ref`, `enum`, `const`, `format`, unknown + * keywords and any other construct a valid schema may carry are left alone. + * Only a `type` that is not one of the seven JSON Schema types is refused, at + * the root or anywhere the specification places a subschema. + */ +export function assertCompilableSchema(schema, label = 'response_format.json_schema.schema') { + const visit = (node, path, depth) => { + if (depth > 32) return; + if (Object.hasOwn(node, 'type')) { + const value = node.type; + const values = Array.isArray(value) ? value : [value]; + if (!values.length || values.some((item) => typeof item !== 'string' || !JSON_SCHEMA_TYPES.has(item))) { + throw new StructuredOutputError( + `${label}${path}: "type" must be one of ${[...JSON_SCHEMA_TYPES].join(', ')}`, + 'invalid_json_schema' + ); + } + } + for (const [step, child] of childSchemas(node)) visit(child, `${path}.${step}`, depth + 1); + }; + visit(schema, '', 0); + return schema; +} + +function validateCallerResponseFormat(body) { + const format = body.response_format; + if (format == null) return; + if (!isObject(format)) { + throw new StructuredOutputError('response_format must be an object', 'invalid_response_format'); + } + if (format.type !== 'json_schema') return; + if (!isObject(format.json_schema)) { + throw new StructuredOutputError('response_format.json_schema must be an object', 'invalid_json_schema'); + } + const { schema } = format.json_schema; + if (schema != null) { + if (!isObject(schema)) { + throw new StructuredOutputError( + 'response_format.json_schema.schema must be a JSON Schema object', + 'invalid_json_schema' + ); + } + assertCompilableSchema(schema); + } +} + export function prepareStructuredOutputForBackend(body = {}, resolved = {}) { + validateCallerResponseFormat(body); const contract = isObject(body.lloom) ? body.lloom.outputSchema : null; if (contract == null) return { body: prepareToolChoiceForBackend(body, resolved), output: null }; @@ -57,6 +135,7 @@ export function prepareStructuredOutputForBackend(body = {}, resolved = {}) { if (!isObject(contract) || !isObject(contract.schema)) { throw new StructuredOutputError('lloom.outputSchema.schema must be a JSON Schema object'); } + assertCompilableSchema(contract.schema, 'lloom.outputSchema.schema'); const name = typeof contract.name === 'string' && contract.name.trim() ? contract.name.trim() : 'structured_output'; const settings = isObject(resolved.backend?.structuredOutput) ? resolved.backend.structuredOutput : {}; diff --git a/src/runtime-policy-config.mjs b/src/runtime-policy-config.mjs new file mode 100644 index 0000000..8c58d04 --- /dev/null +++ b/src/runtime-policy-config.mjs @@ -0,0 +1,254 @@ +// Explicit numeric memory-policy configuration for the CLI. +// +// This module owns the `runtime-policy` command surface: parsing numeric +// memory limits, building a read-only dry-run report, and applying a +// minimal, validated mutation to the resolved installed config file. +// +// Design boundaries (see AGENTS.md): +// - runtime-plan stays read-only. Mutating anything requires +// `--apply --yes` on `runtime-policy`. +// - We never set memorySafety.mode: no hidden yolo, no implicit +// enforcement disable. +// - We never touch cluster.nodes.*.resources; per-node overrides stay +// operator-owned. Recent stricter node overrides are surfaced in the +// dry-run report with an explicit warning instead. +// - Writes go through mutateConfigSource (atomic, validated), never a raw +// fs write. +import { mutateConfigSource } from './config-mutation.mjs'; + +export const NUMERIC_MEMORY_FLAGS = Object.freeze(['--max-memory-utilization', '--reserve-memory-gb']); + +class NumericMemoryFlagError extends Error { + constructor(message) { + super(message); + this.name = 'NumericMemoryFlagError'; + this.code = 'invalid_memory_policy_flag'; + this.statusCode = 2; + } +} + +function rawFlagValue(args, name) { + const tokens = Array.isArray(args) ? args : []; + if (tokens.filter((token) => token === name).length > 1) { + throw new NumericMemoryFlagError(`${name} must be supplied only once`); + } + const index = tokens.indexOf(name); + if (index === -1) return undefined; + const next = tokens[index + 1]; + if (next === undefined || (typeof next === 'string' && next.startsWith('--'))) { + throw new NumericMemoryFlagError(`${name} requires a numeric value`); + } + return next; +} + +function parseMaxMemoryUtilization(raw) { + if (typeof raw !== 'string' || raw.trim() === '') { + throw new NumericMemoryFlagError('--max-memory-utilization requires a numeric value'); + } + const value = Number(raw); + if (!Number.isFinite(value)) { + throw new NumericMemoryFlagError(`--max-memory-utilization must be a finite number; received "${raw}"`); + } + if (!(value > 0 && value < 1)) { + throw new NumericMemoryFlagError( + `--max-memory-utilization must be greater than 0 and less than 1; received ${value}` + ); + } + return value; +} + +function parseReserveMemoryGb(raw) { + if (typeof raw !== 'string' || raw.trim() === '') { + throw new NumericMemoryFlagError('--reserve-memory-gb requires a numeric value'); + } + const value = Number(raw); + if (!Number.isFinite(value)) { + throw new NumericMemoryFlagError(`--reserve-memory-gb must be a finite number; received "${raw}"`); + } + if (!(value > 0)) { + throw new NumericMemoryFlagError(`--reserve-memory-gb must be greater than 0; received ${value}`); + } + return value; +} + +/** + * Extract numeric memory-policy flags. Returns null when neither flag is + * present so callers can preserve the existing read-only behavior. + */ +export function parseNumericMemoryFlags(args) { + const tokens = Array.isArray(args) ? args : []; + for (const name of NUMERIC_MEMORY_FLAGS) { + if (tokens.some((token) => token.startsWith(`${name}=`))) { + throw new NumericMemoryFlagError(`${name} requires a separate numeric argument`); + } + } + const hasMax = tokens.includes('--max-memory-utilization'); + const hasReserve = tokens.includes('--reserve-memory-gb'); + if (!hasMax && !hasReserve) return null; + const flags = {}; + if (hasMax) flags.maxMemoryUtilization = parseMaxMemoryUtilization(rawFlagValue(tokens, '--max-memory-utilization')); + if (hasReserve) flags.reserveMemoryGb = parseReserveMemoryGb(rawFlagValue(tokens, '--reserve-memory-gb')); + return Object.keys(flags).length ? flags : null; +} + +function numberOrNull(value) { + if (value == null || value === '') return null; + const number = Number(value); + return Number.isFinite(number) ? number : null; +} + +// Per-node resource overrides that an operator may set to constrain a specific +// node. We surface conflicts but never write these. +function nodeOverrideRows(config, flags) { + const nodes = config?.cluster?.nodes; + if (!nodes || typeof nodes !== 'object' || Array.isArray(nodes)) return []; + const rows = []; + for (const [nodeId, node] of Object.entries(nodes)) { + const resources = node?.resources; + if (!resources || typeof resources !== 'object' || Array.isArray(resources)) continue; + const row = { + nodeId, + reserveMemoryGb: numberOrNull(resources.reserveMemoryGb), + maxMemoryUtilization: numberOrNull(resources.maxMemoryUtilization), + memoryBudgetGb: numberOrNull(resources.memoryBudgetGb) + }; + rows.push(row); + } + return rows.filter((row) => + flags ? row.reserveMemoryGb != null || row.maxMemoryUtilization != null || row.memoryBudgetGb != null : true + ); +} + +// Warn explicitly when a per-node override is stricter than the requested +// global value (so the global change will not take effect on that node). +function nodeOverrideWarnings(config, flags) { + const warnings = []; + const rows = nodeOverrideRows(config, flags); + for (const row of rows) { + if ( + flags.maxMemoryUtilization != null && + row.maxMemoryUtilization != null && + row.maxMemoryUtilization < flags.maxMemoryUtilization + ) { + warnings.push( + `cluster.nodes.${row.nodeId}.resources.maxMemoryUtilization (${row.maxMemoryUtilization}) is stricter than the requested ${flags.maxMemoryUtilization}; the requested global value will not take effect on node ${row.nodeId}` + ); + } + if (flags.reserveMemoryGb != null && row.reserveMemoryGb != null && row.reserveMemoryGb > flags.reserveMemoryGb) { + warnings.push( + `cluster.nodes.${row.nodeId}.resources.reserveMemoryGb (${row.reserveMemoryGb}) is stricter than the requested ${flags.reserveMemoryGb}; the requested global reserve will not take effect on node ${row.nodeId}` + ); + } + if (row.memoryBudgetGb != null) { + const requestedBudget = + flags.maxMemoryUtilization != null ? `≈ totalMemoryGb * ${flags.maxMemoryUtilization}` : null; + warnings.push( + `cluster.nodes.${row.nodeId}.resources.memoryBudgetGb (${row.memoryBudgetGb}) is an absolute per-node budget${ + requestedBudget ? ` and overrides the requested percentage-based budget (${requestedBudget})` : '' + }; per-node overrides are preserved and the requested global policy may not take effect on node ${row.nodeId}` + ); + } + } + return warnings; +} + +function selectedConfigFields(config) { + const policy = config?.runtimePolicy ?? {}; + const safety = policy.memorySafety ?? {}; + return { + runtimePolicy: { + maxMemoryUtilization: numberOrNull(policy.maxMemoryUtilization), + reserveMemoryGb: numberOrNull(policy.reserveMemoryGb), + memoryBudgetGb: numberOrNull(policy.memoryBudgetGb) + }, + memorySafety: { + maxMemoryUtilization: numberOrNull(safety.maxMemoryUtilization), + minAvailableMemoryGb: numberOrNull(safety.minAvailableMemoryGb), + mode: safety.mode ?? null + } + }; +} + +function projectAfter(config, flags) { + const policy = { ...(config?.runtimePolicy ?? {}) }; + const safety = { ...(policy.memorySafety ?? {}) }; + if (flags.maxMemoryUtilization != null) { + policy.maxMemoryUtilization = flags.maxMemoryUtilization; + safety.maxMemoryUtilization = flags.maxMemoryUtilization; + } + if (flags.reserveMemoryGb != null) { + policy.reserveMemoryGb = flags.reserveMemoryGb; + safety.minAvailableMemoryGb = flags.reserveMemoryGb; + } + policy.memorySafety = safety; + return { ...config, runtimePolicy: policy }; +} + +/** + * Build a read-only dry-run report for `lloom runtime-policy`. Returns only + * the selected memory-policy fields before/after plus per-node overrides and + * warnings. It deliberately omits secrets and the entire config body. + */ +export function createRuntimePolicyConfigReport(config, flags) { + if (!flags) { + throw new NumericMemoryFlagError('runtime-policy requires --max-memory-utilization and/or --reserve-memory-gb'); + } + const absolute = numberOrNull(config?.runtimePolicy?.memoryBudgetGb); + if (absolute != null) { + throw new NumericMemoryFlagError( + `runtimePolicy.memoryBudgetGb (${absolute}) is set as an absolute budget and would keep taking precedence over a percentage; remove it explicitly or edit the config before changing utilization/reserve` + ); + } + const after = projectAfter(config, flags); + return { + ok: true, + mode: 'dry-run', + changed: Object.keys(flags), + before: selectedConfigFields(config), + after: selectedConfigFields(after), + nodeOverrides: nodeOverrideRows(config, flags), + warnings: nodeOverrideWarnings(config, flags) + }; +} + +function applyFlagsToConfig(parsed, flags) { + const policy = { ...(parsed.runtimePolicy ?? {}) }; + const safety = { ...(policy.memorySafety ?? {}) }; + if (flags.maxMemoryUtilization != null) { + policy.maxMemoryUtilization = flags.maxMemoryUtilization; + safety.maxMemoryUtilization = flags.maxMemoryUtilization; + } + if (flags.reserveMemoryGb != null) { + policy.reserveMemoryGb = flags.reserveMemoryGb; + safety.minAvailableMemoryGb = flags.reserveMemoryGb; + } + policy.memorySafety = safety; + parsed.runtimePolicy = policy; +} + +/** + * Apply numeric memory-policy flags to the resolved installed config file. + * + * Requires both --apply and --yes (enforced by the CLI handler). The actual + * write goes through mutateConfigSource, which re-reads the file, rejects + * symlinks, writes an atomic temp file, and validates before rename. This + * preserves every unrelated field, per-node overrides, and the existing + * memorySafety.mode. + */ +export async function applyRuntimePolicyConfig(config, flags, { apply = false, yes = false } = {}) { + if (!flags) { + throw new NumericMemoryFlagError('runtime-policy requires --max-memory-utilization and/or --reserve-memory-gb'); + } + if (!apply || !yes) { + return { skipped: true, reason: 'requires --apply --yes', mode: 'dry-run' }; + } + if (!config?.sourcePath) { + throw new NumericMemoryFlagError('resolved config path is required to apply a memory-policy change'); + } + let report; + const result = await mutateConfigSource(config, (parsed) => { + report = createRuntimePolicyConfigReport(parsed, flags); + applyFlagsToConfig(parsed, flags); + }); + return { ...report, mode: 'applied', applied: true, mutated: result?.changed === true }; +} diff --git a/src/server.mjs b/src/server.mjs index 77b174b..7aba96c 100644 --- a/src/server.mjs +++ b/src/server.mjs @@ -343,18 +343,25 @@ function endResponseWithError(res, error, { stream = false, config = {}, status const DEFAULT_IMAGE_TOKEN_ESTIMATE = 4096; const LOW_DETAIL_IMAGE_TOKEN_ESTIMATE = 1024; -function estimateImageTokens(value) { +function estimateMediaTokens(value) { if (!value || typeof value !== 'object') return null; const type = String(value.type ?? '').toLowerCase(); const source = value.source && typeof value.source === 'object' ? value.source : null; - const hasImagePayload = + const hasMediaPayload = type === 'image' || type === 'image_url' || type === 'input_image' || + type === 'video' || + type === 'video_url' || + type === 'input_video' || value.image_url != null || - (source && (source.type === 'base64' || String(source.media_type ?? '').startsWith('image/'))); - if (!hasImagePayload) return null; - const detail = String(value.detail ?? value.image_url?.detail ?? '').toLowerCase(); + value.video_url != null || + (source && + (source.type === 'base64' || + String(source.media_type ?? '').startsWith('image/') || + String(source.media_type ?? '').startsWith('video/'))); + if (!hasMediaPayload) return null; + const detail = String(value.detail ?? value.image_url?.detail ?? value.video_url?.detail ?? '').toLowerCase(); return detail === 'low' ? LOW_DETAIL_IMAGE_TOKEN_ESTIMATE : DEFAULT_IMAGE_TOKEN_ESTIMATE; } @@ -364,13 +371,13 @@ function estimateMessageTokens(value) { // Base64 is opaque media, not prompt text. Counting every encoded byte as // language tokens rejects normal multimodal requests before the backend's // vision processor can turn pixels into its much smaller token sequence. - if (/^data:image\/[a-z0-9.+-]+;base64,/i.test(value)) return DEFAULT_IMAGE_TOKEN_ESTIMATE; + if (/^data:(?:image|video)\/[a-z0-9.+-]+;base64,/i.test(value)) return DEFAULT_IMAGE_TOKEN_ESTIMATE; return Math.ceil(value.length / 3.5); } if (Array.isArray(value)) return value.reduce((sum, item) => sum + estimateMessageTokens(item), 0); if (typeof value === 'object') { - const imageTokens = estimateImageTokens(value); - if (imageTokens != null) return imageTokens; + const mediaTokens = estimateMediaTokens(value); + if (mediaTokens != null) return mediaTokens; if (typeof value.text === 'string') return estimateMessageTokens(value.text); if (typeof value.content === 'string' || Array.isArray(value.content)) { return estimateMessageTokens(value.content); @@ -659,10 +666,26 @@ async function readJson(req) { } function upstreamUrl(backend, path) { + // An absolute URL is already resolved against the backend; joining it to the + // baseUrl would corrupt it. Callers use this for routes a backend serves + // outside its advertised API prefix. + if (/^https?:\/\//i.test(path)) return path; const suffix = path.startsWith('/v1/') ? path.slice(3) : path; return `${stripTrailingSlash(backend.baseUrl)}${suffix}`; } +// The Atlas engine mounts its OpenAI surface under `/v1` but token counting at +// the bare `/tokenize` (`serve_router.rs`: `.route("/v1/chat/completions")` next +// to `.route("/tokenize")`). An OpenAI-typed backend's baseUrl already ends in +// `/v1`, so building this URL the ordinary way would request `…/v1/tokenize` and +// get a 404 — the engine does not serve a `/v1` form of it. Resolve against the +// backend origin instead, so the request lands on `/tokenize` whatever the +// configured baseUrl path is. +function tokenizeUpstreamUrl(backend) { + const origin = new URL(backend.baseUrl).origin; + return `${origin}/tokenize`; +} + function backendHeaders(backend, extra = {}) { const apiKey = backend.apiKeyEnv ? process.env[backend.apiKeyEnv] : backend.apiKey; return { @@ -3202,6 +3225,70 @@ export function createLloomServer( ); } + // Token counting for backends that render a chat template. SparkGLM's Atlas + // engine answers `POST /tokenize` with `{tokens, count}` and applies the same + // template serving uses, so the count matches what a chat request of the same + // messages will really render. The gateway passes the body through untouched + // (including `chat_template_kwargs`) because altering it would change the count + // this endpoint exists to report. + // + // The upstream path is the backend's own route, not the gateway path: an + // OpenAI-typed backend already carries `/v1` in its baseUrl, and `upstreamUrl` + // only strips a `/v1/` prefix when the baseUrl lacks one. Passing `/v1/tokenize` + // here would therefore request `…/v1/v1/tokenize` and 404. + async function handleOpenAITokenize(req, res) { + const body = await readJson(req); + const modelId = body.model ?? config.defaults?.chatModel; + if (body.prompt == null && body.messages == null) { + sendJson( + res, + 400, + errorBody("tokenize request requires 'prompt' or 'messages'", { + code: 'missing_input' + }) + ); + return; + } + if (!modelId) { + sendJson( + res, + 400, + errorBody('tokenize request requires model', { + code: 'missing_model' + }) + ); + return; + } + await recordModelRequestWithFailover( + { + route: '/v1/tokenize', + modelId, + stream: false, + req, + res + }, + async (resolved, { signal, timing, watchdog, hasNext }) => { + watchdog.arm(); + const upstream = await fetchUpstream({ + headers: inferenceGatewayHeaders(req, resolved), + backend: resolved.backend, + // Absolute URL: `fetchUpstream` passes it through untouched, which is + // what keeps the request off the backend's `/v1` base path. + path: tokenizeUpstreamUrl(resolved.backend), + signal, + body: { + ...body, + model: resolved.model.upstreamModel + } + }); + if (!upstream.ok && hasNext && MODEL_FAILOVER_STATUS_CODES.has(upstream.status)) { + throw await upstreamStatusError(upstream); + } + return proxyRawResponse(res, upstream, { signal, timing, corsConfig: config }); + } + ); + } + async function resolveSpeechModelOrError(modelId) { try { const resolved = await resolveRequestModel(modelId); @@ -4537,6 +4624,11 @@ export function createLloomServer( return; } + if (req.method === 'POST' && url.pathname === '/v1/tokenize') { + await handleOpenAITokenize(req, res); + return; + } + if (req.method === 'GET' && (url.pathname === '/v1/audio/voices' || url.pathname === '/gateway/audio/voices')) { await handleSpeechVoices(req, res, url); return; diff --git a/test/model-acquisition.test.mjs b/test/model-acquisition.test.mjs index b2b5e85..b9fb84e 100644 --- a/test/model-acquisition.test.mjs +++ b/test/model-acquisition.test.mjs @@ -297,6 +297,248 @@ const unionSubset = await applyRecipe( ); assert.equal(unionSubset.results[0].status, 'skipped', JSON.stringify(unionSubset.results)); +// Re-entry and recovery safety. Existing checkpoints are moved aside only +// after every known downloader cache directory is writable. A failed update +// must restore the old checkpoint, while a concurrent publisher must retain +// both copies for manual reconciliation. +const recoveryRoot = path.join(root, 'recovery-models'); +const recoveryHf = path.join(root, 'recovery-hf'); +await fs.writeFile( + recoveryHf, + `#!/bin/sh +for last do :; done +case "${'${RECOVERY_MODE:-}'}" in + fail) + mkdir -p "$last" + printf 'staged partial\\n' > "$last/partial.gguf" + exit 9 + ;; + verify) + mkdir -p "$last" + printf 'wrong download\\n' > "$last/wrong.gguf" + exit 0 + ;; + mutate) + mkdir -p "$last" + printf 'mutated payload\\n' > "$last/model.gguf" + exit 9 + ;; + concurrent) + mkdir -p "$last" "$CONCURRENT_DEST" + printf 'staged concurrent\\n' > "$last/staged.gguf" + printf 'published concurrently\\n' > "$CONCURRENT_DEST/concurrent.gguf" + exit 9 + ;; +esac +mkdir -p "$last" +printf 'unexpected downloader invocation\\n' > "$last/unexpected.gguf" +`, + { mode: 0o755 } +); +await fs.chmod(recoveryHf, 0o755); + +const recoveryRevision = 'fedcba9876543210fedcba9876543210fedcba98'; +const recoveryStep = (id, model, files) => ({ + id, + action: 'download-model', + provider: 'huggingface', + model, + revision: recoveryRevision, + integrity: { files } +}); +const recoveryRecipe = (id, stepEntry, model) => ({ + schemaVersion: 1, + id, + name: id, + backend: { id: 'test-backend' }, + models: [{ role: 'default', model }], + setup: { steps: [stepEntry] } +}); +const recoveryOptions = (id, extraEnv = {}) => ({ + dryRun: false, + yes: true, + modelRoot: recoveryRoot, + statePath: path.join(root, `${id}-state.json`), + env: { + ...process.env, + PATH: '/usr/bin:/bin', + LLOOM_HF_BIN: recoveryHf, + HF_HUB_CLI: '', + ...extraEnv + } +}); +const oldPayload = 'old checkpoint\n'; +const oldPayloadSize = Buffer.byteLength(oldPayload); + +// A nested Hugging Face download cache that is not a directory is rejected +// before rename. Fixed names resembling the old probe are user data and must +// survive the failed preflight unchanged. +const preflightModel = 'owner/preflight-model'; +const preflightDestination = path.join(recoveryRoot, 'owner--preflight-model'); +await fs.mkdir(path.join(preflightDestination, '.cache', 'huggingface'), { recursive: true }); +await fs.writeFile(path.join(preflightDestination, 'model.gguf'), oldPayload); +await fs.writeFile(path.join(preflightDestination, '.lloom-write-probe'), 'keep this file\n'); +await fs.writeFile(path.join(preflightDestination, '.lloom-write-probe-tmp'), 'keep this temp\n'); +await fs.writeFile(path.join(preflightDestination, '.cache', 'huggingface', 'download'), 'not a directory\n'); +const preflightResult = await applyRecipe( + recoveryRecipe( + 'recovery-preflight', + recoveryStep('download', preflightModel, [{ path: 'model.gguf', sizeBytes: oldPayloadSize }]), + preflightModel + ), + { models: [], runtimes: {} }, + recoveryOptions('recovery-preflight') +); +assert.equal(preflightResult.results[0].status, 'failed', JSON.stringify(preflightResult.results)); +assert.equal(await fs.readFile(path.join(preflightDestination, 'model.gguf'), 'utf8'), oldPayload); +assert.equal(await fs.readFile(path.join(preflightDestination, '.lloom-write-probe'), 'utf8'), 'keep this file\n'); +assert.equal(await fs.readFile(path.join(preflightDestination, '.lloom-write-probe-tmp'), 'utf8'), 'keep this temp\n'); +assert.equal( + await fs.stat(`${preflightDestination}.incomplete`).then( + () => true, + () => false + ), + false +); +assert.match(preflightResult.results[0].stderr, /cannot write .*download/); + +// A downloader that exits non-zero after writing a partial payload restores an +// existing checkpoint. The new partial file is retained inside that restored +// directory for inspection/resume; the old model data never disappears. +const failedModel = 'owner/failed-model'; +const failedDestination = path.join(recoveryRoot, 'owner--failed-model'); +await fs.mkdir(failedDestination, { recursive: true }); +await fs.writeFile(path.join(failedDestination, 'model.gguf'), oldPayload); +const failedResult = await applyRecipe( + recoveryRecipe( + 'recovery-downloader-failure', + recoveryStep('download', failedModel, [{ path: 'model.gguf', sizeBytes: oldPayloadSize }]), + failedModel + ), + { models: [], runtimes: {} }, + recoveryOptions('recovery-downloader-failure', { RECOVERY_MODE: 'fail' }) +); +assert.equal(failedResult.results[0].status, 'failed', JSON.stringify(failedResult.results)); +assert.equal(await fs.readFile(path.join(failedDestination, 'model.gguf'), 'utf8'), oldPayload); +assert.equal(await fs.readFile(path.join(failedDestination, 'partial.gguf'), 'utf8'), 'staged partial\n'); +assert.equal( + await fs.stat(`${failedDestination}.incomplete`).then( + () => true, + () => false + ), + false +); + +// Verification failures also restore the old directory and never publish a +// completion manifest for a payload that did not pass its declared digest. +const verifyModel = 'owner/verify-model'; +const verifyDestination = path.join(recoveryRoot, 'owner--verify-model'); +await fs.mkdir(verifyDestination, { recursive: true }); +await fs.writeFile(path.join(verifyDestination, 'model.gguf'), oldPayload); +const verifyExpected = 'new verified checkpoint\n'; +const verifyResult = await applyRecipe( + recoveryRecipe( + 'recovery-verification-failure', + recoveryStep('download', verifyModel, [ + { + path: 'model.gguf', + sizeBytes: Buffer.byteLength(verifyExpected), + sha256: crypto.createHash('sha256').update(verifyExpected).digest('hex') + } + ]), + verifyModel + ), + { models: [], runtimes: {} }, + recoveryOptions('recovery-verification-failure', { RECOVERY_MODE: 'verify' }) +); +assert.equal(verifyResult.results[0].status, 'failed', JSON.stringify(verifyResult.results)); +assert.equal(await fs.readFile(path.join(verifyDestination, 'model.gguf'), 'utf8'), oldPayload); +assert.equal( + await fs.stat(path.join(verifyDestination, MODEL_ACQUISITION_MANIFEST)).then( + () => true, + () => false + ), + false +); +assert.equal( + await fs.stat(`${verifyDestination}.incomplete`).then( + () => true, + () => false + ), + false +); +assert.match(verifyResult.results[0].stderr, /download verification failed/); + +// A failed new-revision update may have modified the old payload in place. +// Recovery must leave those bytes staged back at the canonical path without +// reviving the old completion marker; the exact old marker bytes stay in the +// unique acquisition-owned backup for manual reconciliation. +const staleModel = 'owner/stale-model'; +const staleDestination = path.join(recoveryRoot, 'owner--stale-model'); +const oldRevision = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'; +const oldManifestRaw = `${JSON.stringify({ + version: 1, + provider: 'huggingface', + model: staleModel, + revision: oldRevision +})}\n`; +await fs.mkdir(staleDestination, { recursive: true }); +await fs.writeFile(path.join(staleDestination, 'model.gguf'), oldPayload); +await fs.writeFile(path.join(staleDestination, MODEL_ACQUISITION_MANIFEST), oldManifestRaw); +const staleResult = await applyRecipe( + recoveryRecipe( + 'recovery-stale-provenance', + recoveryStep('download', staleModel, [{ path: 'model.gguf', sizeBytes: oldPayloadSize }]), + staleModel + ), + { models: [], runtimes: {} }, + recoveryOptions('recovery-stale-provenance', { RECOVERY_MODE: 'mutate' }) +); +assert.equal(staleResult.results[0].status, 'failed', JSON.stringify(staleResult.results)); +assert.equal(await fs.readFile(path.join(staleDestination, 'model.gguf'), 'utf8'), 'mutated payload\n'); +assert.equal( + await fs.stat(path.join(staleDestination, MODEL_ACQUISITION_MANIFEST)).then( + () => true, + () => false + ), + false +); +assert.equal((await modelAcquisitionStatus({ destination: staleDestination, revision: oldRevision })).complete, false); +const staleBackups = (await fs.readdir(recoveryRoot, { withFileTypes: true })) + .filter((entry) => entry.isDirectory() && entry.name.startsWith('.lloom-acquisition-previous-owner--stale-model-')) + .map((entry) => path.join(recoveryRoot, entry.name)); +assert.equal(staleBackups.length, 1); +assert.equal(await fs.readFile(path.join(staleBackups[0], MODEL_ACQUISITION_MANIFEST), 'utf8'), oldManifestRaw); + +// A destination published while the old checkpoint is staged wins admission, +// but the staged copy is preserved instead of being recursively deleted. +const concurrentModel = 'owner/concurrent-model'; +const concurrentDestination = path.join(recoveryRoot, 'owner--concurrent-model'); +await fs.mkdir(concurrentDestination, { recursive: true }); +await fs.writeFile(path.join(concurrentDestination, 'model.gguf'), oldPayload); +const concurrentResult = await applyRecipe( + recoveryRecipe( + 'recovery-concurrent-destination', + recoveryStep('download', concurrentModel, [{ path: 'model.gguf', sizeBytes: oldPayloadSize }]), + concurrentModel + ), + { models: [], runtimes: {} }, + recoveryOptions('recovery-concurrent-destination', { + RECOVERY_MODE: 'concurrent', + CONCURRENT_DEST: concurrentDestination + }) +); +assert.equal(concurrentResult.results[0].status, 'failed', JSON.stringify(concurrentResult.results)); +assert.equal( + await fs.readFile(path.join(concurrentDestination, 'concurrent.gguf'), 'utf8'), + 'published concurrently\n' +); +assert.equal(await fs.readFile(path.join(`${concurrentDestination}.incomplete`, 'model.gguf'), 'utf8'), oldPayload); +assert.equal( + await fs.readFile(path.join(`${concurrentDestination}.incomplete`, 'staged.gguf'), 'utf8'), + 'staged concurrent\n' +); + assert.equal((await modelAcquisitionStatus({ destination, include: ['missing.gguf'] })).complete, false); assert.ok(validateAcquisitionStep({ include: 'file' }).length); assert.ok(validateAcquisitionStep({ include: [123] }).length); diff --git a/test/protocol.test.mjs b/test/protocol.test.mjs index 71dd2fc..b46d6dc 100644 --- a/test/protocol.test.mjs +++ b/test/protocol.test.mjs @@ -4,6 +4,7 @@ import { openAIToAnthropic, openAIToResponses, normalizeStructuredOutputChatCompletion, + normalizeOpenAIChatCompletionChunk, prepareStructuredOutputForBackend, responsesToOpenAIChat, responseStatusFromFinishReason, @@ -161,6 +162,59 @@ const resolved = { model: { upstreamModel: 'upstream-qwen' } }; ), (error) => error instanceof StructuredOutputError && error.code === 'structured_output_streaming' ); + + const nativeResponseFormat = { + type: 'json_schema', + json_schema: { + name: 'answer', + strict: true, + schema + } + }; + const nativeChatRequest = { + model: 'gateway-model', + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Describe this.' }, + { type: 'image_url', image_url: { url: 'data:image/png;base64,AA', detail: 'high' } }, + { type: 'video_url', video_url: { url: 'data:video/mp4;base64,AA' } } + ] + } + ], + tools: [{ type: 'function', function: { name: 'lookup', parameters: schema } }], + response_format: nativeResponseFormat, + stream: true + }; + const nativePrepared = prepareStructuredOutputForBackend(nativeChatRequest, { + model: { capabilities: ['tools'] }, + backend: { type: 'openai' } + }).body; + assert.deepEqual(nativePrepared.messages, nativeChatRequest.messages); + assert.deepEqual(nativePrepared.tools, nativeChatRequest.tools); + assert.deepEqual(nativePrepared.response_format, nativeResponseFormat); + + const streamedToolChunk = { + id: 'chatcmpl_stream', + choices: [ + { + index: 0, + delta: { + tool_calls: [ + { + index: 0, + id: 'call_lookup', + type: 'function', + function: { name: 'lookup', arguments: '{"q":"atlas"}' } + } + ] + }, + finish_reason: 'tool_calls' + } + ] + }; + assert.deepEqual(normalizeOpenAIChatCompletionChunk(streamedToolChunk), streamedToolChunk); } // Responses → chat @@ -331,4 +385,67 @@ const resolved = { model: { upstreamModel: 'upstream-qwen' } }; assert.equal(raw.rewritten, false); } +// A caller-supplied JSON Schema that no implementation could compile must be +// refused before it reaches a backend. Accepting it silently turns a +// schema-constrained request into an unconstrained one and still answers 200. +{ + const invalid = { + model: 'm', + messages: [{ role: 'user', content: 'hi' }], + response_format: { type: 'json_schema', json_schema: { name: 'x', strict: true, schema: { type: 'invalid' } } } + }; + assert.throws( + () => prepareStructuredOutputForBackend(invalid, resolved), + (error) => + error instanceof StructuredOutputError && error.statusCode === 400 && error.code === 'invalid_json_schema' + ); + + // Nested subschemas are checked too, and the failing path is named. + const nested = { + model: 'm', + messages: [], + response_format: { + type: 'json_schema', + json_schema: { name: 'x', schema: { type: 'object', properties: { a: { type: 'nope' } } } } + } + }; + assert.throws( + () => prepareStructuredOutputForBackend(nested, resolved), + (error) => error.code === 'invalid_json_schema' && /properties\.a/.test(error.message) + ); + + // A schema that is merely unusual is left alone: $ref, enum, const, unknown + // keywords and a type union are all legitimate. + const legal = { + model: 'm', + messages: [], + response_format: { + type: 'json_schema', + json_schema: { + name: 'x', + schema: { + type: ['object', 'null'], + properties: { a: { $ref: '#/$defs/a' }, b: { enum: ['x'] }, c: { const: 1 } }, + $defs: { a: { type: 'string', format: 'date-time' } }, + 'x-vendor': { anything: true } + } + } + } + }; + const passed = prepareStructuredOutputForBackend(legal, resolved); + assert.equal(passed.output, null); + assert.equal(passed.body.response_format.json_schema.schema.$defs.a.type, 'string'); + + // Other response_format shapes are not this validator's business. + for (const value of [undefined, { type: 'text' }, { type: 'json_object' }]) { + const body = { model: 'm', messages: [] }; + if (value !== undefined) body.response_format = value; + assert.doesNotThrow(() => prepareStructuredOutputForBackend(body, resolved)); + } + assert.throws( + () => prepareStructuredOutputForBackend({ model: 'm', messages: [], response_format: 'json' }, resolved), + (error) => error.code === 'invalid_response_format' + ); +} + console.log('protocol tests passed'); diff --git a/test/responses-codex.test.mjs b/test/responses-codex.test.mjs index f03b5f3..d3f8f54 100644 --- a/test/responses-codex.test.mjs +++ b/test/responses-codex.test.mjs @@ -2,7 +2,9 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { openAIToResponses, + responsesContentPartToOpenAI, responsesInputToMessages, + responsesToOpenAIChat, responsesToolChoiceToOpenAI, responsesToolsToOpenAI } from '../src/protocol/responses.mjs'; @@ -49,6 +51,56 @@ test('developer instructions remain privileged and ordered ahead of user input', ]); }); +test('Responses image and video parts preserve compatible URL forms', () => { + const gif = 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP'; + const mp4 = 'data:video/mp4;base64,AAAAIGZ0eXBpc29tAA'; + const messages = responsesInputToMessages({ + input: [ + { + role: 'user', + content: [ + { type: 'input_image', image_url: gif, detail: 'high' }, + { type: 'input_image', image_url: { url: 'https://example.test/image.png', detail: 'low' } }, + { type: 'image_url', image_url: { url: 'https://example.test/other.png', detail: 'high' } }, + { type: 'input_video', video_url: gif }, + { type: 'video_url', video_url: { url: mp4, detail: 'low' } }, + { type: 'video', url: 'https://example.test/movie.mp4' } + ] + } + ] + }); + assert.deepEqual(messages[0].content, [ + { type: 'image_url', image_url: { url: gif, detail: 'high' } }, + { type: 'image_url', image_url: { url: 'https://example.test/image.png', detail: 'low' } }, + { type: 'image_url', image_url: { url: 'https://example.test/other.png', detail: 'high' } }, + { type: 'video_url', video_url: { url: gif } }, + { type: 'video_url', video_url: { url: mp4, detail: 'low' } }, + { type: 'video_url', video_url: { url: 'https://example.test/movie.mp4' } } + ]); + assert.deepEqual(responsesContentPartToOpenAI({ type: 'input_image', url: gif }), { + type: 'image_url', + image_url: { url: gif } + }); +}); + +test('Responses video content reaches the chat adapter alongside tools', () => { + const mp4 = 'data:video/mp4;base64,AAAAIGZ0eXBpc29tAA'; + const translated = responsesToOpenAIChat( + { + input: [{ role: 'user', content: [{ type: 'input_video', video_url: mp4 }] }], + tools: [{ type: 'function', name: 'inspect', parameters: { type: 'object' } }], + tool_choice: { type: 'function', name: 'inspect' }, + stream: true + }, + { model: { upstreamModel: 'glm-5.3-flash-atlas' } } + ); + assert.deepEqual(translated.messages[0].content, [{ type: 'video_url', video_url: { url: mp4 } }]); + assert.deepEqual(translated.tools, [ + { type: 'function', function: { name: 'inspect', parameters: { type: 'object' } } } + ]); + assert.deepEqual(translated.tool_choice, { type: 'function', function: { name: 'inspect' } }); +}); + test('custom tool declarations and explicit choices become callable Chat functions', () => { const converted = responsesToolsToOpenAI(tools); assert.equal(converted.length, 2); diff --git a/test/runtime-policy-config.test.mjs b/test/runtime-policy-config.test.mjs new file mode 100644 index 0000000..7452b5e --- /dev/null +++ b/test/runtime-policy-config.test.mjs @@ -0,0 +1,198 @@ +// Focused tests for explicit numeric memory-policy CLI configuration. +// +// These spawn `node bin/lloom.mjs runtime-policy` against a CPU-only temp +// config file. No real gateway, runtimes, or network calls are involved. +import { test, after } from 'node:test'; +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { promises as fs } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..'); +const cli = path.join(repoRoot, 'bin', 'lloom.mjs'); + +const directories = []; +after(async () => { + await Promise.all(directories.map((dir) => fs.rm(dir, { recursive: true, force: true }))); +}); + +// Minimal config satisfying the real loadConfig validation. +function baseConfig(extra = {}) { + return { + runtimePolicy: { + memorySafety: { mode: 'enforce', pollIntervalMs: 250 } + }, + models: [{ id: 'm1', runtime: 'rt1', backend: 'test' }], + runtimes: { rt1: { command: 'echo' } }, + backends: { test: { type: 'openai', baseUrl: 'http://127.0.0.1:12345/v1' } }, + aliases: {}, + ...extra + }; +} + +async function writeConfig(config) { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), 'mem-policy-')); + directories.push(dir); + const p = path.join(dir, 'config.json'); + await fs.writeFile(p, JSON.stringify(config, null, 2) + '\n', { mode: 0o600 }); + return p; +} + +function runCli(args) { + return new Promise((resolve) => { + execFile( + process.execPath, + [cli, ...args], + { + cwd: repoRoot, + env: { ...process.env, HOME: os.tmpdir(), LLOOM_CONFIG: '' } + }, + (error, stdout, stderr) => { + resolve({ code: error?.code ?? 0, stdout, stderr }); + } + ); + }); +} + +test('dry-run reports selected fields only and leaves config bytes unchanged', async () => { + const config = baseConfig({ + cluster: { nodes: { nodeA: { resources: { reserveMemoryGb: 8 } } } } + }); + const p = await writeConfig(config); + const before = await fs.readFile(p, 'utf8'); + + const result = await runCli([ + 'runtime-policy', + '--config', + p, + '--max-memory-utilization', + '0.85', + '--reserve-memory-gb', + '12' + ]); + assert.equal(result.code, 0, result.stderr); + const report = JSON.parse(result.stdout); + assert.equal(report.mode, 'dry-run'); + assert.equal(report.before.runtimePolicy.maxMemoryUtilization, null); + assert.equal(report.after.runtimePolicy.maxMemoryUtilization, 0.85); + assert.equal(report.after.memorySafety.minAvailableMemoryGb, 12); + assert.equal(report.after.runtimePolicy.reserveMemoryGb, 12); + // Only selected fields + node overrides + warnings; no whole config. + assert.deepEqual(Object.keys(report).sort(), [ + 'after', + 'before', + 'changed', + 'mode', + 'nodeOverrides', + 'ok', + 'warnings' + ]); + assert.equal(result.stdout.includes('http://127.0.0.1:12345'), false, 'config body must not be echoed'); + assert.equal(await fs.readFile(p, 'utf8'), before, 'dry-run must not write config bytes'); +}); + +test('apply without --yes writes nothing', async () => { + const p = await writeConfig(baseConfig()); + const before = await fs.readFile(p, 'utf8'); + const result = await runCli(['runtime-policy', '--config', p, '--max-memory-utilization', '0.7', '--apply']); + assert.notEqual(result.code, 0); + assert.match(result.stderr, /--yes/); + assert.equal(await fs.readFile(p, 'utf8'), before); +}); + +test('apply updates guard and admission fields, preserves unrelated config and node overrides', async () => { + const config = baseConfig({ + gatewayNote: 'keep me', + cluster: { nodes: { nodeA: { resources: { reserveMemoryGb: 8, memoryBudgetGb: 200 } } } }, + runtimePolicy: { + memorySafety: { mode: 'enforce', pollIntervalMs: 250 }, + maxMemoryUtilization: 0.5, + memoryBudgetGb: 0 + } + }); + delete config.runtimePolicy.memoryBudgetGb; + const p = await writeConfig(config); + + const result = await runCli([ + 'runtime-policy', + '--config', + p, + '--apply', + '--yes', + '--max-memory-utilization', + '0.9', + '--reserve-memory-gb', + '16' + ]); + assert.equal(result.code, 0, result.stderr); + const applied = JSON.parse(result.stdout); + assert.equal(applied.applied, true); + + const written = JSON.parse(await fs.readFile(p, 'utf8')); + assert.equal(written.runtimePolicy.maxMemoryUtilization, 0.9); + assert.equal(written.runtimePolicy.memorySafety.maxMemoryUtilization, 0.9); + assert.equal(written.runtimePolicy.reserveMemoryGb, 16); + assert.equal(written.runtimePolicy.memorySafety.minAvailableMemoryGb, 16); + // Mode and polling preserved; no yolo introduced. + assert.equal(written.runtimePolicy.memorySafety.mode, 'enforce'); + assert.equal(written.runtimePolicy.memorySafety.pollIntervalMs, 250); + assert.equal(written.gatewayNote, 'keep me'); + assert.deepEqual(written.cluster.nodes.nodeA.resources, { reserveMemoryGb: 8, memoryBudgetGb: 200 }); +}); + +test('invalid numeric flags are rejected and write nothing', async () => { + const p = await writeConfig(baseConfig()); + const before = await fs.readFile(p, 'utf8'); + const cases = [ + ['--max-memory-utilization'], + ['--max-memory-utilization', 'NaN'], + ['--max-memory-utilization', 'Infinity'], + ['--max-memory-utilization', '0'], + ['--max-memory-utilization', '1.5'], + ['--reserve-memory-gb', '0'], + ['--reserve-memory-gb', 'abc'], + ['--max-memory-utilization=0.95'], + ['--max-memory-utilization', '0.8', '--max-memory-utilization', '0.95'], + ['--reserve-memory-gb'] + ]; + for (const flags of cases) { + const result = await runCli(['runtime-policy', '--config', p, '--apply', '--yes', ...flags]); + assert.notEqual(result.code, 0, `expected failure for ${flags.join(' ')}`); + assert.equal(await fs.readFile(p, 'utf8'), before, `config changed for ${flags.join(' ')}`); + } +}); + +test('top-level absolute memoryBudgetGb fails actionably instead of changing an unused percentage', async () => { + const p = await writeConfig( + baseConfig({ runtimePolicy: { memoryBudgetGb: 220, memorySafety: { mode: 'enforce', pollIntervalMs: 250 } } }) + ); + const before = await fs.readFile(p, 'utf8'); + for (const mode of [[], ['--apply', '--yes']]) { + const result = await runCli(['runtime-policy', '--config', p, '--max-memory-utilization', '0.8', ...mode]); + assert.notEqual(result.code, 0); + assert.match(result.stderr, /memoryBudgetGb/); + assert.equal(await fs.readFile(p, 'utf8'), before); + } +}); + +test('runtime-plan never mutates and rejects numeric/apply flags', async () => { + const p = await writeConfig(baseConfig()); + const before = await fs.readFile(p, 'utf8'); + + const withNumeric = await runCli(['runtime-plan', '--config', p, '--max-memory-utilization', '0.8']); + assert.match(withNumeric.stderr, /read-only/); + const withApply = await runCli(['runtime-plan', '--config', p, '--apply', '--yes']); + assert.match(withApply.stderr, /read-only/); + assert.equal(await fs.readFile(p, 'utf8'), before, 'runtime-plan must never write config'); +}); + +test('--yes without --apply is rejected without writing', async () => { + const p = await writeConfig(baseConfig()); + const before = await fs.readFile(p, 'utf8'); + const result = await runCli(['runtime-policy', '--config', p, '--max-memory-utilization', '0.8', '--yes']); + assert.notEqual(result.code, 0); + assert.match(result.stderr, /--yes requires --apply/); + assert.equal(await fs.readFile(p, 'utf8'), before); +}); diff --git a/test/server-resilience.test.mjs b/test/server-resilience.test.mjs index 664c301..3fd7c87 100644 --- a/test/server-resilience.test.mjs +++ b/test/server-resilience.test.mjs @@ -203,6 +203,46 @@ function close(server) { ] }); assert.equal(anthropicEstimate, 4096); + + const encodedGif = 'A'.repeat(2_000_000); + const videoEstimates = [ + estimateRequestPromptTokens({ + messages: [ + { + role: 'user', + content: [{ type: 'video_url', video_url: { url: `data:image/gif;base64,${encodedGif}` } }] + } + ] + }), + estimateRequestPromptTokens({ + input: [ + { + role: 'user', + content: [{ type: 'input_video', video_url: `data:video/mp4;base64,${encodedGif}` }] + } + ] + }) + ]; + assert.deepEqual(videoEstimates, [4096, 4096]); + assert.equal( + estimateRequestPromptTokens({ + messages: [ + { role: 'user', content: [{ type: 'video_url', video_url: { url: 'https://example.test/movie.mp4' } }] } + ] + }), + 4096 + ); + assert.equal( + estimateRequestPromptTokens({ + messages: [ + { + role: 'user', + content: [{ type: 'video_url', video_url: { url: 'https://example.test/movie.mp4', detail: 'low' } }] + } + ] + }), + 1024 + ); } // Upstream dies after opening an SSE stream: gateway must not crash, must end response. @@ -756,8 +796,8 @@ function close(server) { kind: 'chat', supportsTools: true, capabilities: ['chat', 'tools'], - contextWindow: 8192, - maxPromptTokens: 1000 + contextWindow: 16384, + maxPromptTokens: 16384 } ], runtimes: {} @@ -795,8 +835,34 @@ function close(server) { assert.equal(responses.status, 200); assert.equal((await responses.json()).output_text, '{"answer":"ok"}'); - assert.equal(requests.length, 2); - for (const request of requests) { + const multimodal = await fetch(`http://127.0.0.1:${port}/v1/chat/completions`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + model: 'test-model', + messages: [ + { + role: 'user', + content: [ + { type: 'text', text: 'Describe these.' }, + { type: 'video_url', video_url: { url: 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP' } }, + { type: 'video_url', video_url: 'data:video/mp4;base64,AAAAIGZ0eXBpc29tAA' }, + { type: 'image_url', image_url: { url: 'https://example.test/frame.png', detail: 'low' } } + ] + } + ], + response_format: { + type: 'json_schema', + json_schema: { name: 'caption', strict: true, schema } + }, + tools: [{ type: 'function', function: { name: 'save_caption', parameters: schema } }] + }) + }); + assert.equal(multimodal.status, 200); + await multimodal.json(); + + assert.equal(requests.length, 3); + for (const request of requests.slice(0, 2)) { assert.equal(request.lloom, undefined); assert.equal(request.response_format, undefined); assert.equal(request.tools[0].function.name, 'answer'); @@ -808,6 +874,54 @@ function close(server) { assert.deepEqual(request.provider, { require_parameters: true }); assert.deepEqual(request.thinking, { type: 'disabled' }); } + assert.deepEqual(requests[2].messages, [ + { + role: 'user', + content: [ + { type: 'text', text: 'Describe these.' }, + { type: 'video_url', video_url: { url: 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP' } }, + { type: 'video_url', video_url: 'data:video/mp4;base64,AAAAIGZ0eXBpc29tAA' }, + { type: 'image_url', image_url: { url: 'https://example.test/frame.png', detail: 'low' } } + ] + } + ]); + assert.deepEqual(requests[2].response_format, { + type: 'json_schema', + json_schema: { name: 'caption', strict: true, schema } + }); + assert.deepEqual(requests[2].tools, [{ type: 'function', function: { name: 'save_caption', parameters: schema } }]); + + const responsesMedia = await fetch(`http://127.0.0.1:${port}/v1/responses`, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify({ + model: 'test-model', + input: [ + { + role: 'user', + content: [ + { type: 'input_image', image_url: { url: 'https://example.test/response-image.png', detail: 'high' } }, + { type: 'input_video', video_url: 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP' }, + { type: 'input_video', video_url: { url: 'data:video/mp4;base64,AAAAIGZ0eXBpc29tAA' } } + ] + } + ] + }) + }); + assert.equal(responsesMedia.status, 200); + await responsesMedia.json(); + + assert.equal(requests.length, 4); + assert.deepEqual(requests[3].messages, [ + { + role: 'user', + content: [ + { type: 'image_url', image_url: { url: 'https://example.test/response-image.png', detail: 'high' } }, + { type: 'video_url', video_url: { url: 'data:image/gif;base64,R0lGODlhAQABAIAAAAAAAP' } }, + { type: 'video_url', video_url: { url: 'data:video/mp4;base64,AAAAIGZ0eXBpc29tAA' } } + ] + } + ]); const invalid = await fetch(`http://127.0.0.1:${port}/v1/chat/completions`, { method: 'POST', @@ -820,7 +934,7 @@ function close(server) { }); assert.equal(invalid.status, 400); assert.equal((await invalid.json()).error.code, 'invalid_structured_output'); - assert.equal(requests.length, 2); + assert.equal(requests.length, 4); for (const tool_choice of ['required', { type: 'function', function: { name: 'answer' } }]) { const streamed = await fetch(`http://127.0.0.1:${port}/v1/chat/completions`, { diff --git a/test/tokenize-route.test.mjs b/test/tokenize-route.test.mjs new file mode 100644 index 0000000..6472995 --- /dev/null +++ b/test/tokenize-route.test.mjs @@ -0,0 +1,192 @@ +import assert from 'node:assert/strict'; +import http from 'node:http'; +import { createLloomServer } from '../src/server.mjs'; + +// `/v1/tokenize` exists so an exact-count client can ask the backend that will +// actually serve a request how many tokens that request renders to. The gateway +// must therefore forward the body untouched (bar the upstream model name) and +// return the backend's own answer rather than re-deriving a count. + +function listen(server) { + return new Promise((resolve) => server.listen(0, '127.0.0.1', () => resolve(server.address().port))); +} + +function close(server) { + return new Promise((resolve) => server.close(resolve)); +} + +function postJson(url, body) { + return fetch(url, { + method: 'POST', + headers: { 'content-type': 'application/json' }, + body: JSON.stringify(body) + }); +} + +async function harness({ status = 200, baseUrlSuffix = '/v1' } = {}) { + const state = { seen: [] }; + const upstream = http.createServer(async (req, res) => { + let raw = ''; + for await (const chunk of req) raw += chunk; + state.seen.push({ url: req.url, method: req.method, body: JSON.parse(raw || '{}') }); + if (status !== 200) { + res.writeHead(status, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ error: { message: `backend status ${status}` } })); + return; + } + res.writeHead(200, { 'content-type': 'application/json' }); + res.end(JSON.stringify({ tokens: [11, 22, 33], count: 3 })); + }); + const upstreamPort = await listen(upstream); + const config = { + name: 'tokenize-test', + server: { host: '127.0.0.1', port: 0, inferenceEnabled: true }, + security: { allowMissingAuth: true, apiKeys: [] }, + logging: { metricsPersistence: false }, + cluster: { routingStatusCacheMs: 0 }, + defaults: { chatModel: 'atlas-chat' }, + backends: { + atlas: { type: 'openai', baseUrl: `http://127.0.0.1:${upstreamPort}${baseUrlSuffix}`, timeoutMs: 5000 } + }, + models: [ + { + id: 'atlas-chat', + kind: 'chat', + backend: 'atlas', + upstreamModel: 'glm-5.3-flash-atlas', + contextWindow: 262144, + maxOutputTokens: 131072 + } + ] + }; + const app = createLloomServer(config, { logger: { error() {}, warn() {} } }); + const port = await listen(app.server); + return { + state, + base: `http://127.0.0.1:${port}`, + async close() { + await close(app.server); + await close(upstream); + } + }; +} + +{ + const context = await harness(); + try { + const messages = [{ role: 'user', content: 'hello world' }]; + const response = await postJson(`${context.base}/v1/tokenize`, { + model: 'atlas-chat', + messages, + chat_template_kwargs: { thinking: false, enable_thinking: false } + }); + assert.equal(response.status, 200); + assert.deepEqual(await response.json(), { tokens: [11, 22, 33], count: 3 }); + + assert.equal(context.state.seen.length, 1); + // The Atlas engine serves token counting at the bare `/tokenize` while its + // OpenAI surface lives under `/v1`. The backend baseUrl already ends in + // `/v1`, so the handler resolves the backend origin explicitly; routing this + // through the ordinary baseUrl join would request `/v1/tokenize` and 404. + assert.equal(context.state.seen[0].url, '/tokenize'); + assert.equal(context.state.seen[0].method, 'POST'); + // The upstream model replaces the requested id, and the template kwargs are + // preserved: changing them would change the very count being reported. + assert.equal(context.state.seen[0].body.model, 'glm-5.3-flash-atlas'); + assert.deepEqual(context.state.seen[0].body.messages, messages); + assert.deepEqual(context.state.seen[0].body.chat_template_kwargs, { + thinking: false, + enable_thinking: false + }); + } finally { + await context.close(); + } +} + +{ + // A raw `prompt` is the other accepted input and must reach the backend too. + const context = await harness(); + try { + const response = await postJson(`${context.base}/v1/tokenize`, { + model: 'atlas-chat', + prompt: 'count me' + }); + assert.equal(response.status, 200); + assert.equal(context.state.seen[0].body.prompt, 'count me'); + } finally { + await context.close(); + } +} + +{ + // A baseUrl without `/v1` resolves to the same bare route. + const context = await harness({ baseUrlSuffix: '' }); + try { + const response = await postJson(`${context.base}/v1/tokenize`, { + model: 'atlas-chat', + prompt: 'count me' + }); + assert.equal(response.status, 200); + assert.equal(context.state.seen[0].url, '/tokenize'); + } finally { + await context.close(); + } +} + +{ + // A baseUrl with a deeper path still resolves to the backend's origin route + // rather than inheriting that prefix. + const context = await harness({ baseUrlSuffix: '/v1/openai' }); + try { + const response = await postJson(`${context.base}/v1/tokenize`, { + model: 'atlas-chat', + prompt: 'count me' + }); + assert.equal(response.status, 200); + assert.equal(context.state.seen[0].url, '/tokenize'); + } finally { + await context.close(); + } +} + +{ + // Neither input is a client error, and must not reach the backend. + const context = await harness(); + try { + const response = await postJson(`${context.base}/v1/tokenize`, { model: 'atlas-chat' }); + assert.equal(response.status, 400); + const body = await response.json(); + assert.equal(body.error.code, 'missing_input'); + assert.equal(context.state.seen.length, 0); + } finally { + await context.close(); + } +} + +{ + // An unresolvable model is a client error, not a silent pass-through. + const context = await harness(); + try { + const response = await postJson(`${context.base}/v1/tokenize`, { + model: 'no-such-model', + prompt: 'x' + }); + assert.ok(response.status >= 400, `expected a client error, got ${response.status}`); + assert.equal(context.state.seen.length, 0); + } finally { + await context.close(); + } +} + +{ + // A backend rejection reaches the client with the backend's own status. + const context = await harness({ status: 400 }); + try { + const response = await postJson(`${context.base}/v1/tokenize`, { model: 'atlas-chat', prompt: 'x' }); + assert.equal(response.status, 400); + } finally { + await context.close(); + } +} + +console.log('tokenize route tests passed');