From 92cdefa6104767e3b891abd173b14071afef9ba2 Mon Sep 17 00:00:00 2001 From: ttbombadil Date: Fri, 9 Oct 2026 12:13:16 +0200 Subject: [PATCH] fix(sandbox): end runs that stay paused beyond a pause budget A paused run keeps its runner, admission and frozen container, and the pause clock had no limit. `docker pause` also freezes the sandbox's `timeout --signal=KILL`, so the documented hard lifetime "including paused time" never fired for a paused container; a forgotten paused tab held its capacity for as long as its WebSocket answered pings. - SIMULATION_MAX_PAUSED_SECONDS (default 600, below the hard lifetime) bounds the total paused time of a run. - When the budget runs out while paused, the runner ends the run like the runtime timeout: notice, termination, then onExit releases runner, admission and container and the client receives the stopped status. - Pause/Resume SSOT, SECURITY, INSTALL_SERVER and ARCHITECTURE describe the actual semantics; Compose passes the value through. Co-Authored-By: Claude Opus 5.5 --- docker-compose.yml | 2 + docs/ARCHITECTURE.md | 2 +- docs/INSTALL_SERVER.md | 11 ++- docs/SECURITY.md | 9 +- server/config.ts | 26 +++++- server/services/sandbox-runner.ts | 39 +++++++- ssot/ssot_function_definition_PauseResume.md | 13 ++- tests/integration/paused-run-budget.test.ts | 74 +++++++++++++++ .../routes/simulation-paused-run-end.test.ts | 87 ++++++++++++++++++ .../sandbox-runner-pause-budget.test.ts | 91 +++++++++++++++++++ vitest.config.ts | 2 + 11 files changed, 345 insertions(+), 11 deletions(-) create mode 100644 tests/integration/paused-run-budget.test.ts create mode 100644 tests/server/routes/simulation-paused-run-end.test.ts create mode 100644 tests/server/services/sandbox-runner-pause-budget.test.ts diff --git a/docker-compose.yml b/docker-compose.yml index 8398df654..bb675881d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,6 +59,8 @@ services: # measured on its reference VM. See docs/CAPACITY_VALIDATION_PLAN.md before raising. - SIMULATION_ADMISSION_MAX=${SIMULATION_ADMISSION_MAX:-25} - SIMULATION_MAX_CONCURRENT=200 + # Total paused time of one run; a paused run still holds its capacity. + - SIMULATION_MAX_PAUSED_SECONDS=${SIMULATION_MAX_PAUSED_SECONDS:-600} - SANDBOX_POOL_IDLE_TIMEOUT_MS=300000 # Per-sandbox Docker resource limits # ┌─ Scenario breakdown ────────────────────────────────────────────────────┐ diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 6ca20859b..4d8d570ec 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -94,7 +94,7 @@ graph TD - **Wiederverwendung:** Jeder Lauf trägt eine Generation und ein Abbruchsignal. Ein gestoppter Lauf, der noch auf einen Start-Slot wartet, startet nicht mehr und berührt den Folgelauf auf demselben Runner nicht; Ereignisse eines ersetzten Kindprozesses werden verworfen. Den Reset vor der Wiederverwendung besitzt der Runner (`resetForReuse`). - **Verwaiste Container:** Jeder Sandbox-Container trägt das Label `unosim.owner` – `instance.` (Compose: `unosim-server`) oder ohne ID `:`. Beim Start und am Ende eines geordneten Shutdowns entfernt das Backend die Container dieses Owners, auch nach Absturz oder Redeploy; Container anderer Owner bleiben unberührt. - **Shutdown:** Runner werden parallel (höchstens 8) und je Runner höchstens 6 s lang gestoppt, danach entfernt ein Label-Sweep Reste; Compose gibt dem Backend 15 s vor SIGKILL. - - **Harte Lebensdauer:** `timeout --signal=KILL` beendet jede Sandbox nach `SANDBOX_MAX_LIFETIME_SECONDS` (Standard 7200 s Wanduhrzeit inklusive Pausen), unabhängig vom Backend. + - **Harte Lebensdauer:** `timeout --signal=KILL` beendet jede laufende Sandbox nach `SANDBOX_MAX_LIFETIME_SECONDS` (Standard 7200 s Wanduhrzeit), unabhängig vom Backend. `docker pause` friert diesen Timer ein; pausierte Läufe beendet deshalb das Backend nach insgesamt `SIMULATION_MAX_PAUSED_SECONDS` (Standard 600 s) Pausenzeit und gibt Runner, Admission und Container frei. - **Isolation:** Jeder Sketch läuft in eigenem Container - **Ressourcenkontrolle:** CPU/Memory/PID-Limits pro Container diff --git a/docs/INSTALL_SERVER.md b/docs/INSTALL_SERVER.md index 2b9240870..c43dd409f 100644 --- a/docs/INSTALL_SERVER.md +++ b/docs/INSTALL_SERVER.md @@ -103,9 +103,13 @@ die aktuelle Entscheidung zur optionalen Benutzeranmeldung steht in geordneten Shutdowns entfernt der Server die Container dieses Owners, auch nach Absturz oder Redeploy mit neuem Backend-Container. Gleichzeitig laufende Backends auf demselben Docker-Host brauchen verschiedene IDs. -- Jede Sandbox endet spätestens nach `SANDBOX_MAX_LIFETIME_SECONDS` - (Standard 7200 s, Wanduhrzeit inklusive Pausen), auch wenn das Backend nicht - mehr läuft. +- Jede laufende Sandbox endet spätestens nach `SANDBOX_MAX_LIFETIME_SECONDS` + (Standard 7200 s Wanduhrzeit), auch wenn das Backend nicht mehr läuft. + `docker pause` friert auch diesen Timer ein; pausierte Läufe begrenzt + deshalb das Backend: Ein Lauf darf insgesamt höchstens + `SIMULATION_MAX_PAUSED_SECONDS` (Standard 600 s, stets kleiner als die harte + Lebensdauer) pausiert sein, danach wird er beendet und gibt Runner, Admission + und Container frei. Auf macOS muss der Projektpfad in Docker Desktop für File Sharing freigegeben sein. @@ -147,6 +151,7 @@ muss diese Werte nicht ändern. Für größere Installationen können sie über | `SANDBOX_CPU_LIMIT` | CPU-Limit pro Sandbox | | `UNOSIM_GITHUB_TOKEN` | optionaler Read-only-Token nur für `api.github.com` (Ref-Auflösung der Kursinhalte); ohne Token gilt das GitHub-Limit von 60 Abfragen pro Stunde und Server-IP | | `UNOSIM_LOG_MAX_SIZE`, `UNOSIM_LOG_MAX_FILE` | Rotation des Backend-Logs (Docker `json-file`); Standard `10m` je Datei und `5` Dateien | +| `SIMULATION_MAX_PAUSED_SECONDS` | maximale Gesamtpausenzeit eines Simulationslaufs; danach wird er beendet und gibt seine Kapazität frei; Standard `600`, muss kleiner als `SANDBOX_MAX_LIFETIME_SECONDS` sein | | `WS_HEARTBEAT_INTERVAL_MS` | Ping-Intervall der WebSocket-Verbindungen; eine Verbindung ohne Antwort auf den vorherigen Ping wird getrennt und gibt ihre Simulation frei; Standard `30000` ms | Die Defaults der Anwendung stehen in `server/config.ts` (unter anderem diff --git a/docs/SECURITY.md b/docs/SECURITY.md index c1e5c789d..ff67c4195 100644 --- a/docs/SECURITY.md +++ b/docs/SECURITY.md @@ -90,9 +90,14 @@ Every sandbox container carries the label `unosim.owner`: `instance.` when removes the containers of its owner, so a crashed or redeployed backend of the same deployment cleans up its predecessor's sandboxes; containers of other owners are not touched. Concurrently running backends must use distinct IDs. -Independently of the backend, every sandbox runs under +Independently of the backend, every running sandbox runs under `timeout --signal=KILL` and ends after `SANDBOX_MAX_LIFETIME_SECONDS` (default -7200 s wall clock, including paused time). +7200 s wall clock). `docker pause` freezes that timer's process too, so a +paused sandbox is bounded by the backend instead: the paused time of a run is +limited to `SIMULATION_MAX_PAUSED_SECONDS` (default 600 s, always below the +hard lifetime); then the run ends and its runner, admission and container are +released. A paused container left by a crashed backend is removed by the next +start's owner sweep. ## Compile boundary diff --git a/server/config.ts b/server/config.ts index 6c8dd4604..eb7afd7c2 100644 --- a/server/config.ts +++ b/server/config.ts @@ -243,6 +243,20 @@ const simulationAdmissionMax = envInt("SIMULATION_ADMISSION_MAX", 25, { min: 1, const simulationQueueTimeoutMs = envInt("SIMULATION_QUEUE_TIMEOUT_MS", 60_000, { min: 1_000, max: 900_000 }); const sandboxStartSlotTimeoutMs = envInt("SANDBOX_START_SLOT_TIMEOUT_MS", 30_000, { min: 1_000, max: 900_000 }); +const sandboxMaxLifetimeSeconds = envInt("SANDBOX_MAX_LIFETIME_SECONDS", 7_200, { min: 900, max: 86_400 }); +const simulationMaxPausedSeconds = envInt("SIMULATION_MAX_PAUSED_SECONDS", 600, { min: 10, max: 86_400 }); + +/** + * A paused run must end before the sandbox's own hard lifetime, which cannot + * fire while `docker pause` freezes the container. + */ +export function validatePausedBudget(maxPausedSeconds: number, maxLifetimeSeconds: number): void { + if (maxPausedSeconds >= maxLifetimeSeconds) { + throw new Error("SIMULATION_MAX_PAUSED_SECONDS must be lower than SANDBOX_MAX_LIFETIME_SECONDS"); + } +} +validatePausedBudget(simulationMaxPausedSeconds, sandboxMaxLifetimeSeconds); + /** Logical warm-object floor. It never pre-creates Docker containers. */ const logicalWarmRunnerFloor = Math.min(5, simulationMaxConcurrent); @@ -593,10 +607,16 @@ export const config = { instanceId: parseInstanceId(process.env.UNOSIM_INSTANCE_ID), /** * Absolute wall-clock lifetime of a sandbox container, enforced inside the - * container so it also ends when the backend dies. Includes paused time; - * far above compile time plus the longest simulation timeout. + * container so it also ends when the backend dies; far above compile time + * plus the longest simulation timeout. `docker pause` freezes this clock's + * process, so a paused container ends on its pause budget instead. + */ + maxLifetimeSeconds: sandboxMaxLifetimeSeconds, + /** + * Total time one run may spend paused. The backend ends a run that exceeds + * it, which releases its runner, admission and container. */ - maxLifetimeSeconds: envInt("SANDBOX_MAX_LIFETIME_SECONDS", 7_200, { min: 900, max: 86_400 }), + maxPausedSeconds: simulationMaxPausedSeconds, }, // ── Compilation ───────────────────────────────────────────────── diff --git a/server/services/sandbox-runner.ts b/server/services/sandbox-runner.ts index 26b8ac0f1..149247378 100644 --- a/server/services/sandbox-runner.ts +++ b/server/services/sandbox-runner.ts @@ -42,6 +42,9 @@ export class SandboxRunner { private controlCompletion?: Promise; private controlPending = false; + /** Total time one run may spend paused before the runner ends it. */ + private readonly maxPausedMs: number; + private pauseBudgetTimer: NodeJS.Timeout | null = null; /** Completion of the last accepted control; Stop never waits for it. */ get controlResult(): Promise | undefined { @@ -60,8 +63,9 @@ export class SandboxRunner { private get state(): SimulationState { return this.executionState?.state ?? SimulationState.STOPPED; } private set state(v: SimulationState | string) { this.executionState.state = v as SimulationState; } - constructor(options?: { tempDir?: string; processController?: IProcessController }) { + constructor(options?: { tempDir?: string; processController?: IProcessController; maxPausedMs?: number }) { this.processController = options?.processController ?? new ProcessController(); + this.maxPausedMs = options?.maxPausedMs ?? config.sandbox.maxPausedSeconds * 1_000; this.tempDir = options?.tempDir ?? sandboxTempDir(); this.timeoutManager = new SimulationTimeoutManager(); this.fileBuilder = new SketchFileBuilder(this.tempDir); @@ -330,9 +334,39 @@ export class SandboxRunner { if (!s.processKilled) this.processController.writeStdin("[[PAUSE_TIME]]\n"); s.pauseStartTime = Date.now(); this.registryManager.markPauseTime(s.pauseStartTime); + this.startPauseBudget(); this.logger.info("Simulation paused"); } + /** + * The paused time of all pauses of a run is bounded: a paused run keeps its + * runner, admission and (frozen) container, and `docker pause` also freezes + * the sandbox's own hard lifetime. + */ + private startPauseBudget(): void { + this.clearPauseBudget(); + const remainingMs = Math.max(0, this.maxPausedMs - this.executionState.totalPausedTime); + this.pauseBudgetTimer = setTimeout(() => this.endPausedRun(), remainingMs); + this.pauseBudgetTimer.unref?.(); + } + + private clearPauseBudget(): void { + if (this.pauseBudgetTimer) clearTimeout(this.pauseBudgetTimer); + this.pauseBudgetTimer = null; + } + + /** Ends the run like the runtime timeout does: its exit releases the session. */ + private endPausedRun(): void { + this.pauseBudgetTimer = null; + const s = this.executionState; + if (this.state !== SimulationState.PAUSED || s.processKilled || s.terminationRequested) return; + const limitSeconds = Math.round(this.maxPausedMs / 1_000); + this.logger.info(`Paused run exceeded its pause budget of ${limitSeconds}s; ending it`); + s.onOutputCallback?.(`--- Simulation stopped: paused longer than ${limitSeconds} s ---`, true); + s.terminationRequested = true; + this.processController.kill("SIGKILL"); + } + pause(): boolean { if (this.controlPending || this.executionState.terminationRequested || this.state !== SimulationState.RUNNING || !this.processController.hasProcess()) return false; if (this.executionState.currentContainerName) { @@ -347,6 +381,7 @@ export class SandboxRunner { private commitResume(): void { const s = this.executionState; + this.clearPauseBudget(); const pauseDuration = Date.now() - (this.pauseStartTime ?? Date.now()); s.totalPausedTime += pauseDuration; if (!s.processKilled) this.processController.writeStdin(`[[RESUME_TIME:${pauseDuration}]]\n`); @@ -413,6 +448,7 @@ export class SandboxRunner { // Cancels a run that is still preparing or waiting for a start slot. s.runAbort?.abort(); this.controlPending = false; + this.clearPauseBudget(); if (this.state === SimulationState.STOPPED || s.processKilled) { await this.cleanupDockerContainer(s.currentContainerName); return; @@ -483,6 +519,7 @@ export class SandboxRunner { s.runAbort?.abort(); this.controlPending = false; this.controlCompletion = undefined; + this.clearPauseBudget(); if (s.currentContainerName) { await this.cleanupDockerContainer(s.currentContainerName); if (s.currentContainerName) throw new Error(`Docker cleanup unconfirmed for ${s.currentContainerName}`); diff --git a/ssot/ssot_function_definition_PauseResume.md b/ssot/ssot_function_definition_PauseResume.md index f3b270a5d..f11f146c9 100644 --- a/ssot/ssot_function_definition_PauseResume.md +++ b/ssot/ssot_function_definition_PauseResume.md @@ -57,7 +57,7 @@ Die **Pause/Resume-Funktionalität** ermöglicht es dem Benutzer, die laufende A **Gültige Übergänge:** - `RUNNING` → `PAUSED` (nur via Pause-Button möglich) - `PAUSED` → `RUNNING` (nur via Resume-Button möglich) -- `PAUSED` → `STOPPED` (nur via Stop-Button möglich) +- `PAUSED` → `STOPPED` (via Stop-Button oder serverseitig nach Ablauf des Pausenbudgets, siehe 7.2) - `RUNNING` → `STOPPED` (Code-Änderung oder Stop-Button) --- @@ -188,6 +188,16 @@ Beim Start der Simulation wird ein Execution-Timeout geplant. Beim Pausieren wir Beim Fortsetzen wird ein neuer Timeout-Timer mit der gespeicherten Restzeit gestartet. Die reale Dauer der Pause zählt nicht gegen das Simulations-Timeout. Für Benutzer bedeutet das: Eine Simulation, die z. B. nach 50 von 60 Sekunden pausiert wird, hat nach Resume weiterhin etwa 10 Sekunden Restlaufzeit. +### 7.2 Pausenbudget + +Ein pausierter Lauf belegt weiterhin Simulationskapazität (Runner, Admission und den eingefrorenen Sandbox-Container). Die Pausenzeit aller Pausen eines Laufs ist deshalb insgesamt auf `SIMULATION_MAX_PAUSED_SECONDS` begrenzt (Standard 600 s). Ist das Budget während einer Pause aufgebraucht, beendet der Server den Lauf wie bei einem Timeout: + +1. Ausgabe `--- Simulation stopped: paused longer than s ---` +2. Prozess bzw. Container wird beendet und entfernt; Runner und Admission werden freigegeben +3. Nachricht an den Client: `{ "type": "simulation_status", "status": "stopped" }` + +Das Budget gehört zum Lauf, nicht zur einzelnen Pause; ein neuer Start beginnt mit vollem Budget. + --- ## 8. Backend-Implementierung @@ -411,6 +421,7 @@ useEffect(() => { **Stress-Tests:** - Schnelle Pause/Resume-Zyklen - Pause länger als Original-Timeout +- Pause länger als das Pausenbudget (Lauf endet, Kapazität wird frei; auch mit echtem pausiertem Docker-Container) - Serielle Eingaben versuchen während Pause (werden abgelehnt) --- diff --git a/tests/integration/paused-run-budget.test.ts b/tests/integration/paused-run-budget.test.ts new file mode 100644 index 000000000..c154964e4 --- /dev/null +++ b/tests/integration/paused-run-budget.test.ts @@ -0,0 +1,74 @@ +/** + * Integration Test: pause budget of a real, frozen Docker sandbox + * + * `docker pause` freezes every process of the container, including the + * `timeout --signal=KILL` hard lifetime, so a paused sandbox would otherwise + * live (and hold its runner and admission) as long as its WebSocket does. The + * backend's pause budget must end the run and remove the frozen container. + */ + +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { afterEach, describe, expect, test } from "vitest"; +import { SandboxRunner } from "../../server/services/sandbox-runner"; + +const run = promisify(execFile); +const skipHeavy = process.env.SKIP_HEAVY_TESTS !== "0" && process.env.SKIP_HEAVY_TESTS !== "false"; +const maybeDescribe = skipHeavy ? describe.skip : describe; +const PAUSE_BUDGET_MS = 3_000; + +const SKETCH = String.raw` +void setup() { Serial.begin(115200); } +void loop() { Serial.println("tick"); delay(100); } +`.trim(); + +async function containerStatus(name: string): Promise { + const { stdout } = await run("docker", ["ps", "-a", "--filter", `name=^${name}$`, "--format", "{{.Status}}"]); + return stdout.trim(); +} + +async function waitFor(condition: () => boolean | Promise, timeoutMs: number, label: string): Promise { + const deadline = Date.now() + timeoutMs; + while (!(await condition())) { + if (Date.now() > deadline) throw new Error(`Timed out waiting for ${label}`); + await new Promise((resolve) => setTimeout(resolve, 100)); + } +} + +maybeDescribe("paused run budget (real Docker)", () => { + let runner: SandboxRunner | undefined; + + afterEach(async () => { + await runner?.stop(); + }); + + test("ends a paused run after its pause budget and removes the frozen container", async () => { + runner = new SandboxRunner({ maxPausedMs: PAUSE_BUDGET_MS }); + const outputs: string[] = []; + let exitCode: number | null | undefined; + const ready = await runner.runSketch({ + code: SKETCH, + onOutput: (line) => outputs.push(line), + onError: () => {}, + onExit: (code) => { exitCode = code; }, + onCompileError: (error) => { throw new Error(`compile failed: ${error}`); }, + timeoutSec: 60, + }); + expect(ready).toBe(true); + const containerName = (runner as unknown as { executionState: { currentContainerName: string } }) + .executionState.currentContainerName; + await waitFor(() => outputs.join("").includes("tick"), 15_000, "serial output"); + + expect(runner.pause()).toBe(true); + expect(await runner.controlResult).toBe(true); + expect(await containerStatus(containerName)).toContain("Paused"); + const pausedAt = Date.now(); + + await waitFor(() => exitCode !== undefined, PAUSE_BUDGET_MS + 15_000, "end of the paused run"); + expect(Date.now() - pausedAt).toBeGreaterThanOrEqual(PAUSE_BUDGET_MS - 100); + expect(outputs.join("")).toContain("paused longer than"); + // The caller's onExit releases the run; this mirrors what the WebSocket session does. + await runner.stop(); + await waitFor(async () => (await containerStatus(containerName)) === "", 15_000, "container removal"); + }, 90_000); +}); diff --git a/tests/server/routes/simulation-paused-run-end.test.ts b/tests/server/routes/simulation-paused-run-end.test.ts new file mode 100644 index 000000000..1bddaf087 --- /dev/null +++ b/tests/server/routes/simulation-paused-run-end.test.ts @@ -0,0 +1,87 @@ +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { WebSocket } from "ws"; +import type { Logger } from "@shared/logger"; +import { registerSimulationWebSocket } from "../../../server/routes/simulation.ws"; +import type { RunSketchOptions } from "../../../server/services/run-sketch-types"; +import type { SandboxRunner } from "../../../server/services/sandbox-runner"; +import type { SandboxRunnerPool } from "../../../server/services/sandbox-runner-pool"; +import { SimulationAdmissionController } from "../../../server/services/simulation-admission-controller"; + +const SECRET = "a-secure-gateway-secret-with-32-characters"; +const ORIGIN = "https://classroom.example"; +let server: Server | undefined; +let socket: WebSocket | undefined; + +async function waitFor(predicate: () => boolean, label: string): Promise { + const deadline = Date.now() + 3_000; + while (!predicate()) { + if (Date.now() >= deadline) throw new Error(`Timed out waiting for ${label}`); + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +afterEach(async () => { + socket?.terminate(); + await new Promise((resolve) => (server ? server.close(() => resolve()) : resolve())); +}); + +describe("a paused run that ends on its pause budget", () => { + it("releases runner and admission and reports the stopped status", async () => { + const admission = new SimulationAdmissionController(25); + let runOptions: RunSketchOptions | undefined; + const runner = { + runSketch: vi.fn(async (options: RunSketchOptions) => { runOptions = options; return true; }), + pause: vi.fn(() => true), + resume: vi.fn(() => true), + controlResult: Promise.resolve(true), + stop: vi.fn().mockResolvedValue(undefined), + setPinValue: vi.fn(), + sendSerialInput: vi.fn(), + }; + const pool = { + acquireRunner: vi.fn(async () => runner), + releaseRunner: vi.fn().mockResolvedValue(undefined), + getRunnerIndex: vi.fn(() => 0), + getStats: vi.fn(() => ({ availableRunners: 1, totalRunners: 1, maxRunners: 1, queuedRequests: 0 })), + }; + server = createServer(); + registerSimulationWebSocket(server, { + SandboxRunner: class {} as typeof SandboxRunner, + getSimulationRateLimiter: () => ({ checkLimit: () => ({ allowed: true }) }), + getSimulationAdmissionController: () => admission, + shouldSendSimulationEndMessage: () => true, + getLastCompiledCode: () => null, + logger: { debug: vi.fn(), info: vi.fn(), warn: vi.fn(), error: vi.fn() } as unknown as Logger, + runnerPool: pool as unknown as SandboxRunnerPool, + trust: { mode: "gateway", gatewaySecret: SECRET, trustedProxy: "127.0.0.1" }, + allowedWebSocketOrigins: [ORIGIN], + disableRateLimit: true, + }); + await new Promise((resolve) => server!.listen(0, "127.0.0.1", resolve)); + socket = new WebSocket(`ws://127.0.0.1:${(server.address() as AddressInfo).port}/ws`, { + headers: { Origin: ORIGIN, "X-UnoSim-Gateway-Secret": SECRET, "X-UnoSim-Subject": "student", "X-UnoSim-Roles": "user" }, + }); + const statuses: string[] = []; + socket.on("message", (raw) => { + const message = JSON.parse(raw.toString()) as { type: string; status?: string }; + if (message.type === "simulation_status" && message.status) statuses.push(message.status); + }); + await new Promise((resolve, reject) => { socket!.once("open", resolve); socket!.once("error", reject); }); + + socket.send(JSON.stringify({ type: "start_simulation", code: "void setup(){} void loop(){}" })); + await waitFor(() => statuses.includes("running"), "running"); + socket.send(JSON.stringify({ type: "pause_simulation" })); + await waitFor(() => statuses.at(-1) === "paused", "paused"); + expect(admission.getStats().active).toBe(1); + + // The runner's pause budget ends the process; its exit reaches the session as onExit. + runOptions!.onExit?.(137); + + await waitFor(() => statuses.at(-1) === "stopped", "stopped"); + await waitFor(() => admission.getStats().active === 0, "admission release"); + expect(runner.stop).toHaveBeenCalled(); + expect(pool.releaseRunner).toHaveBeenCalledWith(runner); + }); +}); diff --git a/tests/server/services/sandbox-runner-pause-budget.test.ts b/tests/server/services/sandbox-runner-pause-budget.test.ts new file mode 100644 index 000000000..9c68f634e --- /dev/null +++ b/tests/server/services/sandbox-runner-pause-budget.test.ts @@ -0,0 +1,91 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { validatePausedBudget } from "../../../server/config"; +import { SandboxRunner } from "../../../server/services/sandbox-runner"; + +const success = { code: 0, stdout: "", stderr: "", error: null }; +const PAUSE_BUDGET_MS = 1_000; +async function flush() { for (let i = 0; i < 8; i++) await Promise.resolve(); } + +function fixture() { + const pc = { + spawn: vi.fn(), onStdout: vi.fn(), onStderr: vi.fn(), onStderrLine: vi.fn(), + supportsStderrLineStreaming: vi.fn(() => true), onClose: vi.fn(), onError: vi.fn(), + writeStdin: vi.fn(() => true), kill: vi.fn(), destroySockets: vi.fn(), + hasProcess: vi.fn(() => true), clearListeners: vi.fn(), getPid: vi.fn(() => null), + }; + const runner = new SandboxRunner({ processController: pc, maxPausedMs: PAUSE_BUDGET_MS }); + const internals = runner as any; + const s = internals.executionState; + const output = vi.fn(); + Object.assign(s, { + state: "running", currentContainerName: "unosim-synthetic-paused", runGeneration: 1, + runAbort: new AbortController(), processKilled: false, onOutputCallback: output, + }); + vi.spyOn(internals.processExecutor, "execute").mockResolvedValue(success); + return { runner, pc, s, output }; +} + +const killedWithSigkill = (pc: ReturnType["pc"]) => + pc.kill.mock.calls.some(([signal]) => signal === "SIGKILL"); + +describe("paused run budget", () => { + beforeEach(() => vi.useFakeTimers()); + afterEach(() => { vi.restoreAllMocks(); vi.clearAllTimers(); vi.useRealTimers(); }); + + it("ends a run that stays paused beyond its pause budget", async () => { + const f = fixture(); + f.runner.pause(); + await flush(); + expect(f.runner.isPaused).toBe(true); + + await vi.advanceTimersByTimeAsync(PAUSE_BUDGET_MS - 1); + expect(killedWithSigkill(f.pc)).toBe(false); + + await vi.advanceTimersByTimeAsync(1); + // Same termination path as the runtime timeout: the process exit reports onExit, + // which releases runner and admission and sends the stopped status. + expect(killedWithSigkill(f.pc)).toBe(true); + expect(f.s.terminationRequested).toBe(true); + expect(f.output).toHaveBeenCalledWith(expect.stringContaining("paused longer than"), true); + }); + + it("counts the paused time of all pauses of a run together", async () => { + const f = fixture(); + f.runner.pause(); + await flush(); + await vi.advanceTimersByTimeAsync(600); + f.runner.resume(); + await flush(); + expect(f.runner.isPaused).toBe(false); + + await vi.advanceTimersByTimeAsync(10_000); + expect(killedWithSigkill(f.pc)).toBe(false); + + f.runner.pause(); + await flush(); + await vi.advanceTimersByTimeAsync(399); + expect(killedWithSigkill(f.pc)).toBe(false); + await vi.advanceTimersByTimeAsync(1); + expect(killedWithSigkill(f.pc)).toBe(true); + }); + + it("does not end a run that was stopped while paused", async () => { + const f = fixture(); + f.runner.pause(); + await flush(); + await f.runner.stop(); + f.pc.kill.mockClear(); + + await vi.advanceTimersByTimeAsync(PAUSE_BUDGET_MS * 2); + + expect(f.pc.kill).not.toHaveBeenCalled(); + expect(f.output).not.toHaveBeenCalled(); + }); +}); + +describe("pause budget configuration", () => { + it("must end a paused run before the sandbox's own hard lifetime", () => { + expect(() => validatePausedBudget(600, 7_200)).not.toThrow(); + expect(() => validatePausedBudget(7_200, 7_200)).toThrow(/SIMULATION_MAX_PAUSED_SECONDS/); + }); +}); diff --git a/vitest.config.ts b/vitest.config.ts index 101a902e3..2aad2a39d 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -126,6 +126,7 @@ export default defineConfig({ "tests/integration/docker-security-contract.test.ts", "tests/integration/sandbox-orphan-sweep.test.ts", "tests/integration/runtime-output-storm.test.ts", + "tests/integration/paused-run-budget.test.ts", "tests/server/telemetry-heartbeat-integration.test.ts", "tests/core/sandbox-stress.test.ts", ...serializedHttpUnitTests, @@ -177,6 +178,7 @@ export default defineConfig({ "tests/integration/docker-security-contract.test.ts", "tests/integration/sandbox-orphan-sweep.test.ts", "tests/integration/runtime-output-storm.test.ts", + "tests/integration/paused-run-budget.test.ts", ], environment: "node", setupFiles: ["./tests/setup.node.ts"],