Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ changing the server default; see

For host-specific Capacity sizing, use `npm run capacity:calibrate` and follow
the review workflow in [`docs/CAPACITY_MODEL.md`](docs/CAPACITY_MODEL.md).
Calibration needs a built application (`npm run build`) and the sandbox image
(`npm run build:sandbox`); it aborts early with a hint if the build is missing.

### Architecture Overview

Expand Down
9 changes: 9 additions & 0 deletions docs/CAPACITY_MODEL.md
Original file line number Diff line number Diff line change
Expand Up @@ -485,8 +485,17 @@ remain under the ignored capacity-test-results/ directory.
Use the calibration command when a host, Docker runtime, sandbox resource
limit, or CPU-intensive workload changes materially:

npm ci
npm run build
npm run build:sandbox
npm run capacity:calibrate -- --expected-users 200

The calibration starts its own backend from the source tree, which serves the
built client from `dist/public`. If `dist/public/index.html` is missing, the
command aborts before any Docker or load measurement with a message to run
`npm run build`; it never builds the application itself. `--dry-run` starts no
backend and does not require the build.

The command follows **Calibrate -> Review -> Apply**. It measures Docker
control latency, directly measured active-simulation candidates, startup
concurrency candidates, and (unless `--skip-classroom` is supplied) a
Expand Down
14 changes: 13 additions & 1 deletion docs/INSTALL_SERVER.md
Original file line number Diff line number Diff line change
Expand Up @@ -231,12 +231,24 @@ Ist der Compile-Worker-Pool ausgelastet (500 wartende Compiles oder 30 s
Wartezeit), antwortet `/api/compile` mit `503` und `SYSTEM_BUSY`.

Für größere Installationen sollte vor einer manuellen Erhöhung der
Kapazitätswerte die Host-Kalibrierung ausgeführt werden:
Kapazitätswerte die Host-Kalibrierung ausgeführt werden. Sie läuft direkt auf
dem Host im Repository-Checkout und startet ein eigenes Test-Backend aus dem
Quellcode; dieses liefert den gebauten Client aus `dist/public` aus. Das
Server-Image `unosim-server` wird dafür nicht benötigt.

```bash
npm ci
npm run build
npm run build:sandbox
npm run capacity:calibrate -- --expected-users 200
```

`npm run build:sandbox` ist derselbe Befehl wie
`docker build -f Dockerfile.sandbox -t unosim-sandbox:latest .` oben. Fehlt der
gebaute Client (`dist/public/index.html`), bricht die Kalibrierung vor allen
Docker- und Lastmessungen mit dem Hinweis ab, `npm run build` auszuführen. Sie
baut die Anwendung nicht selbst.

Die Ergebnisse sind host-spezifisch und hängen unter anderem von Docker,
Sandbox-CPU-/Memory-Limits und dem tatsächlichen Workload ab. `capacity.env`
ist nur ein Review-Vorschlag; fehlende Werte müssen geprüft und ausdrücklich
Expand Down
26 changes: 25 additions & 1 deletion scripts/calibrate-capacity.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,11 @@
import { execFileSync, spawn, type ChildProcess } from "node:child_process";
import { randomUUID } from "node:crypto";
import fs from "node:fs";
import net from "node:net";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { INPUT_LIMITS, TEST_RUN_ID_PATTERN } from "@shared/input-limits";
import {
CALIBRATION_POLICY_VERSION,
planActiveCandidates,
Expand Down Expand Up @@ -47,6 +50,8 @@ const DOCKER_CONTROL_RANGE = { min: 100, max: 30_000 };
const SANDBOX_START_SLOT_DEFAULT_MS = 30_000;
const SANDBOX_START_SLOT_RANGE = { min: 1_000, max: 900_000 };
const FIXED_CLASSROOM_DURATION_SEC = 60 as const;
// The owned backend runs server/index.ts with NODE_ENV=test, which serves the built client via serveStatic().
const BUILT_CLIENT_INDEX = fileURLToPath(new URL("../dist/public/index.html", import.meta.url));

export type SafetyEvent = {
atMs: number;
Expand Down Expand Up @@ -118,8 +123,24 @@ export type CalibrationDependencies = {
runScenario?: (options: CapacityScenarioOptions) => Promise<CapacityScenarioMeasurement>;
startBackend?: (capacity: EffectiveCapacityConfiguration, runId: string) => Promise<{ baseUrl: string; stop: () => Promise<void> }>;
ownedContainerCount?: (runId: string, includeStopped?: boolean) => number;
verifyBuildArtifacts?: () => void;
};

/** Fails early when the built client that the owned backend serves is missing. */
export function assertCalibrationBuildArtifacts(indexPath: string = BUILT_CLIENT_INDEX): void {
if (fs.existsSync(indexPath)) return;
throw new Error(`Capacity calibration requires a built application (${indexPath} not found); run npm run build.`);
}

/** Creates a calibration run ID that satisfies the central test-run-ID pattern and length limit. */
export function createCalibrationRunId(now: () => number = Date.now, uuid: () => string = randomUUID): string {
const runId = `cap_${now()}_${uuid()}`;
if (!TEST_RUN_ID_PATTERN.test(runId)) {
throw new Error(`Calibration run ID must match ${TEST_RUN_ID_PATTERN} (max ${INPUT_LIMITS.webSocket.maxTestRunIdChars} characters)`);
}
return runId;
}

function timestampOutputDir(): string {
return path.join("capacity-test-results", `calibration-${new Date().toISOString().replaceAll(":", "-").replaceAll(".", "-")}`);
}
Expand Down Expand Up @@ -527,7 +548,7 @@ type PhaseRunResult = {

async function measureCalibrationCandidate(context: CalibrationPhaseContext, phase: SafetyEvent["phase"], active: number, startup: number, scenario: "burst" | "classroom" = "burst"): Promise<CapacityScenarioMeasurement> {
if (context.now() >= context.deadline) throw new Error("calibration deadline reached");
const runId = `capacity_calibration_${Date.now()}_${randomUUID()}`;
const runId = createCalibrationRunId();
const classroom = scenario === "classroom";
const clientCount = classroom ? context.config.expectedUsers : active;
const admission = classroom ? context.config.expectedUsers : Math.max(context.baseCapacity.simulationAdmissionMax, clientCount);
Expand Down Expand Up @@ -754,11 +775,14 @@ export async function runCalibration(
const runScenario = dependencies.runScenario ?? ((options: CapacityScenarioOptions) => runCapacityScenario(options));
const startBackend = dependencies.startBackend ?? defaultStartBackend;
const countOwnedContainers = dependencies.ownedContainerCount ?? ownedContainerCount;
const verifyBuildArtifacts = dependencies.verifyBuildArtifacts ?? (() => assertCalibrationBuildArtifacts());
const image = process.env.DOCKER_SANDBOX_IMAGE ?? DEFAULT_IMAGE;
const startedAt = now();
const deadline = startedAt + config.maxDurationMin * 60_000;
const safetyEvents: SafetyEvent[] = [];
console.log(`UnoSim Capacity Calibration\nPolicy: expected users=${config.expectedUsers}, target CPU=${config.targetCpuPercent}%, hard CPU=${config.maxCpuPercent}%, max wait=${config.maxUserWaitSec}s, duration=${config.maxDurationMin}min`);
// Dry runs start no backend, so only measuring runs need the built client.
if (!config.dryRun) verifyBuildArtifacts();
let host = unavailableHostProbe();
let docker = unavailableDockerProbe(image);
try {
Expand Down
126 changes: 126 additions & 0 deletions tests/server/calibrate-capacity-cli.test.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,11 @@
import { describe, expect, it, vi } from "vitest";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { INPUT_LIMITS, TEST_RUN_ID_PATTERN } from "@shared/input-limits";
import {
assertCalibrationBuildArtifacts,
createCalibrationRunId,
effectiveCapacityForPhase,
getClassroomOutcomeCounts,
parseCalibrationArgs,
Expand Down Expand Up @@ -46,6 +50,14 @@ describe("capacity calibration CLI", () => {
expect(documentation).toContain("60-second active duration");
});

it("documents the build preconditions in order for host calibration", () => {
const sequence = /npm ci\n\s*npm run build\n\s*npm run build:sandbox\n\s*npm run capacity:calibrate -- --expected-users/;
for (const file of ["docs/INSTALL_SERVER.md", "docs/CAPACITY_MODEL.md"]) {
const documentation = fs.readFileSync(path.resolve(process.cwd(), file), "utf8");
expect(documentation, file).toMatch(sequence);
}
});

it("applies documented defaults and fixed classroom duration", async () => {
await expect(parseCalibrationArgs([])).resolves.toMatchObject({
expectedUsers: 200,
Expand Down Expand Up @@ -151,6 +163,7 @@ describe("capacity calibration CLI", () => {
});
let stopCount = 0;
const deps: CalibrationDependencies = {
verifyBuildArtifacts: () => undefined,
collectHostProbe: vi.fn(async () => {
order.push("host");
return {
Expand Down Expand Up @@ -226,6 +239,7 @@ describe("capacity calibration CLI", () => {
dryRun: false,
verbose: false,
}, {
verifyBuildArtifacts: () => undefined,
collectHostProbe: async () => ({
required: { gitSha: "abc", gitDirty: false, nodeVersion: "v24.20.0", architecture: "x64", logicalCpus: 20 },
optional: { physicalMemoryBytes: 16_000_000_000, loadAverage: [1], availableMemoryBytes: 8_000_000_000, swapUsedBytes: 0, iowaitPercent: 1, thermalPressure: null },
Expand Down Expand Up @@ -288,6 +302,7 @@ describe("capacity calibration CLI", () => {
dryRun: false,
verbose: false,
}, {
verifyBuildArtifacts: () => undefined,
collectHostProbe: async () => ({
required: { gitSha: "abc", gitDirty: false, nodeVersion: "v24.20.0", architecture: "x64", logicalCpus: 4 },
optional: { physicalMemoryBytes: 16_000_000_000, loadAverage: [1], availableMemoryBytes: 8_000_000_000, swapUsedBytes: 0, iowaitPercent: 1, thermalPressure: null },
Expand Down Expand Up @@ -318,6 +333,7 @@ describe("capacity calibration CLI", () => {
dryRun: false,
verbose: false,
}, {
verifyBuildArtifacts: () => undefined,
collectHostProbe: async () => ({
required: { gitSha: "abc", gitDirty: false, nodeVersion: "v24.20.0", architecture: "x64", logicalCpus: 4 },
optional: { physicalMemoryBytes: 16_000_000_000, loadAverage: [1], availableMemoryBytes: 8_000_000_000, swapUsedBytes: 0, iowaitPercent: 1, thermalPressure: null },
Expand All @@ -332,4 +348,114 @@ describe("capacity calibration CLI", () => {
expect(report.stopReason).toBe("Docker unavailable");
fs.rmSync(outputDir, { recursive: true, force: true });
});
describe("calibration run IDs", () => {
it("matches the central test-run-ID pattern and length limit", () => {
const runId = createCalibrationRunId();
expect(runId).toMatch(TEST_RUN_ID_PATTERN);
expect(runId.length).toBeLessThanOrEqual(INPUT_LIMITS.webSocket.maxTestRunIdChars);
});

it("stays within the limit for the longest realistic timestamp and a canonical UUID", () => {
const runId = createCalibrationRunId(() => 9_999_999_999_999, () => "00000000-0000-4000-8000-000000000000");
expect(runId).toBe("cap_9999999999999_00000000-0000-4000-8000-000000000000");
expect(runId).toMatch(TEST_RUN_ID_PATTERN);
expect(runId.length).toBeLessThanOrEqual(INPUT_LIMITS.webSocket.maxTestRunIdChars);
});

it("distinguishes runs by timestamp and by UUID", () => {
const uuid = "11111111-1111-4111-8111-111111111111";
const other = "22222222-2222-4222-8222-222222222222";
const ids = new Set([
createCalibrationRunId(() => 1_000, () => uuid),
createCalibrationRunId(() => 1_001, () => uuid),
createCalibrationRunId(() => 1_000, () => other),
]);
expect(ids.size).toBe(3);
});

it("rejects an ID that the backend would refuse instead of starting a doomed backend", () => {
expect(() => createCalibrationRunId(() => 1, () => "x".repeat(INPUT_LIMITS.webSocket.maxTestRunIdChars))).toThrow(/Calibration run ID must match/);
expect(() => createCalibrationRunId(() => 1, () => "not url safe/..")).toThrow(/Calibration run ID must match/);
});
});

describe("built-application preflight", () => {
const withTempDir = (run: (dir: string) => void | Promise<void>) => async () => {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "unosim-calibration-build-"));
try { await run(dir); } finally { fs.rmSync(dir, { recursive: true, force: true }); }
};
const baseConfig = {
expectedUsers: 20,
targetCpuPercent: 75,
maxCpuPercent: 85,
maxUserWaitSec: 240,
maxDurationMin: 30,
classroomDurationSec: 60,
skipClassroom: false,
skipStartupTuning: false,
verbose: false,
} as const;

it("accepts an existing built client index", withTempDir((dir) => {
const indexPath = path.join(dir, "index.html");
fs.writeFileSync(indexPath, "<!doctype html>");
expect(() => assertCalibrationBuildArtifacts(indexPath)).not.toThrow();
}));

it("tells the operator to run npm run build when the client index is missing", withTempDir((dir) => {
const indexPath = path.join(dir, "dist", "public", "index.html");
expect(() => assertCalibrationBuildArtifacts(indexPath)).toThrow(
/Capacity calibration requires a built application.*run npm run build\./,
);
expect(() => assertCalibrationBuildArtifacts(indexPath)).toThrow(indexPath);
}));

it("aborts before any Docker probe, control measurement, backend, or report when the build is missing", withTempDir(async (dir) => {
const outputDir = path.join(dir, "results");
const missingIndex = path.join(dir, "dist", "public", "index.html");
const collectHostProbe = vi.fn();
const collectDockerProbe = vi.fn();
const measureDockerControlLatency = vi.fn();
const startBackend = vi.fn();
const runScenario = vi.fn();
await expect(runCalibration(
{ ...baseConfig, outputDir, dryRun: false },
{
verifyBuildArtifacts: () => assertCalibrationBuildArtifacts(missingIndex),
collectHostProbe,
collectDockerProbe,
measureDockerControlLatency,
startBackend,
runScenario,
},
)).rejects.toThrow(/requires a built application.*npm run build/);
expect(collectHostProbe).not.toHaveBeenCalled();
expect(collectDockerProbe).not.toHaveBeenCalled();
expect(measureDockerControlLatency).not.toHaveBeenCalled();
expect(startBackend).not.toHaveBeenCalled();
expect(runScenario).not.toHaveBeenCalled();
expect(fs.existsSync(outputDir)).toBe(false);
}));

it("does not require the build for a dry run, which starts no backend", withTempDir(async (dir) => {
const verifyBuildArtifacts = vi.fn(() => { throw new Error("must not be called"); });
const startBackend = vi.fn();
const result = await runCalibration(
{ ...baseConfig, outputDir: path.join(dir, "results"), dryRun: true },
{
verifyBuildArtifacts,
collectHostProbe: async () => ({
required: { gitSha: "abc", gitDirty: false, nodeVersion: "v24.20.0", architecture: "x64", logicalCpus: 4 },
optional: { physicalMemoryBytes: 16_000_000_000, loadAverage: [1], availableMemoryBytes: 8_000_000_000, swapUsedBytes: 0, iowaitPercent: 1, thermalPressure: null },
safetySignals: { cpuAvailable: true, memoryAvailable: true },
}),
collectDockerProbe: async () => ({ clientVersion: "29", serverVersion: "29", architecture: "x64", cpus: 4, memoryBytes: 16_000_000_000, storageDriver: "overlay2", daemonHealthy: true, image: { reference: "unosim-sandbox:latest", id: "img", digest: null }, runningContainers: [] }),
startBackend,
},
);
expect(verifyBuildArtifacts).not.toHaveBeenCalled();
expect(startBackend).not.toHaveBeenCalled();
expect(result.phases.active).toEqual([]);
}));
});
});
Loading