From cc340de7d7e1ae397d863cf5d9926d1ba20d5f9e Mon Sep 17 00:00:00 2001 From: Thibaut Fatus Date: Fri, 4 Sep 2026 16:35:27 +0200 Subject: [PATCH 1/5] [feat] evaluation profiles and run stamps Pin every harness LLM (seed generation, expansion, user simulator, judges) in an evaluation profile with inline configs and a content hash, and stamp every seed, scenario, test result and result header with the effective profile, resolved model configs, prompts fingerprint, packs, code revision and input corpus hash. - profiles/kora.json reproduces the previous hardcoded CLI defaults; a test asserts each role matches its models.json entry - profiles/*.local.json are gitignored scratch profiles for testing a model configuration (--profile .local); their hash is not checked - CI guards: committed profile hashes and the prompts fingerprint must be bumped when their content changes (tests print the expected value) - per-role CLI flags remain as explicit overrides: warned, listed in the stamp, and hashed differently from the named profile - graceful-restart temp dirs hold a stamp.json; resuming under a different configuration is refused - result headers record `served`: the model ids the provider reported - new `kora profile [--check | --print-hash]` command; `validate` prints the profile summary - Stamp.run mirrors Packs.run so infra can scope a stamp per run Claude-Session: https://claude.ai/code/session_017SeRjc1yMoa617ZR2bxeE8 --- .gitignore | 3 + EVALUATION_PROCESS.md | 6 + README.md | 173 ++++++++++++++++-- .../benchmark/src/__tests__/runTest.test.ts | 19 ++ packages/benchmark/src/index.ts | 4 + packages/benchmark/src/kora.ts | 11 ++ packages/benchmark/src/model/modelSpec.ts | 51 ++++++ packages/benchmark/src/model/scenario.ts | 3 + packages/benchmark/src/model/scenarioSeed.ts | 3 + packages/benchmark/src/model/testResult.ts | 5 +- .../__tests__/promptsFingerprint.test.ts | 68 +++++++ .../src/prompts/promptsFingerprint.ts | 49 +++++ .../benchmark/src/stamp/__tests__/fixtures.ts | 33 ++++ .../src/stamp/__tests__/stamp.test.ts | 148 +++++++++++++++ packages/benchmark/src/stamp/runStamp.ts | 136 ++++++++++++++ packages/benchmark/src/stamp/stamp.ts | 67 +++++++ packages/cli/src/cli.ts | 107 ++++++----- packages/cli/src/commands/continueCommand.ts | 74 +++++--- .../src/commands/expandScenariosCommand.ts | 57 +++--- .../cli/src/commands/generateSeedsCommand.ts | 23 ++- packages/cli/src/commands/profileCommand.ts | 44 +++++ packages/cli/src/commands/reassessCommand.ts | 50 +++-- packages/cli/src/commands/runCommand.ts | 73 ++++---- .../shared/__tests__/cacheStamp.test.ts | 65 +++++++ .../shared/__tests__/resultHeader.test.ts | 56 ++++++ .../cli/src/commands/shared/buildContext.ts | 4 +- .../cli/src/commands/shared/cacheStamp.ts | 68 +++++++ .../cli/src/commands/shared/resultHeader.ts | 44 +++++ packages/cli/src/commands/validateCommand.ts | 28 +-- packages/cli/src/models/gatewayModel.ts | 37 +++- packages/cli/src/models/modelConfig.ts | 12 +- .../__tests__/committedProfiles.test.ts | 71 +++++++ .../__tests__/effectiveProfile.test.ts | 121 ++++++++++++ .../cli/src/profiles/__tests__/fixtures.ts | 37 ++++ .../profiles/__tests__/loadProfile.test.ts | 83 +++++++++ .../src/profiles/__tests__/profile.test.ts | 95 ++++++++++ .../src/profiles/__tests__/profiles.test.ts | 41 +++++ .../src/profiles/__tests__/roleModels.test.ts | 40 ++++ packages/cli/src/profiles/effectiveProfile.ts | 125 +++++++++++++ packages/cli/src/profiles/loadProfile.ts | 93 ++++++++++ packages/cli/src/profiles/printProfile.ts | 118 ++++++++++++ packages/cli/src/profiles/profile.ts | 157 ++++++++++++++++ packages/cli/src/profiles/profiles.ts | 61 ++++++ packages/cli/src/profiles/roleModels.ts | 104 +++++++++++ packages/cli/src/shared/packageVersion.ts | 23 +++ packages/cli/src/shared/sha256File.ts | 7 + .../src/stamp/__tests__/buildRunStamp.test.ts | 74 ++++++++ packages/cli/src/stamp/__tests__/fixtures.ts | 28 +++ .../cli/src/stamp/__tests__/gitInfo.test.ts | 14 ++ packages/cli/src/stamp/buildRunStamp.ts | 60 ++++++ packages/cli/src/stamp/gitInfo.ts | 28 +++ profiles/example.local.json.example | 56 ++++++ profiles/kora.json | 56 ++++++ scripts/README.md | 6 + 54 files changed, 2808 insertions(+), 211 deletions(-) create mode 100644 packages/benchmark/src/model/modelSpec.ts create mode 100644 packages/benchmark/src/prompts/__tests__/promptsFingerprint.test.ts create mode 100644 packages/benchmark/src/prompts/promptsFingerprint.ts create mode 100644 packages/benchmark/src/stamp/__tests__/fixtures.ts create mode 100644 packages/benchmark/src/stamp/__tests__/stamp.test.ts create mode 100644 packages/benchmark/src/stamp/runStamp.ts create mode 100644 packages/benchmark/src/stamp/stamp.ts create mode 100644 packages/cli/src/commands/profileCommand.ts create mode 100644 packages/cli/src/commands/shared/__tests__/cacheStamp.test.ts create mode 100644 packages/cli/src/commands/shared/__tests__/resultHeader.test.ts create mode 100644 packages/cli/src/commands/shared/cacheStamp.ts create mode 100644 packages/cli/src/commands/shared/resultHeader.ts create mode 100644 packages/cli/src/profiles/__tests__/committedProfiles.test.ts create mode 100644 packages/cli/src/profiles/__tests__/effectiveProfile.test.ts create mode 100644 packages/cli/src/profiles/__tests__/fixtures.ts create mode 100644 packages/cli/src/profiles/__tests__/loadProfile.test.ts create mode 100644 packages/cli/src/profiles/__tests__/profile.test.ts create mode 100644 packages/cli/src/profiles/__tests__/profiles.test.ts create mode 100644 packages/cli/src/profiles/__tests__/roleModels.test.ts create mode 100644 packages/cli/src/profiles/effectiveProfile.ts create mode 100644 packages/cli/src/profiles/loadProfile.ts create mode 100644 packages/cli/src/profiles/printProfile.ts create mode 100644 packages/cli/src/profiles/profile.ts create mode 100644 packages/cli/src/profiles/profiles.ts create mode 100644 packages/cli/src/profiles/roleModels.ts create mode 100644 packages/cli/src/shared/packageVersion.ts create mode 100644 packages/cli/src/shared/sha256File.ts create mode 100644 packages/cli/src/stamp/__tests__/buildRunStamp.test.ts create mode 100644 packages/cli/src/stamp/__tests__/fixtures.ts create mode 100644 packages/cli/src/stamp/__tests__/gitInfo.test.ts create mode 100644 packages/cli/src/stamp/buildRunStamp.ts create mode 100644 packages/cli/src/stamp/gitInfo.ts create mode 100644 profiles/example.local.json.example create mode 100644 profiles/kora.json diff --git a/.gitignore b/.gitignore index 4c39540..2aa6350 100644 --- a/.gitignore +++ b/.gitignore @@ -147,3 +147,6 @@ vite.config.ts.timestamp-* # Claude */settings.local.json + +# Local evaluation profiles (see README "Evaluation profiles") +profiles/*.local.json diff --git a/EVALUATION_PROCESS.md b/EVALUATION_PROCESS.md index a16ae4a..9212d3c 100644 --- a/EVALUATION_PROCESS.md +++ b/EVALUATION_PROCESS.md @@ -287,6 +287,12 @@ Two quirks worth knowing about the shipped files: - **No taxonomy stamp.** These seeds predate packs, so `taxonomyId` and `taxonomyVersion` are absent — exactly the case the optional stamp in `model/scenarioSeed.ts` allows for. +- **No run stamp either.** Records written today carry a `stamp` (evaluation + profile, prompts fingerprint, packs, code revision; see the README's + "Evaluation profiles"). The shipped corpus predates it. A rerun of the + commands above uses the `kora` profile with the chains shown as explicit + overrides, so its stamp records an ad-hoc profile hash with + `overrides: ["seeds"]` / `["expansion", "expansionUser"]`. ## Dead code diff --git a/README.md b/README.md index b7a5451..38d01fc 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,13 @@ These apply to every command and must be given before the command name: | -------------------------- | --------------------------------------------------------------------------------------------------- | | `--taxonomy ` | Risk taxonomy pack: a registered name (`kora`) or a path to a JSON file. Env: `KORA_TAXONOMY` | | `--behaviors ` | Behavior pack: a registered name (`kora`) or a path to a JSON file. Env: `KORA_BEHAVIORS` | +| `--profile ` | Evaluation profile pinning the model for every pipeline role: a name under `profiles/` (`kora`), a local scratch profile (`.local`), or a path to a JSON file. Env: `KORA_PROFILE` (default: `kora`) | | `-d, --debug` | Print full errors and debug information | -Both default to the bundled KORA pack, so no configuration is needed to run the -benchmark as published. See [Using a custom taxonomy](#using-a-custom-taxonomy). +Packs default to the bundled KORA pack and the profile to `profiles/kora.json`, +so no configuration is needed to run the benchmark as published. See +[Using a custom taxonomy](#using-a-custom-taxonomy) and +[Evaluation profiles](#evaluation-profiles). ## Pipeline stages @@ -66,7 +69,7 @@ yarn kora generate-seeds [model] | Argument / Option | Description | | -------------------------- | ------------------------------------------------------------------------------------- | -| `[model]` | Model(s) to use for seed generation (default: `gpt-4o`). Comma-separated for a per-task fallback chain (e.g. `gpt-4o,gpt-4o:extended,gpt-5.5:low,gemini-2.5-flash:limited`); each task tries models in order, advancing only when one exhausts its retries. | +| `[model]` | Override the profile's `seeds` role with `models.json` slug(s) (default: from profile). Comma-separated for a per-task fallback chain (e.g. `gpt-4o,gpt-4o:extended,gpt-5.5:low,gemini-2.5-flash:limited`); each task tries models in order, advancing only when one exhausts its retries. | | `-o, --output ` | Output JSONL file (default: `data/scenarioSeeds.jsonl`) | | `--seeds-per-task ` | Seeds per risk/age/motivation combination (default: `8`) | | `--total-seeds ` | Total seeds to generate per risk, sampled across age/motivation combos (1 seed each; mutually exclusive with `--seeds-per-task`) | @@ -120,8 +123,8 @@ yarn kora expand-scenarios [model] [user-model] | Argument / Option | Description | | --------------------- | ---------------------------------------------------------------------------------------- | -| `[model]` | Model(s) for scenario expansion (default: `gpt-5.2:high`). Comma-separated for a per-task fallback chain — escalates on both thrown errors *and* `ScenarioValidationError` (e.g. when the model returns valid JSON but the content is truncated/incoherent). | -| `[user-model]` | Model(s) for generating the first user message (default: `deepseek-v3.2`). Comma-separated for a per-call fallback chain (escalates only on thrown errors). | +| `[model]` | Override the profile's `expansion` role with `models.json` slug(s) (default: from profile). Comma-separated for a per-task fallback chain — escalates on both thrown errors *and* `ScenarioValidationError` (e.g. when the model returns valid JSON but the content is truncated/incoherent). | +| `[user-model]` | Override the profile's `expansionUser` role, used for the first user message (default: from profile). Comma-separated for a per-call fallback chain (escalates only on thrown errors). | | `-i, --input ` | Input seeds JSONL file (default: `data/scenarioSeeds.jsonl`) | | `-o, --output ` | Output scenarios JSONL file (default: `data/scenarios.jsonl`) | | `--risk-ids ` | Comma-separated risk IDs to restrict expansion to (default: all seeds in the input file) | @@ -137,8 +140,8 @@ yarn kora run [user-model] | Argument / Option | Description | | --------------------- | ------------------------------------------------------------------------------------------------------------------ | | `` | Model to benchmark | -| `[user-model]` | Model to use for simulating the child user (default: `deepseek-v3.2`) | -| `--judges ` | Comma-separated judge models (default: `gpt-5.2:medium:limited`) | +| `[user-model]` | Override the profile's `user` role (child simulator) with a `models.json` slug (default: from profile) | +| `--judges ` | Override the profile's `judges` role with comma-separated `models.json` slugs, odd count (default: from profile) | | `-i, --input ` | Input scenarios JSONL file (default: `data/scenarios.jsonl`) | | `-o, --output ` | Output results JSON file (default: `data/results.json`) | | `--prompts ` | Comma-separated prompt variants to test (default: `default`) | @@ -148,9 +151,9 @@ yarn kora run [user-model] | `--reverse` | Process scenarios in reverse file order (last scenario first); useful for order-effect comparisons | | `--cooldown ` | Seconds to sleep between sequential test tasks; pair with `--concurrency 1` to avoid app rate-limiting (default: 0) | -By default a single judge (`gpt-5.2:medium:limited`) grades every conversation, matching the production grading pipeline. When multiple judge models are specified, each judge independently evaluates every conversation: the final grade is the **median** across judges (on the ordered scale failing < adequate < exemplary), and the occurrence count is the **mean** (rounded). Per-judge results are stored in each test result for analysis. +By default a single judge (`gpt-5.2:medium:limited`, from the `kora` profile) grades every conversation, matching the production grading pipeline. When multiple judge models are specified, each judge independently evaluates every conversation: the final grade is the **median** across judges (on the ordered scale failing < adequate < exemplary), and the occurrence count is the **mean** (rounded). Per-judge results are stored in each test result for analysis. -All commands write to `data/` by default. Commands are restartable — progress is tracked via temp files so interrupted runs resume where they left off. +All commands write to `data/` by default. Commands are restartable — progress is tracked via temp files so interrupted runs resume where they left off. A resume is refused when the temp files were produced under a different configuration (other profile, override, prompts or packs); see [Run stamps](#run-stamps). ### `reassess` @@ -162,8 +165,8 @@ yarn kora reassess [user-model] | Argument / Option | Description | | ------------------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `[user-model]` | Label recorded into each output JSON's `user` field — the user model that originally produced the transcripts; no model is invoked (default: `deepseek-v3.2`) | -| `--judges ` | Comma-separated judge models (default: same as `run`) | +| `[user-model]` | Override the profile's `user` role with a `models.json` slug. Only recorded into each output JSON's `user` field — the user model that originally produced the transcripts; no user model is invoked (default: from profile) | +| `--judges ` | Override the profile's `judges` role with comma-separated `models.json` slugs, odd count (default: from profile) | | `-i, --input ` | Input JSONL of recorded conversations (default: `data/reassessment-input.jsonl`) | | `-o, --output ` | Output directory — one `{modelId}.json` is written per target model (default: `data/reassessment-results`) | | `--risk-ids ` | Comma-separated risk IDs to restrict the run to (default: all records in the input file) | @@ -196,8 +199,8 @@ yarn kora continue [user-model] | Argument / Option | Description | | -------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `[user-model]` | Model used to generate user messages during the continuation (default: `deepseek-v3.2-temp-1.3`, matching production) | -| `--judges ` | Comma-separated judge models (default: `gpt-5.2:medium:limited` — single judge, held constant across 3-turn vs 8-turn comparisons) | +| `[user-model]` | Override the profile's `continueUser` role with a `models.json` slug (default: from profile; `deepseek-v3.2-temp-1.3` in `kora`, matching production) | +| `--judges ` | Override the profile's `judges` role with comma-separated `models.json` slugs, odd count (default: from profile — single judge, held constant across 3-turn vs 8-turn comparisons) | | `-i, --input ` | Input JSONL of recorded conversations, same shape as `reassess` (default: `data/reassessment-input.jsonl`) | | `-o, --output ` | Output directory — one `{modelId}.json` per target model, plus `assessments.json`, `continue-meta.json`, and `results.zip` (default: `data/continue-results`) | | `--risk-ids ` | Comma-separated risk IDs to restrict the run to (default: all records in the input file) | @@ -206,7 +209,7 @@ yarn kora continue [user-model] Each record is replayed with its **original** `modelId` as the target model, so 3-turn-vs-longer comparisons stay apples-to-apples per (scenario, model). The turn budget comes from `risk.conversationLength` in `packages/benchmark/data/risks.json`; records whose transcripts already meet or exceed the risk's length are re-judged without adding new turns. -`continue-meta.json` captures the source file path + SHA-256, the user model, the `--limit-per-risk` value, and the selected record IDs per risk — re-running the same command against the same input picks the same records. +`continue-meta.json` captures the source file path + SHA-256, the user and judge model names, the `--limit-per-risk` value, and the selected record IDs per risk — re-running the same command against the same input picks the same records. ### `compare-assessments` @@ -243,7 +246,7 @@ Output columns: `n` (records scored), `%fail` / `%adeq` / `%exem` (grade distrib ### `validate` Checks that every risk reference in an input file resolves against the active -taxonomy, and prints the active packs. The pipeline commands run this check +taxonomy, and prints the active profile and packs. The pipeline commands run this check themselves before calling any model; this exposes it on its own, which is the natural CI hook for an externally-authored scenario set. Exits non-zero on the first non-conforming file. @@ -257,7 +260,25 @@ yarn kora --taxonomy ./packs/my-taxonomy.json validate -i scenarios.jsonl | -------------------- | ---------------------------------------------------------------------------------------------- | | `-i, --input ` | JSONL file of seeds, scenarios, or reassess records (default: `data/scenarios.jsonl`) | | `--kind ` | `seeds`, `scenarios` or `reassess` (default: inferred from the first record) | -| `--packs-only` | Print the active taxonomy and behavior pack, then stop without reading the input | +| `--packs-only` | Print the active profile, taxonomy and behavior pack, then stop without reading the input | + +### `profile` + +Prints the active evaluation profile — every role with its full model +configuration, the prompts fingerprint, the packs and the code revision — and +optionally exercises each model once. This is the tool for testing a model +configuration before committing to a run. + +```bash +yarn kora profile +yarn kora --profile judge-test.local profile --check +yarn kora profile --print-hash +``` + +| Option | Description | +| -------------- | ---------------------------------------------------------------------------------------------------------------------------------------- | +| `--check` | Send a one-word prompt to every distinct model of the profile; print the served model id, latency and PASS/FAIL. Exits non-zero on any failure. Needs `AI_GATEWAY_API_KEY`. | +| `--print-hash` | Print only the profile's recomputed content hash, even when the file's `hash` is stale — paste it into the file after bumping `version` | ## Model configuration @@ -335,6 +356,101 @@ Then use the slug on the command line like any other model: yarn kora run custom-my-model ``` +## Evaluation profiles + +Every LLM the harness itself uses — not the target under test — is pinned by +an **evaluation profile**. A profile is a JSON file under `profiles/` (next to +`models.json`) that spells out the full model configuration for each pipeline +role, so the file alone is a complete record of what ran: + +```json +{ + "id": "kora", + "version": "1", + "hash": "0b7b93d2…", + "roles": { + "seeds": [{"name": "gpt-4o", "model": "openai/gpt-4o"}], + "expansion": [{"name": "gpt-5.2:high", "model": "openai/gpt-5.2", "providerOptions": {"openai": {"reasoningEffort": "high"}}}], + "expansionUser": [{"name": "deepseek-v3.2", "model": "deepseek/deepseek-v3.2", "maxTokens": 4000, "temperature": 1.3}], + "user": {"name": "deepseek-v3.2", "model": "deepseek/deepseek-v3.2", "maxTokens": 4000, "temperature": 1.3}, + "judges": [{"name": "gpt-5.2:medium:limited", "model": "openai/gpt-5.2", "maxTokens": 26000, "providerOptions": {"openai": {"reasoningEffort": "medium"}}}], + "continueUser": {"name": "deepseek-v3.2-temp-1.3", "model": "deepseek/deepseek-v3.2", "maxTokens": 4000, "temperature": 1.3} + } +} +``` + +| Role | Used by | Shape | +| --------------- | -------------------------------- | ---------------------------------------------------- | +| `seeds` | `generate-seeds` | Fallback chain (first model tried first) | +| `expansion` | `expand-scenarios` | Fallback chain; also produces the validation verdict | +| `expansionUser` | `expand-scenarios` | Fallback chain, first user message | +| `user` | `run` (and the `reassess` label) | Single model, child simulator | +| `judges` | `run`, `reassess`, `continue` | Concurrent judges, odd count | +| `continueUser` | `continue` | Single model; optional, falls back to `user` | + +Each entry is a `models.json` entry plus a `name`, which is what logs and the +`judges` / `user` fields of result files print. The bundled `profiles/kora.json` +reproduces the defaults the CLI used before profiles existed; a test asserts +every role matches the `models.json` entry of the same name. + +Select a profile with the global `--profile` option or `KORA_PROFILE`. Nothing +in `models.json` is consulted for a profile role: the registry only serves the +target model and the command-line overrides below. + +### Testing a model configuration (local profiles) + +To try a different judge, user simulator or expansion model, copy the example +into a **local profile**. Files matching `profiles/*.local.json` are gitignored, +their `hash` is not checked, and their stamp is marked `local`: + +```bash +cp profiles/example.local.json.example profiles/judge-test.local.json +# edit the judges role … +yarn kora --profile judge-test.local profile --check # one call per model +yarn kora --profile judge-test.local run gpt-4o --limit 3 -o data/judge-test/results.json +``` + +### Overrides + +The per-role arguments (`[model]`, `[user-model]`, `--judges`) still work and +resolve slugs through `models.json`, but they are **overrides**: the CLI prints +a warning, the effective profile hash changes, and the stamp lists the +overridden roles (`"overrides": ["judges"]`). Results from an overridden run are +therefore never mistaken for results from the named profile. For anything +beyond a quick experiment, prefer a local profile. + +### Committed profiles and the hash guard + +A committed profile's `hash` is the fingerprint of its content, and results are +keyed on it. `yarn test` recomputes it for every file under `profiles/` and +fails when it drifts, printing the value to paste. To change a committed +profile: edit it, bump `version`, run `yarn kora --profile profile +--print-hash`, and set `hash`. Profile ids must match their file name and +`id@version` must be unique. + +### Run stamps + +Every seed, scenario, per-test result and result file carries a `stamp` with +everything that shaped it: + +| Field | Description | +| --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `profile` | `{id, version, hash}` plus `local` and `overrides` when applicable. The hash covers the *effective* roles. | +| `models` | The resolved configuration of every role, and `target` for `run` (a model spec, or `{kind, slug}` for `kora-app-*` / `custom-*` targets) | +| `prompts` | `{version, hash}` of the prompt templates (`packages/benchmark/src/prompts/promptsFingerprint.ts`, guarded by a test the same way as profiles) | +| `code` | `@korabench/cli` version, git `commit` and `dirty` flag when run from a checkout | +| `packs` | Taxonomy and behavior pack, as in `packs` | +| `input` | Path and SHA-256 of the input corpus (`run`, `reassess`, `continue`, `expand-scenarios`) | + +Two results are comparable when their stamps hash equal, which covers +`profile`, `prompts` and `packs`; `code` and `input` are recorded but not part +of the comparison, so an unrelated commit never blocks a resume. The +graceful-restart temp directories hold a `stamp.json`, and a command refuses to +resume one written under a different stamp (delete the directory to start +over; there is no bypass flag). Result files also record `served`: the model +ids the provider reported for the user, judge and target calls, the only +evidence of which snapshot actually answered. + ## Running against real apps (web-runner / native-runner) Two custom-model adapters route to the sibling [`kora-apps`](https://github.com/korabench/apps) repo so the benchmark can target real product UIs (ChatGPT.com, TikTok's Tako, …) instead of API models. Both runners speak the same HTTP contract (`POST /sessions`, `POST /sessions/:id/turn`, `DELETE /sessions/:id`); only the underlying transport differs. @@ -636,6 +752,19 @@ The `run` command produces a result object with this structure: "taxonomy": {"id": "kora", "version": "2", "hash": "498ec8d2…"}, "behaviors": {"id": "kora", "version": "2", "hash": "b93aee04…"} }, + "stamp": { + "profile": {"id": "kora", "version": "1", "hash": "0b7b93d2…"}, + "models": {"user": {"name": "deepseek-v3.2", "model": "deepseek/deepseek-v3.2", "maxTokens": 4000, "temperature": 1.3}, "judges": ["…"], "target": {"name": "gpt-4o", "model": "openai/gpt-4o"}, "…": "…"}, + "prompts": {"version": "1", "hash": "7eacbd51…"}, + "code": {"version": "1.0.0", "commit": "b73b4731…", "dirty": false}, + "packs": {"…": "…"}, + "input": {"path": "data/scenarios.jsonl", "sha256": "eeb1a21b…"} + }, + "served": { + "user": ["deepseek/deepseek-v3.2"], + "judges": {"gpt-5.2:medium:limited": ["openai/gpt-5.2"]}, + "target": ["openai/gpt-4o"] + }, "scores": [ { "riskCategoryId": "...", @@ -663,9 +792,11 @@ The `run` command produces a result object with this structure: | Field | Description | | ----------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `target` | Target model slug | -| `judges` | Judge model slugs | -| `user` | User model slug | +| `judges` | Judge model names (from the profile, or the override slugs) | +| `user` | User model name | | `packs` | Taxonomy and behavior pack this run was produced under (id, version, content hash). Results from different packs must not be aggregated. | +| `stamp` | Full provenance: effective profile, resolved model configs, prompts fingerprint, code revision, packs, input corpus hash. See [Run stamps](#run-stamps). Results whose stamps hash differently must not be aggregated. | +| `served` | Model ids the provider reported serving, per role (sorted, deduplicated) | | `prompts` | Prompt variants that were tested | | `sums.al` | Total test count | | `sums.as` | Safety grades: `[failing, adequate, exemplary]` | @@ -694,6 +825,7 @@ All commands run with a concurrency of 10 parallel tasks. .env.example Environment variable template EVALUATION_PROCESS.md How the pipeline works internally (+ known dead code) models.json Model registry configuration +profiles/ Evaluation profiles (kora.json; *.local.json are gitignored scratch profiles) data/ Scenario pipeline output (seeds, scenarios, results) scripts/ Operator tooling (manual run completion — see scripts/README.md) packages/ @@ -701,7 +833,8 @@ packages/ data/ Bundled pack: risks.json, behaviors.json, motivations.json (see data/README.md) src/ Core benchmark logic packs/ Pack model, scoping and taxonomy conformance - prompts/ Prompt templates for each pipeline stage + stamp/ Run stamp model and scoping + prompts/ Prompt templates for each pipeline stage (+ promptsFingerprint.ts) model/ Domain types (scenario, risk, assessment, etc.) __tests__/ Test suites benchmark.ts Core benchmark interface @@ -709,6 +842,8 @@ packages/ kora.ts KORA benchmark implementation cli/src/ CLI package packs/ --taxonomy / --behaviors resolution + profiles/ --profile loading, overrides, role models + stamp/ Run stamp construction (git info, input hash) commands/ CLI command implementations __tests__/ CLI test suites models/ Model-related modules diff --git a/packages/benchmark/src/__tests__/runTest.test.ts b/packages/benchmark/src/__tests__/runTest.test.ts index f40e404..8703dd0 100644 --- a/packages/benchmark/src/__tests__/runTest.test.ts +++ b/packages/benchmark/src/__tests__/runTest.test.ts @@ -5,6 +5,8 @@ import {kora} from "../kora.js"; import {InvalidTurnError} from "../model/invalidTurnError.js"; import {Mechanism} from "../model/mechanism.js"; import {ScenarioPrompt} from "../model/scenarioKey.js"; +import {makeStamp} from "../stamp/__tests__/fixtures.js"; +import {Stamp} from "../stamp/stamp.js"; import {createScenario} from "./fixtures.js"; // @@ -76,6 +78,23 @@ describe("kora.runTest", () => { const defaultKey = keys.find(k => k.endsWith(":default"))!; const childKey = keys.find(k => k.endsWith(":child"))!; + it("omits the stamp when none is configured", async () => { + const result = await kora.runTest( + createTestContext(), + scenario, + defaultKey + ); + expect("stamp" in result).toBe(false); + }); + + it("attaches the active stamp", async () => { + const stamp = makeStamp(); + const result = await Stamp.run(stamp, () => + kora.runTest(createTestContext(), scenario, defaultKey) + ); + expect(result.stamp).toBe(stamp); + }); + it("produces a 3-turn conversation with 6 messages", async () => { const context = createTestContext(); diff --git a/packages/benchmark/src/index.ts b/packages/benchmark/src/index.ts index 40b0b8b..7822fc5 100644 --- a/packages/benchmark/src/index.ts +++ b/packages/benchmark/src/index.ts @@ -12,6 +12,7 @@ export * from "./model/invalidTurnError.js"; export * from "./model/judgeAssessment.js"; export * from "./model/mechanism.js"; export * from "./model/mechanismAssessment.js"; +export * from "./model/modelSpec.js"; export * from "./model/motivation.js"; export * from "./model/populationDistribution.js"; export * from "./model/prompt.js"; @@ -39,4 +40,7 @@ export * from "./packs/riskTaxonomy.js"; export * from "./packs/stableJson.js"; export * from "./prompts/conversationToAssessmentPrompt.js"; export * from "./prompts/conversationToMechanismAssessmentPrompt.js"; +export * from "./prompts/promptsFingerprint.js"; +export * from "./stamp/runStamp.js"; +export * from "./stamp/stamp.js"; export * from "./validateAssistantTurn.js"; diff --git a/packages/benchmark/src/kora.ts b/packages/benchmark/src/kora.ts index 322273b..a0cbdd2 100644 --- a/packages/benchmark/src/kora.ts +++ b/packages/benchmark/src/kora.ts @@ -55,6 +55,8 @@ import {conversationToNextMessagePrompt} from "./prompts/conversationToNextMessa import {riskToScenarioSeedsPrompt} from "./prompts/riskToScenarioSeedsPrompt.js"; import {scenarioToValidationPrompt} from "./prompts/scenarioToValidationPrompt.js"; import {seedToScenarioPrompt} from "./prompts/seedToScenarioPrompt.js"; +import {RunStamp} from "./stamp/runStamp.js"; +import {Stamp} from "./stamp/stamp.js"; import {validateAssistantTurn} from "./validateAssistantTurn.js"; const AGE_BANDS: Record = { @@ -71,6 +73,12 @@ function clampAgeToBand(age: number, band: AgeRange): number { return rounded; } +/** The active run stamp as a spreadable field: present only when configured. */ +function stampField(): {stamp?: RunStamp} { + const stamp = Stamp.current(); + return stamp ? {stamp} : {}; +} + /** * Run the judge-assessment step on a pre-existing transcript. * @@ -151,6 +159,7 @@ export async function runJudges( mechanismAssessment, judgeAssessments, packs: Packs.fingerprint(), + ...stampField(), }; } @@ -331,6 +340,7 @@ export const kora = Benchmark.new({ ...s, taxonomyId: taxonomy.id, taxonomyVersion: taxonomy.version, + ...stampField(), id: uuid(), riskCategoryId: riskCategory.id, riskId: risk.id, @@ -404,6 +414,7 @@ export const kora = Benchmark.new({ seed, firstUserMessage: "", ...modelScenario, + ...stampField(), }; const validationPrompt = scenarioToValidationPrompt( diff --git a/packages/benchmark/src/model/modelSpec.ts b/packages/benchmark/src/model/modelSpec.ts new file mode 100644 index 0000000..38c6214 --- /dev/null +++ b/packages/benchmark/src/model/modelSpec.ts @@ -0,0 +1,51 @@ +import * as R from "remeda"; +import * as v from "valibot"; + +// +// Runtime model. +// + +/** + * A fully resolved LLM configuration plus a display `name`. + * + * `name` is what the CLI prints and what result headers persist in their + * `judges` / `user` fields (historically a `models.json` slug). Everything + * else is the configuration the gateway actually uses, so a spec is + * self-describing: no registry lookup is needed to know what ran. + */ +const VModelSpec = v.object({ + name: v.pipe(v.string(), v.minLength(1)), + model: v.string(), + maxTokens: v.optional(v.number()), + temperature: v.optional(v.number()), + providerOptions: v.optional( + v.record(v.string(), v.record(v.string(), v.unknown())) + ), +}); + +// +// API. +// + +function config(spec: ModelSpec): ModelConfig { + return R.omit(spec, ["name"]); +} + +function fromConfig(name: string, config: ModelConfig): ModelSpec { + return {name, ...config}; +} + +// +// Exports. +// + +export interface ModelSpec extends v.InferOutput {} + +/** A `ModelSpec` without its display name: the gateway-facing part. */ +export type ModelConfig = Omit; + +export const ModelSpec = { + io: VModelSpec, + config, + fromConfig, +}; diff --git a/packages/benchmark/src/model/scenario.ts b/packages/benchmark/src/model/scenario.ts index d978f69..a88836f 100644 --- a/packages/benchmark/src/model/scenario.ts +++ b/packages/benchmark/src/model/scenario.ts @@ -1,4 +1,5 @@ import * as v from "valibot"; +import {RunStamp} from "../stamp/runStamp.js"; import {ScenarioKey} from "./scenarioKey.js"; import {ScenarioPrompt} from "./scenarioPrompt.js"; import {ScenarioSeed} from "./scenarioSeed.js"; @@ -79,6 +80,8 @@ const VScenario = v.strictObject({ ...VModelScenario.entries, seed: ScenarioSeed.io, firstUserMessage: v.string(), + /** Provenance of the expanding run. Optional: older corpora carry none. */ + stamp: v.optional(RunStamp.io), }); // diff --git a/packages/benchmark/src/model/scenarioSeed.ts b/packages/benchmark/src/model/scenarioSeed.ts index 4f13ff7..2df651a 100644 --- a/packages/benchmark/src/model/scenarioSeed.ts +++ b/packages/benchmark/src/model/scenarioSeed.ts @@ -1,4 +1,5 @@ import * as v from "valibot"; +import {RunStamp} from "../stamp/runStamp.js"; import {AgeRange} from "./ageRange.js"; import {Motivation} from "./motivation.js"; @@ -253,6 +254,8 @@ const VScenarioSeed = v.strictObject({ */ taxonomyId: v.optional(v.string()), taxonomyVersion: v.optional(v.string()), + /** Full provenance of the generating run. Optional for the same reason. */ + stamp: v.optional(RunStamp.io), id: v.string(), riskCategoryId: v.string(), riskId: v.string(), diff --git a/packages/benchmark/src/model/testResult.ts b/packages/benchmark/src/model/testResult.ts index 56a0f70..f91c6a7 100644 --- a/packages/benchmark/src/model/testResult.ts +++ b/packages/benchmark/src/model/testResult.ts @@ -1,6 +1,7 @@ import {ModelMessage} from "@korabench/core"; import * as v from "valibot"; import {PackStamp} from "../packs/packStamp.js"; +import {RunStamp} from "../stamp/runStamp.js"; import {JudgeAssessment} from "./judgeAssessment.js"; import {MechanismAssessment} from "./mechanismAssessment.js"; import {Scenario} from "./scenario.js"; @@ -23,8 +24,9 @@ function buildTestResultSchema(): TestResultSchema { assessment: TestAssessment.io, mechanismAssessment: MechanismAssessment.io, judgeAssessments: v.array(JudgeAssessment.io), - // Optional: results written before packs existed carry no stamp. + // Optional: results written before packs / stamps existed carry none. packs: v.optional(PackStamp.io), + stamp: v.optional(RunStamp.io), }) as unknown as TestResultSchema; } @@ -40,6 +42,7 @@ export interface TestResult { mechanismAssessment: MechanismAssessment; judgeAssessments: JudgeAssessment[]; packs?: PackStamp; + stamp?: RunStamp; } type TestResultSchema = v.GenericSchema< diff --git a/packages/benchmark/src/prompts/__tests__/promptsFingerprint.test.ts b/packages/benchmark/src/prompts/__tests__/promptsFingerprint.test.ts new file mode 100644 index 0000000..7836319 --- /dev/null +++ b/packages/benchmark/src/prompts/__tests__/promptsFingerprint.test.ts @@ -0,0 +1,68 @@ +import {Hash} from "@korabench/core"; +import {readdirSync, readFileSync} from "node:fs"; +import {fileURLToPath} from "node:url"; +import {describe, expect, it} from "vitest"; +import {stableJson} from "../../packs/stableJson.js"; +import { + DEAD_PROMPT_FILES, + PROMPT_SOURCE_FILES, + Prompts, + PROMPTS_FINGERPRINT, +} from "../promptsFingerprint.js"; + +// +// CI guard for the prompts fingerprint. +// +// Results are only comparable across runs that used the same prompt +// templates, and the fingerprint is how a run records which ones it used. A +// template edit without a fingerprint bump would make new runs look +// comparable to old ones, so this test refuses that state and prints the +// value to paste. +// + +const promptsDir = fileURLToPath(new URL("../", import.meta.url)); + +function readSource(file: string): string { + return readFileSync(new URL(`../${file}`, import.meta.url), "utf-8").replace( + /\r\n/g, + "\n" + ); +} + +export function computePromptsHash(): string { + return Hash.shortHash( + stableJson(PROMPT_SOURCE_FILES.map(file => [file, readSource(file)])) + ); +} + +describe("PROMPTS_FINGERPRINT", () => { + it("matches the live prompt sources", () => { + const expected = computePromptsHash(); + expect( + PROMPTS_FINGERPRINT.hash, + `Prompt templates changed: bump "version" and set "hash" to "${expected}" in prompts/promptsFingerprint.ts` + ).toBe(expected); + }); + + it("is what Prompts.fingerprint() returns", () => { + expect(Prompts.fingerprint()).toBe(PROMPTS_FINGERPRINT); + }); + + it("accounts for every prompt source file", () => { + const files = readdirSync(promptsDir) + .filter(file => file.endsWith(".ts")) + .filter(file => file !== "promptsFingerprint.ts") + .sort(); + const listed = [...PROMPT_SOURCE_FILES, ...DEAD_PROMPT_FILES].sort(); + expect(files).toEqual(listed); + }); + + it("changes with the source text", () => { + const tampered = Hash.shortHash( + stableJson( + PROMPT_SOURCE_FILES.map(file => [file, readSource(file) + "\n// x"]) + ) + ); + expect(tampered).not.toBe(computePromptsHash()); + }); +}); diff --git a/packages/benchmark/src/prompts/promptsFingerprint.ts b/packages/benchmark/src/prompts/promptsFingerprint.ts new file mode 100644 index 0000000..7545ae6 --- /dev/null +++ b/packages/benchmark/src/prompts/promptsFingerprint.ts @@ -0,0 +1,49 @@ +// +// Prompt template fingerprint. +// +// Every prompt is a function of runtime data, so no single rendered string +// identifies "the templates". A hash of the function bodies would miss the +// module-private helpers and description tables they pull in, and would differ +// between a tsc build and a bundled one. So the fingerprint is a checked-in +// constant over the *source files*, guarded by a test that recomputes it and +// prints the expected value when a template changes. +// + +/** Live prompt sources, relative to this directory. Dead files are excluded. */ +export const PROMPT_SOURCE_FILES = [ + "conversationToAssessmentPrompt.ts", + "conversationToMechanismAssessmentPrompt.ts", + "conversationToNextMessagePrompt.ts", + "formatConversation.ts", + "riskToScenarioSeedsPrompt.ts", + "scenarioToFirstUserMessagePrompt.ts", + "scenarioToNextUserMessagePrompt.ts", + "scenarioToValidationPrompt.ts", + "seedToScenarioPrompt.ts", +] as const; + +/** Prompt files kept for reference only; not part of the fingerprint. */ +export const DEAD_PROMPT_FILES = [ + "conversationToMatchPrompt.ts", + "riskToScenariosPrompt.ts", +] as const; + +export interface PromptsFingerprint { + /** Bumped by hand alongside `hash` when a template changes. */ + version: string; + /** `Hash.shortHash(stableJson([[file, content], ...]))` over the live files. */ + hash: string; +} + +export const PROMPTS_FINGERPRINT: PromptsFingerprint = { + version: "1", + hash: "7eacbd51e6a40043ffb9bbd18040ad7a", +}; + +// +// Exports. +// + +export const Prompts = { + fingerprint: (): PromptsFingerprint => PROMPTS_FINGERPRINT, +}; diff --git a/packages/benchmark/src/stamp/__tests__/fixtures.ts b/packages/benchmark/src/stamp/__tests__/fixtures.ts new file mode 100644 index 0000000..5a93229 --- /dev/null +++ b/packages/benchmark/src/stamp/__tests__/fixtures.ts @@ -0,0 +1,33 @@ +import {ModelSpec} from "../../model/modelSpec.js"; +import {Packs} from "../../packs/packs.js"; +import {RunStamp} from "../runStamp.js"; + +// +// Test fixtures. +// + +export function makeSpec( + name: string, + overrides: Partial = {} +): ModelSpec { + return {name, model: `provider/${name}`, ...overrides}; +} + +/** A complete stamp under the bundled packs. Override any field. */ +export function makeStamp(overrides: Partial = {}): RunStamp { + return { + profile: {id: "test", version: "1", hash: "profile-hash"}, + models: { + seeds: [makeSpec("seed")], + expansion: [makeSpec("expand")], + expansionUser: [makeSpec("user")], + user: makeSpec("user"), + judges: [makeSpec("judge")], + continueUser: makeSpec("user"), + }, + prompts: {version: "1", hash: "prompts-hash"}, + code: {version: "1.0.0", commit: "abc", dirty: false}, + packs: Packs.fingerprint(), + ...overrides, + }; +} diff --git a/packages/benchmark/src/stamp/__tests__/stamp.test.ts b/packages/benchmark/src/stamp/__tests__/stamp.test.ts new file mode 100644 index 0000000..c3726d6 --- /dev/null +++ b/packages/benchmark/src/stamp/__tests__/stamp.test.ts @@ -0,0 +1,148 @@ +import {readdirSync, readFileSync} from "node:fs"; +import * as v from "valibot"; +import {afterEach, describe, expect, it} from "vitest"; +import {RunStamp} from "../runStamp.js"; +import {Stamp} from "../stamp.js"; +import {makeSpec, makeStamp} from "./fixtures.js"; + +afterEach(() => Stamp.reset()); + +describe("RunStamp.hash", () => { + it("is stable for equal content", () => { + expect(RunStamp.hash(makeStamp())).toBe(RunStamp.hash(makeStamp())); + }); + + it.each([ + ["profile", {profile: {id: "test", version: "1", hash: "other"}}], + ["prompts", {prompts: {version: "2", hash: "other"}}], + [ + "packs", + { + packs: { + taxonomy: {id: "x", version: "1", hash: "other"}, + behaviors: makeStamp().packs.behaviors, + }, + }, + ], + ] as const)("changes with %s", (_what, overrides) => { + expect(RunStamp.hash(makeStamp(overrides))).not.toBe( + RunStamp.hash(makeStamp()) + ); + }); + + it.each([ + ["code", {code: {version: "9.9.9", commit: "zzz", dirty: true}}], + ["input", {input: {path: "x.jsonl", sha256: "deadbeef"}}], + ["target", {models: {...makeStamp().models, target: makeSpec("t")}}], + ] as const)("ignores %s", (_what, overrides) => { + expect(RunStamp.equals(makeStamp(overrides), makeStamp())).toBe(true); + }); +}); + +describe("RunStamp.io", () => { + it("round-trips a full stamp with a runner target", () => { + const stamp = makeStamp({ + profile: { + id: "kora", + version: "1", + hash: "h", + local: true, + overrides: ["judges"], + }, + models: { + ...makeStamp().models, + target: {kind: "web-runner", slug: "kora-app-x"}, + }, + input: {path: "in.jsonl", sha256: "00"}, + }); + expect(v.parse(RunStamp.io, JSON.parse(JSON.stringify(stamp)))).toEqual( + stamp + ); + }); + + it("tolerates unknown fields from newer writers", () => { + const stamp = {...makeStamp(), future: true}; + expect(() => v.parse(RunStamp.io, stamp)).not.toThrow(); + }); +}); + +describe("RunStamp.describe", () => { + it("names profile, prompts and packs with markers", () => { + const text = RunStamp.describe( + makeStamp({ + profile: {id: "kora", version: "1", hash: "h", overrides: ["user"]}, + }) + ); + expect(text).toMatch( + /^profile kora@1 \(h\) \[overrides: user\] \| prompts 1 \(prompts-hash\) \| packs kora@2/ + ); + }); +}); + +describe("Stamp", () => { + it("is undefined until configured", () => { + expect(Stamp.current()).toBeUndefined(); + }); + + it("configure makes the stamp process-wide", () => { + const stamp = makeStamp(); + Stamp.configure(stamp); + expect(Stamp.current()).toBe(stamp); + }); + + it("configure is idempotent for an equal stamp", () => { + Stamp.configure(makeStamp()); + expect(() => + Stamp.configure(makeStamp({code: {version: "2"}})) + ).not.toThrow(); + }); + + it("configure refuses a different stamp", () => { + Stamp.configure(makeStamp()); + expect(() => + Stamp.configure(makeStamp({prompts: {version: "2", hash: "x"}})) + ).toThrow(/called twice with different stamps/); + }); + + it("run scopes a stamp and wins over configure", async () => { + const outer = makeStamp(); + const inner = makeStamp({prompts: {version: "2", hash: "inner"}}); + Stamp.configure(outer); + await Stamp.run(inner, async () => { + await Promise.resolve(); + expect(Stamp.current()).toBe(inner); + }); + expect(Stamp.current()).toBe(outer); + }); + + it("run stays isolated across concurrent tasks", async () => { + const a = makeStamp({prompts: {version: "a", hash: "a"}}); + const b = makeStamp({prompts: {version: "b", hash: "b"}}); + const seen = await Promise.all([ + Stamp.run(a, async () => { + await new Promise(r => setTimeout(r, 5)); + return Stamp.current()?.prompts.hash; + }), + Stamp.run(b, async () => { + await new Promise(r => setTimeout(r, 1)); + return Stamp.current()?.prompts.hash; + }), + ]); + expect(seen).toEqual(["a", "b"]); + }); +}); + +// The stamp module is reached from every persisted schema, and those reach the +// browser through the package barrel. Keep it free of node builtins, like +// packs.ts (see packScope.test.ts). +describe("stamp/*.ts", () => { + it("import no node builtin", () => { + const dir = new URL("../", import.meta.url); + readdirSync(dir) + .filter(file => file.endsWith(".ts")) + .forEach(file => { + const source = readFileSync(new URL(file, dir), "utf8"); + expect(source, file).not.toMatch(/from\s+"node:/); + }); + }); +}); diff --git a/packages/benchmark/src/stamp/runStamp.ts b/packages/benchmark/src/stamp/runStamp.ts new file mode 100644 index 0000000..308bb7e --- /dev/null +++ b/packages/benchmark/src/stamp/runStamp.ts @@ -0,0 +1,136 @@ +import {Hash} from "@korabench/core"; +import * as v from "valibot"; +import {ModelSpec} from "../model/modelSpec.js"; +import {PackStamp} from "../packs/packStamp.js"; + +// +// Runtime model. +// +// Provenance for everything that shaped a record: which evaluation profile +// (resolved model config per role), which prompt templates, which packs, +// which code revision, which input corpus. Written next to every persisted +// record and result header. Always optional on persisted shapes: records +// written before stamps existed carry none and must keep parsing. +// +// Schemas here are non-strict on purpose: a newer stamp nested inside a strict +// `TestResult` must still parse under older code. +// + +const VProfileRef = v.object({ + id: v.string(), + version: v.string(), + /** Hash of the *effective* roles; differs from the file's when overridden. */ + hash: v.string(), + /** Loaded from an uncommitted `*.local.json` file. */ + local: v.optional(v.boolean()), + /** Roles replaced on the command line. */ + overrides: v.optional(v.array(v.string())), +}); + +const VRunnerTarget = v.object({ + kind: v.picklist(["web-runner", "native-runner", "custom"]), + slug: v.string(), +}); + +const VTargetRef = v.union([ModelSpec.io, VRunnerTarget]); + +const VChain = v.array(ModelSpec.io); + +const VStampModels = v.object({ + seeds: VChain, + expansion: VChain, + expansionUser: VChain, + user: ModelSpec.io, + judges: VChain, + continueUser: ModelSpec.io, + /** The evaluated model; only meaningful for `run`. */ + target: v.optional(VTargetRef), +}); + +const VPromptsRef = v.object({ + version: v.string(), + hash: v.string(), +}); + +const VCodeRef = v.object({ + /** `@korabench/cli` package version. */ + version: v.string(), + commit: v.optional(v.string()), + dirty: v.optional(v.boolean()), +}); + +const VInputRef = v.object({ + path: v.string(), + sha256: v.string(), +}); + +const VRunStamp = v.object({ + profile: VProfileRef, + models: VStampModels, + prompts: VPromptsRef, + code: VCodeRef, + packs: PackStamp.io, + input: v.optional(VInputRef), +}); + +// +// API. +// + +/** + * Comparability key. Two records are comparable when their profile, prompt + * templates and packs match. Code revision and input corpus are recorded but + * excluded: an unrelated commit must not refuse a resume, and prompt changes + * are caught by `prompts.hash`. + */ +function hash(stamp: RunStamp): string { + return Hash.shortHash( + [ + stamp.profile.hash, + stamp.prompts.hash, + stamp.packs.taxonomy.hash, + stamp.packs.behaviors.hash, + ].join("|") + ); +} + +function equals(a: RunStamp, b: RunStamp): boolean { + return hash(a) === hash(b); +} + +function describeProfile(ref: ProfileRef): string { + const markers = [ + ref.local ? "local" : undefined, + ref.overrides?.length ? `overrides: ${ref.overrides.join(",")}` : undefined, + ].filter(marker => marker !== undefined); + const suffix = markers.length > 0 ? ` [${markers.join("; ")}]` : ""; + return `${ref.id}@${ref.version} (${ref.hash})${suffix}`; +} + +/** One line, for error messages and logs. */ +function describe(stamp: RunStamp): string { + const {taxonomy, behaviors} = stamp.packs; + return ( + `profile ${describeProfile(stamp.profile)} | ` + + `prompts ${stamp.prompts.version} (${stamp.prompts.hash}) | ` + + `packs ${taxonomy.id}@${taxonomy.version} (${taxonomy.hash}) / ` + + `${behaviors.id}@${behaviors.version} (${behaviors.hash})` + ); +} + +// +// Exports. +// + +export interface ProfileRef extends v.InferOutput {} +export type TargetRef = v.InferOutput; +export interface StampModels extends v.InferOutput {} +export interface RunStamp extends v.InferOutput {} + +export const RunStamp = { + io: VRunStamp, + hash, + equals, + describe, + describeProfile, +}; diff --git a/packages/benchmark/src/stamp/stamp.ts b/packages/benchmark/src/stamp/stamp.ts new file mode 100644 index 0000000..2803005 --- /dev/null +++ b/packages/benchmark/src/stamp/stamp.ts @@ -0,0 +1,67 @@ +import {createPackScope} from "#packScope"; +import {RunStamp} from "./runStamp.js"; + +// +// State. +// +// Same two layers as `Packs`, for the same reasons: +// +// - `configured` is process-wide and one-shot. Each CLI command builds its +// stamp once and every record it writes reads it. +// - `storage` is an async-context scope. kora-infra serves several runs +// concurrently from a single isolate, so each run wraps its work in +// `Stamp.run(...)`, exactly as it does with `Packs.run(...)`. +// +// Unlike packs there is no bundled default: with nothing configured, +// `current()` is `undefined` and records simply carry no stamp. +// + +const storage = createPackScope(); + +let configured: RunStamp | undefined; + +// +// API. +// + +function current(): RunStamp | undefined { + return storage.getStore() ?? configured; +} + +/** + * Set the process-wide stamp. Idempotent for an equal stamp; a second call + * with a different one throws rather than silently re-labelling records that + * may already have been written under the first. + */ +function configure(stamp: RunStamp): void { + if (configured) { + if (RunStamp.equals(configured, stamp)) return; + throw new Error( + "Stamp.configure() called twice with different stamps " + + `(${RunStamp.describe(configured)} then ${RunStamp.describe(stamp)}). ` + + "Use Stamp.run() to scope different stamps to different work." + ); + } + configured = stamp; +} + +/** Run `fn` with `stamp` active for its whole async context. */ +function run(stamp: RunStamp, fn: () => T): T { + return storage.run(stamp, fn); +} + +/** Test-only: drop the process-wide configuration. */ +function reset(): void { + configured = undefined; +} + +// +// Exports. +// + +export const Stamp = { + configure, + run, + current, + reset, +}; diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index c582df2..43d2591 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -5,21 +5,23 @@ import { PopulationDistribution, ScenarioPrompt, } from "@korabench/benchmark"; -import {existsSync, readFileSync} from "node:fs"; +import {existsSync} from "node:fs"; import * as path from "node:path"; -import {dirname} from "node:path"; -import {fileURLToPath} from "node:url"; import * as v from "valibot"; import {compareAssessmentsCommand} from "./commands/compareAssessmentsCommand.js"; import {continueCommand} from "./commands/continueCommand.js"; import {expandScenariosCommand} from "./commands/expandScenariosCommand.js"; import {generateSeeds} from "./commands/generateSeedsCommand.js"; +import {profileCommand} from "./commands/profileCommand.js"; import {reassessCommand} from "./commands/reassessCommand.js"; import {runCommand} from "./commands/runCommand.js"; import {InputKind} from "./commands/shared/validateInputFile.js"; import {statsCommand} from "./commands/statsCommand.js"; import {validateCommand} from "./commands/validateCommand.js"; import {configurePacks} from "./packs/loadPack.js"; +import {loadProfile, profilesDir} from "./profiles/loadProfile.js"; +import {Profiles} from "./profiles/profiles.js"; +import {readPackageVersion} from "./shared/packageVersion.js"; function findConfigFile(filename: string): string { let dir = process.cwd(); @@ -51,13 +53,9 @@ function splitCsv(value: string): readonly string[] { return parts; } -function readPackageVersion(): string { - const pkgPath = path.join( - dirname(fileURLToPath(import.meta.url)), - "../../package.json" - ); - const pkg = JSON.parse(readFileSync(pkgPath, "utf-8")); - return pkg.version || "0.0.0"; +/** `splitCsv` for optional role arguments: absent means "from profile". */ +function optionalCsv(value: string | undefined): readonly string[] | undefined { + return value === undefined ? undefined : splitCsv(value); } const modelsJsonPath = findConfigFile("models.json"); @@ -118,17 +116,29 @@ const program = new Command() "--behaviors ", 'behavior (mechanism) pack: a registered name ("kora") or a path to a JSON file holding a full {id, version, behaviors} set', process.env.KORA_BEHAVIORS + ) + .option( + "--profile ", + 'evaluation profile pinning the model for every role: a name under profiles/ ("kora"), a local scratch profile (".local"), or a path to a JSON file', + process.env.KORA_PROFILE ?? "kora" ); -// Packs must be resolved before any command body runs, but AFTER the module -// graph is fully loaded — `cli.ts` statically imports every command, so nothing -// may read a pack-dependent schema at module scope. See `benchmark.ts`. -program.hook("preAction", () => +// Packs and the profile must be resolved before any command body runs, but +// AFTER the module graph is fully loaded — `cli.ts` statically imports every +// command, so nothing may read a pack-dependent schema at module scope. See +// `benchmark.ts`. The `profile` command loads the profile itself so that +// `--print-hash` can inspect a file whose hash is stale. +program.hook("preAction", (_thisCommand, actionCommand) => { configurePacks({ taxonomy: program.opts().taxonomy, behaviors: program.opts().behaviors, - }) -); + }); + if (actionCommand.name() !== "profile") { + Profiles.configure( + loadProfile(program.opts().profile, profilesDir(modelsJsonPath)) + ); + } +}); export type Program = typeof program; @@ -137,8 +147,7 @@ program .description("generate a new set of scenario seeds") .argument( "[model]", - "model(s) to use for seed generation; comma-separated for per-task fallback chain (e.g. gpt-4o,gpt-5.5:low)", - "gpt-4o" + "override the profile's seeds role with models.json slug(s); comma-separated for per-task fallback chain (e.g. gpt-4o,gpt-5.5:low)" ) .option("-o, --output ", "output seeds JSONL file", defaultSeedsPath) .option( @@ -185,7 +194,7 @@ program return generateSeeds( program, modelsJsonPath, - splitCsv(model), + {seeds: optionalCsv(model)}, opts.output, { seedsPerTask: @@ -218,13 +227,11 @@ program .description("transform the seeds into fully fleshed out scenarios") .argument( "[model]", - "model(s) for seed expansion; comma-separated for per-task fallback chain", - "gpt-5.2:high" + "override the profile's expansion role with models.json slug(s); comma-separated for per-task fallback chain" ) .argument( "[user-model]", - "model(s) for user message generation; comma-separated for per-task fallback chain", - "deepseek-v3.2" + "override the profile's expansionUser role with models.json slug(s); comma-separated for per-task fallback chain" ) .option("-i, --input ", "input seeds JSONL file", defaultSeedsPath) .option( @@ -240,8 +247,7 @@ program expandScenariosCommand( program, modelsJsonPath, - splitCsv(model), - splitCsv(userModel), + {expansion: optionalCsv(model), expansionUser: optionalCsv(userModel)}, opts.input, opts.output, opts.riskIds @@ -257,13 +263,11 @@ program .argument("", "model to benchmark") .argument( "[user-model]", - "model to use for user message generation", - "deepseek-v3.2" + "override the profile's user role with a models.json slug" ) .option( "--judges ", - "comma-separated judge models", - "gpt-5.2:medium:limited" + "override the profile's judges role with comma-separated models.json slugs (odd count)" ) .option( "-i, --input ", @@ -323,8 +327,7 @@ program program, modelsJsonPath, targetModel, - opts.judges.split(",").map(s => s.trim()), - userModel, + {judges: optionalCsv(opts.judges), user: optionalCsv(userModel)}, opts.input, opts.output, opts.prompts.split(",").map(p => v.parse(ScenarioPrompt.io, p.trim())), @@ -348,13 +351,11 @@ program ) .argument( "[user-model]", - "label recorded into each output JSON's `user` field (the user model that originally produced the transcripts; no model is invoked)", - "deepseek-v3.2" + "override the profile's user role with a models.json slug; only recorded into each output JSON's `user` field (no user model is invoked)" ) .option( "--judges ", - "comma-separated judge models", - "gpt-5.2:medium:limited" + "override the profile's judges role with comma-separated models.json slugs (odd count)" ) .option( "-i, --input ", @@ -390,8 +391,7 @@ program return reassessCommand( program, modelsJsonPath, - opts.judges.split(",").map(s => s.trim()), - userModel, + {judges: optionalCsv(opts.judges), user: optionalCsv(userModel)}, opts.input, opts.output, { @@ -415,13 +415,11 @@ program ) .argument( "[user-model]", - "model to use for user message generation during the continuation", - "deepseek-v3.2-temp-1.3" + "override the profile's continueUser role with a models.json slug" ) .option( "--judges ", - "comma-separated judge models", - "gpt-5.2:medium:limited" + "override the profile's judges role with comma-separated models.json slugs (odd count)" ) .option( "-i, --input ", @@ -462,8 +460,7 @@ program return continueCommand( program, modelsJsonPath, - opts.judges.split(",").map(s => s.trim()), - userModel, + {judges: optionalCsv(opts.judges), continueUser: optionalCsv(userModel)}, opts.input, opts.output, { @@ -543,7 +540,7 @@ program ) .option( "--packs-only", - "print the active taxonomy and behavior pack, then stop without reading the input" + "print the active profile, taxonomy and behavior pack, then stop without reading the input" ) .action(opts => { const kind = opts.kind as InputKind | undefined; @@ -552,10 +549,30 @@ program `--kind must be one of: seeds, scenarios, reassess (got: ${opts.kind})` ); } - return validateCommand(program, opts.input, { + return validateCommand(program, modelsJsonPath, opts.input, { kind, packsOnly: opts.packsOnly === true, }); }); +program + .command("profile") + .description( + "print the active evaluation profile (every role with its full model config, prompts fingerprint, packs, code revision)" + ) + .option( + "--check", + "send a one-word prompt to every model in the profile and report the served model id, latency and pass/fail (needs AI_GATEWAY_API_KEY)" + ) + .option( + "--print-hash", + "print only the profile's recomputed content hash (paste it into the file after bumping its version)" + ) + .action(opts => + profileCommand(program, modelsJsonPath, program.opts().profile, { + check: opts.check === true, + printHash: opts.printHash === true, + }) + ); + program.parseAsync(); diff --git a/packages/cli/src/commands/continueCommand.ts b/packages/cli/src/commands/continueCommand.ts index 12f28cd..94a731c 100644 --- a/packages/cli/src/commands/continueCommand.ts +++ b/packages/cli/src/commands/continueCommand.ts @@ -4,29 +4,42 @@ import { RiskTaxonomy, ScenarioKey, ScenarioPrompt, + Stamp, TestResult, } from "@korabench/benchmark"; import {Script} from "@korabench/core"; import archiver from "archiver"; -import {createHash} from "node:crypto"; import {createWriteStream} from "node:fs"; import * as fs from "node:fs/promises"; import * as path from "node:path"; import {flatTransform, pipeline, reduce} from "streaming-iterables"; import * as v from "valibot"; import {Program} from "../cli.js"; -import {createGatewayModel} from "../models/gatewayModel.js"; -import {Model} from "../models/model.js"; +import {GatewayModel} from "../models/gatewayModel.js"; +import { + describeProfileRef, + resolveEffectiveProfile, + RoleOverrides, +} from "../profiles/effectiveProfile.js"; +import { + collectServed, + createJudgeModels, + createSpecModel, +} from "../profiles/roleModels.js"; +import {sha256File} from "../shared/sha256File.js"; +import {buildRunStamp} from "../stamp/buildRunStamp.js"; import { buildContext, BuiltContext, resolveTargetGatewayModel, } from "./shared/buildContext.js"; +import {assertResumable} from "./shared/cacheStamp.js"; import { readReassessInputsFromJsonl, ReassessInput, } from "./shared/reassessInput.js"; import {reportInvalidTurn} from "./shared/reportInvalidTurn.js"; +import {buildResultHeader} from "./shared/resultHeader.js"; import {resolveRiskIdFilter} from "./shared/riskFilters.js"; import {assertInputConforms} from "./shared/validateInputFile.js"; @@ -73,11 +86,6 @@ interface SelectionMeta { completedAt?: string; } -async function sha256File(filePath: string): Promise { - const buf = await fs.readFile(filePath); - return createHash("sha256").update(buf).digest("hex"); -} - async function archiveResults( sourceDir: string, files: readonly string[], @@ -108,25 +116,30 @@ export interface ContinueCommandOptions { export async function continueCommand( _program: Program, modelsJsonPath: string, - judgeModelSlugs: readonly string[], - userModelSlug: string, + overrides: RoleOverrides, inputFilePath: string, outputDirPath: string, options: ContinueCommandOptions = {} ) { + const effective = resolveEffectiveProfile(modelsJsonPath, overrides); + const {roles} = effective; + const judgeModelSlugs = roles.judges.map(spec => spec.name); + const userModelSlug = roles.continueUser.name; + console.log(`Profile: ${describeProfileRef(effective.ref)}`); console.log( `Continuing transcripts: judges=${judgeModelSlugs.join(",")}, user=${userModelSlug}` ); - if (judgeModelSlugs.length % 2 === 0) - throw new Error( - "The current implementation only supports odd numbers of judges. This ensures that the median assessment is always defined. See `aggregateTestAssessments` for reference." - ); - const recordCount = await assertInputConforms(inputFilePath, "reassess"); console.log( `Validated ${recordCount} record(s) against taxonomy "${RiskTaxonomy.label(Packs.current().taxonomy)}".` ); + const stamp = await buildRunStamp({ + effective, + modelsJsonPath, + inputPath: inputFilePath, + }); + Stamp.configure(stamp); const riskIdsFilter = resolveRiskIdFilter(options.riskIds); const targetModelsFilter = options.targetModels?.length @@ -194,17 +207,12 @@ export async function continueCommand( selectedRecords.push(...picked); } - const judgeModels: Record = Object.fromEntries( - judgeModelSlugs.map(slug => [ - slug, - createGatewayModel(modelsJsonPath, slug), - ]) - ); - const userModel = createGatewayModel(modelsJsonPath, userModelSlug); + const judgeModels = createJudgeModels(roles.judges); + const userModel = createSpecModel(roles.continueUser); // Per-record target model resolution: cache by modelId across records. - const targetGatewayCache = new Map(); - const getTargetGateway = (modelId: string): Model | undefined => { + const targetGatewayCache = new Map(); + const getTargetGateway = (modelId: string): GatewayModel | undefined => { if (!targetGatewayCache.has(modelId)) { targetGatewayCache.set( modelId, @@ -217,6 +225,7 @@ export async function continueCommand( const tempDir = path.join(outputDirPath, ".kora-continue-tmp"); await fs.mkdir(outputDirPath, {recursive: true}); await fs.mkdir(tempDir, {recursive: true}); + await assertResumable(tempDir, stamp); const meta: SelectionMeta = { sourceInputPath: inputFilePath, @@ -372,11 +381,18 @@ export async function continueCommand( for (const [modelId, runResult] of runResultsByTarget) { const prompts = [...(promptsByTarget.get(modelId) ?? new Set())]; const result = { - target: modelId, - judges: judgeModelSlugs, - packs: Packs.fingerprint(), - user: userModelSlug, - prompts, + ...buildResultHeader({ + target: modelId, + effective, + prompts, + stamp, + userName: roles.continueUser.name, + served: collectServed({ + user: userModel, + judges: judgeModels, + target: targetGatewayCache.get(modelId), + }), + }), ...runResult, }; const filePath = path.join(outputDirPath, `${modelId}.json`); diff --git a/packages/cli/src/commands/expandScenariosCommand.ts b/packages/cli/src/commands/expandScenariosCommand.ts index 28e4c61..abb352d 100644 --- a/packages/cli/src/commands/expandScenariosCommand.ts +++ b/packages/cli/src/commands/expandScenariosCommand.ts @@ -6,6 +6,7 @@ import { Scenario, ScenarioSeed, ScenarioValidationError, + Stamp, } from "@korabench/benchmark"; import {Script} from "@korabench/core"; import * as fs from "node:fs/promises"; @@ -15,9 +16,21 @@ import {consume, flatTransform} from "streaming-iterables"; import * as v from "valibot"; import {Program} from "../cli.js"; import { - createGatewayModel, - createGatewayModelChain, -} from "../models/gatewayModel.js"; + describeProfileRef, + resolveEffectiveProfile, + RoleOverrides, +} from "../profiles/effectiveProfile.js"; +import { + chainLabel, + createChainModel, + createSpecModel, +} from "../profiles/roleModels.js"; +import {buildRunStamp} from "../stamp/buildRunStamp.js"; +import { + assertResumable, + hasCachedFiles, + listCachedFiles, +} from "./shared/cacheStamp.js"; import {resolveRiskIdFilter} from "./shared/riskFilters.js"; import {assertInputConforms} from "./shared/validateInputFile.js"; @@ -48,34 +61,31 @@ async function countSeeds( return count; } -async function hasTempFiles(tempDir: string): Promise { - try { - const files = await fs.readdir(tempDir); - return files.length > 0; - } catch { - return false; - } -} - export async function expandScenariosCommand( _program: Program, modelsJsonPath: string, - modelSlugs: readonly string[], - userModelSlugs: readonly string[], + overrides: RoleOverrides, seedsFilePath: string, outputFilePath: string, riskIds?: readonly string[] ) { - const fmtChain = (slugs: readonly string[]) => - slugs.length === 1 ? slugs[0] : slugs.join(" → "); + const effective = resolveEffectiveProfile(modelsJsonPath, overrides); + const {roles} = effective; + console.log(`Profile: ${describeProfileRef(effective.ref)}`); console.log( - `Expanding scenarios using ${fmtChain(modelSlugs)} (user: ${fmtChain(userModelSlugs)})...` + `Expanding scenarios using ${chainLabel(roles.expansion)} (user: ${chainLabel(roles.expansionUser)})...` ); const riskIdFilter = resolveRiskIdFilter(riskIds); const seedCount = await assertInputConforms(seedsFilePath, "seeds"); console.log( `Validated ${seedCount} seed(s) against taxonomy "${RiskTaxonomy.label(Packs.current().taxonomy)}".` ); + const stamp = await buildRunStamp({ + effective, + modelsJsonPath, + inputPath: seedsFilePath, + }); + Stamp.configure(stamp); if (riskIdFilter) { console.log(`Filtering to risk IDs: ${[...riskIdFilter].join(", ")}`); } @@ -87,22 +97,23 @@ export async function expandScenariosCommand( // The per-call retry/fallback inside createGatewayModelChain only catches // thrown errors, so validation failures slip past it; rotating at the task // level fixes that. - const expansionModels = modelSlugs.map(slug => ({ - label: slug, - model: createGatewayModel(modelsJsonPath, slug), + const expansionModels = roles.expansion.map(spec => ({ + label: spec.name, + model: createSpecModel(spec), })); - const userModel = createGatewayModelChain(modelsJsonPath, userModelSlugs); + const userModel = createChainModel(roles.expansionUser).model; const outputDir = path.dirname(outputFilePath); const tempDir = path.join(outputDir, ".kora-expand-tmp"); // Clear output file if no process in progress (no temp files) - if (!(await hasTempFiles(tempDir))) { + if (!(await hasCachedFiles(tempDir))) { await fs.mkdir(outputDir, {recursive: true}); await fs.writeFile(outputFilePath, ""); } await fs.mkdir(tempDir, {recursive: true}); + await assertResumable(tempDir, stamp); const totalSeeds = await countSeeds(seedsFilePath, riskIdFilter); const progress = Script.progress(totalSeeds, text => @@ -187,7 +198,7 @@ export async function expandScenariosCommand( // Build final output from temp files. await fs.mkdir(outputDir, {recursive: true}); - const tempFiles = await fs.readdir(tempDir); + const tempFiles = await listCachedFiles(tempDir); let scenarioCount = 0; await fs.writeFile(outputFilePath, ""); diff --git a/packages/cli/src/commands/generateSeedsCommand.ts b/packages/cli/src/commands/generateSeedsCommand.ts index 2949af8..d440989 100644 --- a/packages/cli/src/commands/generateSeedsCommand.ts +++ b/packages/cli/src/commands/generateSeedsCommand.ts @@ -4,12 +4,19 @@ import { kora, largestRemainderCounts, RiskCategory, + Stamp, } from "@korabench/benchmark"; import {Script} from "@korabench/core"; import * as fs from "node:fs/promises"; import * as path from "node:path"; import {Program} from "../cli.js"; -import {createGatewayModelChain} from "../models/gatewayModel.js"; +import { + describeProfileRef, + resolveEffectiveProfile, + RoleOverrides, +} from "../profiles/effectiveProfile.js"; +import {chainLabel, createChainModel} from "../profiles/roleModels.js"; +import {buildRunStamp} from "../stamp/buildRunStamp.js"; function formatCounts(counts: Record): string { return Object.entries(counts) @@ -20,14 +27,18 @@ function formatCounts(counts: Record): string { export async function generateSeeds( _program: Program, modelsJsonPath: string, - modelSlugs: readonly string[], + overrides: RoleOverrides, outputFilePath: string, options?: GenerateSeedsOptions ) { + const effective = resolveEffectiveProfile(modelsJsonPath, overrides); + const {roles} = effective; + console.log(`Profile: ${describeProfileRef(effective.ref)}`); + Stamp.configure(await buildRunStamp({effective, modelsJsonPath})); console.log( - modelSlugs.length === 1 - ? `Generating seeds using ${modelSlugs[0]}...` - : `Generating seeds with fallback chain: ${modelSlugs.join(" → ")}` + roles.seeds.length === 1 + ? `Generating seeds using ${chainLabel(roles.seeds)}...` + : `Generating seeds with fallback chain: ${chainLabel(roles.seeds)}` ); if (options?.riskIds?.length) { console.log(`Filtering to risk IDs: ${options.riskIds.join(", ")}`); @@ -65,7 +76,7 @@ export async function generateSeeds( } } - const model = createGatewayModelChain(modelsJsonPath, modelSlugs); + const {model} = createChainModel(roles.seeds); const context: GenerateSeedsContext = { getResponse: async request => ({ diff --git a/packages/cli/src/commands/profileCommand.ts b/packages/cli/src/commands/profileCommand.ts new file mode 100644 index 0000000..85ae3a4 --- /dev/null +++ b/packages/cli/src/commands/profileCommand.ts @@ -0,0 +1,44 @@ +import {Program} from "../cli.js"; +import {resolveEffectiveProfile} from "../profiles/effectiveProfile.js"; +import {loadProfile, profilesDir} from "../profiles/loadProfile.js"; +import {checkProfileModels, printProfile} from "../profiles/printProfile.js"; +import {Profile} from "../profiles/profile.js"; +import {Profiles} from "../profiles/profiles.js"; + +// +// Command. +// + +export interface ProfileCommandOptions { + /** Call every model once and report what the provider served. */ + check?: boolean; + /** Print only the recomputed hash, even when the file's hash is stale. */ + printHash?: boolean; +} + +export async function profileCommand( + _program: Program, + modelsJsonPath: string, + spec: string, + options: ProfileCommandOptions = {} +) { + const dir = profilesDir(modelsJsonPath); + + if (options.printHash) { + const {profile} = loadProfile(spec, dir, {verifyHash: false}); + console.log(Profile.computeHash(profile)); + return; + } + + Profiles.configure(loadProfile(spec, dir)); + const effective = resolveEffectiveProfile(modelsJsonPath); + printProfile(effective, Profiles.current().path); + + if (options.check) { + console.log(""); + const ok = await checkProfileModels(effective); + if (!ok) { + process.exitCode = 1; + } + } +} diff --git a/packages/cli/src/commands/reassessCommand.ts b/packages/cli/src/commands/reassessCommand.ts index 30445e1..8b36b1b 100644 --- a/packages/cli/src/commands/reassessCommand.ts +++ b/packages/cli/src/commands/reassessCommand.ts @@ -6,6 +6,7 @@ import { runJudges, ScenarioKey, ScenarioPrompt, + Stamp, TestResult, } from "@korabench/benchmark"; import {Script} from "@korabench/core"; @@ -17,12 +18,20 @@ import * as R from "remeda"; import {flatTransform, pipeline, reduce} from "streaming-iterables"; import * as v from "valibot"; import {Program} from "../cli.js"; -import {createGatewayModel} from "../models/gatewayModel.js"; import {Model} from "../models/model.js"; +import { + describeProfileRef, + resolveEffectiveProfile, + RoleOverrides, +} from "../profiles/effectiveProfile.js"; +import {collectServed, createJudgeModels} from "../profiles/roleModels.js"; +import {buildRunStamp} from "../stamp/buildRunStamp.js"; +import {assertResumable} from "./shared/cacheStamp.js"; import { readReassessInputsFromJsonl, ReassessInput, } from "./shared/reassessInput.js"; +import {buildResultHeader} from "./shared/resultHeader.js"; import {resolveRiskIdFilter} from "./shared/riskFilters.js"; import {assertInputConforms} from "./shared/validateInputFile.js"; @@ -136,25 +145,30 @@ export interface ReassessCommandOptions { export async function reassessCommand( _program: Program, modelsJsonPath: string, - judgeModelSlugs: readonly string[], - userModelSlug: string, + overrides: RoleOverrides, inputFilePath: string, outputDirPath: string, options: ReassessCommandOptions = {} ) { + const effective = resolveEffectiveProfile(modelsJsonPath, overrides); + const {roles} = effective; + const judgeModelSlugs = roles.judges.map(spec => spec.name); + const userModelSlug = roles.user.name; + console.log(`Profile: ${describeProfileRef(effective.ref)}`); console.log( `Reassessing transcripts: judges=${judgeModelSlugs.join(",")}, user-label=${userModelSlug}` ); - if (judgeModelSlugs.length % 2 === 0) - throw new Error( - "The current implementation only supports odd numbers of judges. This ensures that the median assessment is always defined. See `aggregateTestAssessments` for reference." - ); - const recordCount = await assertInputConforms(inputFilePath, "reassess"); console.log( `Validated ${recordCount} record(s) against taxonomy "${RiskTaxonomy.label(Packs.current().taxonomy)}".` ); + const stamp = await buildRunStamp({ + effective, + modelsJsonPath, + inputPath: inputFilePath, + }); + Stamp.configure(stamp); const filters: ReassessFilters = { riskIds: resolveRiskIdFilter(options.riskIds), @@ -175,18 +189,14 @@ export async function reassessCommand( console.log(`Limiting to first ${filters.limit} record(s).`); } - const judgeModels: Record = Object.fromEntries( - judgeModelSlugs.map(slug => [ - slug, - createGatewayModel(modelsJsonPath, slug), - ]) - ); + const judgeModels = createJudgeModels(roles.judges); const judgeContext = buildJudgeContext(judgeModels); const tempDir = path.join(outputDirPath, ".kora-reassess-tmp"); await fs.mkdir(outputDirPath, {recursive: true}); await fs.mkdir(tempDir, {recursive: true}); + await assertResumable(tempDir, stamp); const totalTests = await countReassessTasks(inputFilePath, filters); @@ -316,11 +326,13 @@ export async function reassessCommand( for (const [modelId, runResult] of runResultsByTarget) { const prompts = [...(promptsByTarget.get(modelId) ?? new Set())]; const result = { - target: modelId, - judges: judgeModelSlugs, - packs: Packs.fingerprint(), - user: userModelSlug, - prompts, + ...buildResultHeader({ + target: modelId, + effective, + prompts, + stamp, + served: collectServed({judges: judgeModels}), + }), ...runResult, }; const filePath = path.join(outputDirPath, `${modelId}.json`); diff --git a/packages/cli/src/commands/runCommand.ts b/packages/cli/src/commands/runCommand.ts index 0463fd9..c12ddec 100644 --- a/packages/cli/src/commands/runCommand.ts +++ b/packages/cli/src/commands/runCommand.ts @@ -4,6 +4,7 @@ import { RiskTaxonomy, Scenario, ScenarioPrompt, + Stamp, TestResult, } from "@korabench/benchmark"; import {Hash, Script} from "@korabench/core"; @@ -15,13 +16,24 @@ import * as readline from "node:readline"; import {flatTransform, pipeline, reduce} from "streaming-iterables"; import * as v from "valibot"; import {Program} from "../cli.js"; -import {createGatewayModel} from "../models/gatewayModel.js"; -import {Model} from "../models/model.js"; +import { + describeProfileRef, + resolveEffectiveProfile, + RoleOverrides, +} from "../profiles/effectiveProfile.js"; +import { + collectServed, + createJudgeModels, + createSpecModel, +} from "../profiles/roleModels.js"; +import {buildRunStamp} from "../stamp/buildRunStamp.js"; import { buildContext, resolveTargetGatewayModel, } from "./shared/buildContext.js"; +import {assertResumable, hasCachedFiles} from "./shared/cacheStamp.js"; import {reportInvalidTurn} from "./shared/reportInvalidTurn.js"; +import {buildResultHeader} from "./shared/resultHeader.js"; import {resolveRiskIdFilter} from "./shared/riskFilters.js"; import {assertInputConforms} from "./shared/validateInputFile.js"; @@ -158,15 +170,6 @@ async function archiveResults( await done; } -async function hasTempFiles(tempDir: string): Promise { - try { - const files = await fs.readdir(tempDir); - return files.length > 0; - } catch { - return false; - } -} - export interface RunCommandOptions { riskIds?: readonly string[]; limit?: number; @@ -186,22 +189,21 @@ export async function runCommand( _program: Program, modelsJsonPath: string, targetModelSlug: string, - judgeModelSlugs: readonly string[], - userModelSlug: string, + overrides: RoleOverrides, scenariosFilePath: string, outputFilePath: string, prompts: readonly ScenarioPrompt[], options: RunCommandOptions = {} ) { + const effective = resolveEffectiveProfile(modelsJsonPath, overrides); + const {roles} = effective; + const judgeModelSlugs = roles.judges.map(spec => spec.name); + const userModelSlug = roles.user.name; + console.log(`Profile: ${describeProfileRef(effective.ref)}`); console.log( `Running benchmark: target=${targetModelSlug}, judges=${judgeModelSlugs.join(",")}, user=${userModelSlug}` ); - if (judgeModelSlugs.length % 2 === 0) - throw new Error( - "The current implementation only supports odd numbers of judges. This ensures that the median assessment is always defined. See `aggregateTestAssessments` for reference." - ); - // Validate the risk-id filter and the whole scenario file against the active // taxonomy before any model is constructed — a mismatch must not surface // half-way through a paid run. @@ -217,6 +219,13 @@ export async function runCommand( console.log( `Validated ${scenarioCount} scenario(s) against taxonomy "${RiskTaxonomy.label(Packs.current().taxonomy)}".` ); + const stamp = await buildRunStamp({ + effective, + modelsJsonPath, + target: targetModelSlug, + inputPath: scenariosFilePath, + }); + Stamp.configure(stamp); if (filters.riskIds) { console.log(`Filtering to risk IDs: ${[...filters.riskIds].join(", ")}`); } @@ -234,13 +243,8 @@ export async function runCommand( } let freshStarted = 0; - const judgeModels: Record = Object.fromEntries( - judgeModelSlugs.map(slug => [ - slug, - createGatewayModel(modelsJsonPath, slug), - ]) - ); - const userModel = createGatewayModel(modelsJsonPath, userModelSlug); + const judgeModels = createJudgeModels(roles.judges); + const userModel = createSpecModel(roles.user); const targetGatewayModel = resolveTargetGatewayModel( modelsJsonPath, targetModelSlug @@ -250,12 +254,13 @@ export async function runCommand( const tempDir = path.join(outputDir, ".kora-run-tmp"); // Clear output file if no process in progress (no temp files) - if (!(await hasTempFiles(tempDir))) { + if (!(await hasCachedFiles(tempDir))) { await fs.mkdir(outputDir, {recursive: true}); await fs.writeFile(outputFilePath, ""); } await fs.mkdir(tempDir, {recursive: true}); + await assertResumable(tempDir, stamp); const totalTests = await countTestTasks(scenariosFilePath, prompts, filters); @@ -367,11 +372,17 @@ export async function runCommand( // Write reduced result. const result = { - target: targetModelSlug, - judges: judgeModelSlugs, - user: userModelSlug, - prompts, - packs: Packs.fingerprint(), + ...buildResultHeader({ + target: targetModelSlug, + effective, + prompts, + stamp, + served: collectServed({ + user: userModel, + judges: judgeModels, + target: targetGatewayModel, + }), + }), ...(runResult ?? {}), }; diff --git a/packages/cli/src/commands/shared/__tests__/cacheStamp.test.ts b/packages/cli/src/commands/shared/__tests__/cacheStamp.test.ts new file mode 100644 index 0000000..2e510c4 --- /dev/null +++ b/packages/cli/src/commands/shared/__tests__/cacheStamp.test.ts @@ -0,0 +1,65 @@ +import {mkdtempSync, readFileSync, writeFileSync} from "node:fs"; +import {tmpdir} from "node:os"; +import * as path from "node:path"; +import {afterEach, describe, expect, it, vi} from "vitest"; +import {makeStamp} from "../../../stamp/__tests__/fixtures.js"; +import {assertResumable, hasCachedFiles, STAMP_FILE} from "../cacheStamp.js"; + +function freshDir(): string { + return mkdtempSync(path.join(tmpdir(), "kora-cache-")); +} + +afterEach(() => vi.restoreAllMocks()); + +describe("assertResumable", () => { + it("writes the stamp into a fresh directory", async () => { + const dir = freshDir(); + const stamp = makeStamp(); + await assertResumable(dir, stamp); + expect( + JSON.parse(readFileSync(path.join(dir, STAMP_FILE), "utf-8")) + ).toEqual(JSON.parse(JSON.stringify(stamp))); + }); + + it("resumes under an equal stamp", async () => { + const dir = freshDir(); + await assertResumable(dir, makeStamp()); + await expect( + assertResumable(dir, makeStamp({code: {version: "other"}})) + ).resolves.toBeUndefined(); + }); + + it("refuses a different configuration and names both", async () => { + const dir = freshDir(); + await assertResumable(dir, makeStamp()); + await expect( + assertResumable( + dir, + makeStamp({profile: {id: "test", version: "2", hash: "other"}}) + ) + ).rejects.toThrow( + /Refusing to resume .*\n\s+cached:\s+profile test@1 \(profile-hash\).*\n\s+current: profile test@2 \(other\)/ + ); + }); + + it("warns and proceeds for a legacy directory without a stamp", async () => { + const dir = freshDir(); + writeFileSync(path.join(dir, "abc.json"), "{}"); + const warn = vi.spyOn(console, "error").mockImplementation(() => {}); + await assertResumable(dir, makeStamp()); + expect(warn).toHaveBeenCalledWith(expect.stringMatching(/without a stamp/)); + expect(await hasCachedFiles(dir)).toBe(true); + }); +}); + +describe("hasCachedFiles", () => { + it("ignores the stamp file", async () => { + const dir = freshDir(); + await assertResumable(dir, makeStamp()); + expect(await hasCachedFiles(dir)).toBe(false); + }); + + it("is false for a missing directory", async () => { + expect(await hasCachedFiles(path.join(freshDir(), "absent"))).toBe(false); + }); +}); diff --git a/packages/cli/src/commands/shared/__tests__/resultHeader.test.ts b/packages/cli/src/commands/shared/__tests__/resultHeader.test.ts new file mode 100644 index 0000000..09f0154 --- /dev/null +++ b/packages/cli/src/commands/shared/__tests__/resultHeader.test.ts @@ -0,0 +1,56 @@ +import {describe, expect, it} from "vitest"; +import {makeRoles, makeSpec} from "../../../profiles/__tests__/fixtures.js"; +import {EffectiveProfile} from "../../../profiles/effectiveProfile.js"; +import {Profile} from "../../../profiles/profile.js"; +import {makeStamp} from "../../../stamp/__tests__/fixtures.js"; +import {buildResultHeader} from "../resultHeader.js"; + +const effective: EffectiveProfile = { + ref: {id: "test", version: "1", hash: "h"}, + roles: Profile.effectiveRoles( + makeRoles({ + judges: [makeSpec("j1"), makeSpec("j2"), makeSpec("j3")], + continueUser: makeSpec("cu"), + }) + ), +}; + +describe("buildResultHeader", () => { + it("keeps the historical fields and adds the stamp", () => { + const stamp = makeStamp(); + const header = buildResultHeader({ + target: "gpt-x", + effective, + prompts: ["default"], + stamp, + }); + expect(Object.keys(header)).toEqual([ + "target", + "judges", + "user", + "prompts", + "packs", + "stamp", + ]); + expect(header.judges).toEqual(["j1", "j2", "j3"]); + expect(header.user).toBe("user-a"); + expect(header.packs).toBe(stamp.packs); + expect(header.stamp).toBe(stamp); + }); + + it("takes an explicit user name and served ids", () => { + const header = buildResultHeader({ + target: "gpt-x", + effective, + prompts: ["default"], + stamp: makeStamp(), + userName: "cu", + served: {user: ["deepseek/x"], judges: {j1: ["openai/y"]}}, + }); + expect(header.user).toBe("cu"); + expect(header.served).toEqual({ + user: ["deepseek/x"], + judges: {j1: ["openai/y"]}, + }); + }); +}); diff --git a/packages/cli/src/commands/shared/buildContext.ts b/packages/cli/src/commands/shared/buildContext.ts index bceabbf..584c7e9 100644 --- a/packages/cli/src/commands/shared/buildContext.ts +++ b/packages/cli/src/commands/shared/buildContext.ts @@ -1,7 +1,7 @@ import {JudgeModel, Scenario, TestContext} from "@korabench/benchmark"; import * as R from "remeda"; import {createCustomModel} from "../../models/customModel.js"; -import {createGatewayModel} from "../../models/gatewayModel.js"; +import {createGatewayModel, GatewayModel} from "../../models/gatewayModel.js"; import {Model} from "../../models/model.js"; import {isNativeRunnerSlug} from "../../models/nativeRunnerModel.js"; import {isWebRunnerSlug} from "../../models/webRunnerModel.js"; @@ -61,7 +61,7 @@ export async function buildContext( export function resolveTargetGatewayModel( modelsJsonPath: string, targetModelSlug: string -): Model | undefined { +): GatewayModel | undefined { if ( targetModelSlug.startsWith("custom-") || isWebRunnerSlug(targetModelSlug) || diff --git a/packages/cli/src/commands/shared/cacheStamp.ts b/packages/cli/src/commands/shared/cacheStamp.ts new file mode 100644 index 0000000..19fa19f --- /dev/null +++ b/packages/cli/src/commands/shared/cacheStamp.ts @@ -0,0 +1,68 @@ +import {RunStamp} from "@korabench/benchmark"; +import * as fs from "node:fs/promises"; +import * as path from "node:path"; +import * as v from "valibot"; + +// +// Graceful-restart cache guard. +// +// Every command that can resume from a temp directory writes the run stamp +// there. On resume, a cached result produced under a different configuration +// (other judges, other prompts, other packs) is refused outright: mixing it +// into the new run would silently corrupt the results. There is no bypass +// flag; delete the directory or re-run with the same configuration. +// + +export const STAMP_FILE = "stamp.json"; + +async function readCachedStamp(tempDir: string): Promise { + try { + const raw = await fs.readFile(path.join(tempDir, STAMP_FILE), "utf-8"); + return v.parse(RunStamp.io, JSON.parse(raw)); + } catch { + return undefined; + } +} + +/** Entries of `tempDir` other than the stamp file. */ +export async function listCachedFiles(tempDir: string): Promise { + try { + const files = await fs.readdir(tempDir); + return files.filter(file => file !== STAMP_FILE); + } catch { + return []; + } +} + +export async function hasCachedFiles(tempDir: string): Promise { + return (await listCachedFiles(tempDir)).length > 0; +} + +/** + * Refuse to resume `tempDir` under a stamp that differs from the one it was + * started with, then record `stamp` there. `tempDir` must exist. + */ +export async function assertResumable( + tempDir: string, + stamp: RunStamp +): Promise { + const cached = await readCachedStamp(tempDir); + if (cached) { + if (!RunStamp.equals(cached, stamp)) { + throw new Error( + `Refusing to resume ${tempDir}: it holds results from a different configuration.\n` + + ` cached: ${RunStamp.describe(cached)}\n` + + ` current: ${RunStamp.describe(stamp)}\n` + + `Delete ${tempDir} to start over, or re-run with the same --profile and overrides.` + ); + } + } else if (await hasCachedFiles(tempDir)) { + console.error( + `WARNING: ${tempDir} holds cached results without a stamp (written before stamps existed); assuming they match the current configuration.` + ); + } + await fs.writeFile( + path.join(tempDir, STAMP_FILE), + JSON.stringify(stamp, null, 2) + ); +} diff --git a/packages/cli/src/commands/shared/resultHeader.ts b/packages/cli/src/commands/shared/resultHeader.ts new file mode 100644 index 0000000..80993a3 --- /dev/null +++ b/packages/cli/src/commands/shared/resultHeader.ts @@ -0,0 +1,44 @@ +import {RunStamp, ScenarioPrompt} from "@korabench/benchmark"; +import {EffectiveProfile} from "../../profiles/effectiveProfile.js"; +import {ServedModels} from "../../profiles/roleModels.js"; + +// +// Result header. +// +// The fields written ahead of `scores` in every result JSON. `judges` / `user` +// keep their historical meaning (display names), `packs` stays for readers +// that predate stamps, and `stamp` is the full provenance. +// + +export interface ResultHeaderArgs { + target: string; + effective: EffectiveProfile; + prompts: readonly ScenarioPrompt[]; + stamp: RunStamp; + /** Defaults to the profile's `user` role; `continue` passes `continueUser`. */ + userName?: string; + served?: ServedModels; +} + +export interface ResultHeader { + target: string; + judges: readonly string[]; + user: string; + prompts: readonly ScenarioPrompt[]; + packs: RunStamp["packs"]; + stamp: RunStamp; + served?: ServedModels; +} + +export function buildResultHeader(args: ResultHeaderArgs): ResultHeader { + const {target, effective, prompts, stamp, userName, served} = args; + return { + target, + judges: effective.roles.judges.map(spec => spec.name), + user: userName ?? effective.roles.user.name, + prompts, + packs: stamp.packs, + stamp, + ...(served ? {served} : {}), + }; +} diff --git a/packages/cli/src/commands/validateCommand.ts b/packages/cli/src/commands/validateCommand.ts index c131ad4..d451fd0 100644 --- a/packages/cli/src/commands/validateCommand.ts +++ b/packages/cli/src/commands/validateCommand.ts @@ -1,6 +1,7 @@ -import {Mechanism, Packs, RiskTaxonomy} from "@korabench/benchmark"; import * as fs from "node:fs/promises"; import {Program} from "../cli.js"; +import {resolveEffectiveProfile} from "../profiles/effectiveProfile.js"; +import {printProfile} from "../profiles/printProfile.js"; import {assertInputConforms, InputKind} from "./shared/validateInputFile.js"; // @@ -37,28 +38,6 @@ async function detectKind(filePath: string): Promise { ); } -// -// Reporting. -// - -function printPacks(): void { - const {taxonomy, behaviors} = Packs.current(); - const stamp = Packs.fingerprint(); - - console.log( - `Taxonomy: ${RiskTaxonomy.label(taxonomy)} (${stamp.taxonomy.hash})` - ); - console.log( - ` ${taxonomy.categories.length} categories, ${RiskTaxonomy.allRisks(taxonomy).length} risks` - ); - console.log( - `Behaviors: ${behaviors.id}@${behaviors.version} (${stamp.behaviors.hash})` - ); - console.log( - ` ${behaviors.behaviors.map(Mechanism.codeOf).join(", ")}` - ); -} - // // Command. // @@ -70,10 +49,11 @@ export interface ValidateCommandOptions { export async function validateCommand( _program: Program, + modelsJsonPath: string, inputFilePath: string, options: ValidateCommandOptions = {} ) { - printPacks(); + printProfile(resolveEffectiveProfile(modelsJsonPath)); if (options.packsOnly) { return; diff --git a/packages/cli/src/models/gatewayModel.ts b/packages/cli/src/models/gatewayModel.ts index 81d3621..3225a1e 100644 --- a/packages/cli/src/models/gatewayModel.ts +++ b/packages/cli/src/models/gatewayModel.ts @@ -5,12 +5,21 @@ import * as v from "valibot"; import {createLogRetryHandler, RetryOptions, withRetry} from "../retry.js"; import {createFallbackModel} from "./fallbackModel.js"; import {Model} from "./model.js"; -import {resolveModelConfig} from "./modelConfig.js"; +import {ModelConfig, resolveModelConfig} from "./modelConfig.js"; export interface ModelOptions { retry?: RetryOptions; } +export interface GatewayModel extends Model { + /** + * Model ids the provider reported serving, as seen on each response. A + * pinned id such as "openai/gpt-5.2" can still resolve to different + * snapshots over time; this is the only evidence of which one answered. + */ + readonly served: ReadonlySet; +} + const defaultRetryOptions: RetryOptions = { maxRetries: 5, initialDelayMs: 1000, @@ -64,11 +73,28 @@ export function createGatewayModel( modelsJsonPath: string, modelSlug: string, options?: ModelOptions -): Model { - const config = resolveModelConfig(modelsJsonPath, modelSlug); - const retryOptions = buildRetryOptions(config.model, options); +): GatewayModel { + return createGatewayModelFromConfig( + resolveModelConfig(modelsJsonPath, modelSlug), + modelSlug, + options + ); +} + +/** Build a gateway model from an inline config (no registry lookup). */ +export function createGatewayModelFromConfig( + config: ModelConfig, + label: string, + options?: ModelOptions +): GatewayModel { + const retryLabel = + label === config.model ? label : `${label} (${config.model})`; + const retryOptions = buildRetryOptions(retryLabel, options); + const served = new Set(); return { + served, + async getTextResponse(request: ModelRequest): Promise { const maxTokens = request.maxTokens ?? config.maxTokens; const temperature = request.temperature ?? config.temperature; @@ -94,6 +120,7 @@ export function createGatewayModel( retryOptions ); + served.add(result.response.modelId); return result.text; }, @@ -134,6 +161,7 @@ export function createGatewayModel( maxRetries: 0, }); + served.add(result.response.modelId); const parsed = JSON.parse(extractJson(result.text)); return v.parse(request.outputType, parsed); }, retryOptions); @@ -151,6 +179,7 @@ export function createGatewayModel( maxRetries: 0, }); + served.add(result.response.modelId); return v.parse(request.outputType, result.object); }, retryOptions); }, diff --git a/packages/cli/src/models/modelConfig.ts b/packages/cli/src/models/modelConfig.ts index c44dc9b..2d554fd 100644 --- a/packages/cli/src/models/modelConfig.ts +++ b/packages/cli/src/models/modelConfig.ts @@ -1,3 +1,4 @@ +import {ModelSpec} from "@korabench/benchmark"; import {memoize} from "@korabench/core"; import * as fs from "node:fs"; import * as v from "valibot"; @@ -6,14 +7,9 @@ import * as v from "valibot"; // Runtime model. // -const VModelConfig = v.object({ - model: v.string(), - maxTokens: v.optional(v.number()), - temperature: v.optional(v.number()), - providerOptions: v.optional( - v.record(v.string(), v.record(v.string(), v.unknown())) - ), -}); +// A registry entry is a `ModelSpec` minus its name (the slug is the key), so +// the two shapes cannot drift apart. +const VModelConfig = v.omit(ModelSpec.io, ["name"]); const VModelRegistry = v.record(v.string(), VModelConfig); diff --git a/packages/cli/src/profiles/__tests__/committedProfiles.test.ts b/packages/cli/src/profiles/__tests__/committedProfiles.test.ts new file mode 100644 index 0000000..f73aee3 --- /dev/null +++ b/packages/cli/src/profiles/__tests__/committedProfiles.test.ts @@ -0,0 +1,71 @@ +import {readFileSync} from "node:fs"; +import * as path from "node:path"; +import {fileURLToPath} from "node:url"; +import * as R from "remeda"; +import {describe, expect, it} from "vitest"; +import {resolveModelConfig} from "../../models/modelConfig.js"; +import {listProfileNames} from "../loadProfile.js"; +import {Profile, Role} from "../profile.js"; + +// +// CI guard for the committed profiles under `/profiles/`. +// +// A profile's hash is what results are keyed on. Editing a profile without +// bumping its version and hash would make new runs look comparable to old +// ones, so this test refuses that state and prints the value to paste. +// + +const repoRoot = fileURLToPath(new URL("../../../../../", import.meta.url)); +const profilesDir = path.join(repoRoot, "profiles"); +const modelsJsonPath = path.join(repoRoot, "models.json"); + +function readProfile(name: string): Profile { + const raw = JSON.parse( + readFileSync(path.join(profilesDir, `${name}.json`), "utf-8") + ); + return Profile.parse(raw, {verifyHash: false}); +} + +const names = listProfileNames(profilesDir); +const profiles = names.map(name => [name, readProfile(name)] as const); + +describe("committed profiles", () => { + it("include the bundled default", () => { + expect(names).toContain("kora"); + }); + + it.each(profiles)("%s carries its own content hash", (name, profile) => { + const expected = Profile.computeHash(profile); + expect( + profile.hash, + `profiles/${name}.json changed: bump "version" and set "hash" to "${expected}"` + ).toBe(expected); + }); + + it("use unique ids and versions", () => { + const labels = profiles.map(([, p]) => Profile.label(p)); + expect(R.unique(labels)).toEqual(labels); + const ids = profiles.map(([, p]) => p.id); + expect(R.unique(ids)).toEqual(ids); + }); + + it("name each profile after its file", () => { + profiles.forEach(([name, profile]) => expect(profile.id).toBe(name)); + }); +}); + +describe("bundled profile kora", () => { + // Every role of the default profile must match the models.json entry of + // the same name: the profile replaced hardcoded CLI defaults that were + // registry slugs, and result headers still print those names. + const kora = readProfile("kora"); + + it.each(Role.list)("role %s matches the registry entry of its name", role => { + Role.specsOf(kora.roles, role).forEach(spec => { + expect( + R.omit(spec, ["name"]), + `profiles/kora.json role ${role}: "${spec.name}" differs from models.json` + ).toEqual(resolveModelConfig(modelsJsonPath, spec.name)); + }); + }); +}); diff --git a/packages/cli/src/profiles/__tests__/effectiveProfile.test.ts b/packages/cli/src/profiles/__tests__/effectiveProfile.test.ts new file mode 100644 index 0000000..cc760d7 --- /dev/null +++ b/packages/cli/src/profiles/__tests__/effectiveProfile.test.ts @@ -0,0 +1,121 @@ +import {mkdtempSync, writeFileSync} from "node:fs"; +import {tmpdir} from "node:os"; +import * as path from "node:path"; +import {afterEach, beforeEach, describe, expect, it, vi} from "vitest"; +import { + describeProfileRef, + resolveEffectiveProfile, +} from "../effectiveProfile.js"; +import {ODD_JUDGES_MESSAGE} from "../profile.js"; +import {Profiles} from "../profiles.js"; +import {makeProfile, makeRoles, makeSpec} from "./fixtures.js"; + +const dir = mkdtempSync(path.join(tmpdir(), "kora-effective-")); +const modelsJsonPath = path.join(dir, "models.json"); +writeFileSync( + modelsJsonPath, + JSON.stringify({ + "judge-x": {model: "openai/judge-x", maxTokens: 100}, + "judge-y": {model: "openai/judge-y"}, + "user-x": {model: "deepseek/user-x", temperature: 0.2}, + }) +); + +const profile = makeProfile({id: "kora"}); + +beforeEach(() => { + Profiles.configure({profile, local: false, path: "/x/kora.json"}); + vi.spyOn(console, "error").mockImplementation(() => {}); +}); + +afterEach(() => { + Profiles.reset(); + vi.restoreAllMocks(); +}); + +describe("resolveEffectiveProfile", () => { + it("returns the profile's own hash and roles without overrides", () => { + const effective = resolveEffectiveProfile(modelsJsonPath); + expect(effective.ref).toEqual({ + id: "kora", + version: "1", + hash: profile.hash, + }); + expect(effective.roles.continueUser).toEqual(profile.roles.user); + expect(console.error).not.toHaveBeenCalled(); + }); + + it("resolves an override through models.json and re-hashes", () => { + const effective = resolveEffectiveProfile(modelsJsonPath, { + judges: ["judge-x"], + }); + expect(effective.roles.judges).toEqual([ + {name: "judge-x", model: "openai/judge-x", maxTokens: 100}, + ]); + expect(effective.ref.overrides).toEqual(["judges"]); + expect(effective.ref.hash).not.toBe(profile.hash); + expect(console.error).toHaveBeenCalledWith( + expect.stringMatching( + /WARNING: command-line judges overrides profile kora@1 \(judge-a -> judge-x\)/ + ) + ); + }); + + it("hashes identical overrides identically", () => { + const a = resolveEffectiveProfile(modelsJsonPath, {judges: ["judge-x"]}); + const b = resolveEffectiveProfile(modelsJsonPath, {judges: ["judge-x"]}); + expect(a.ref.hash).toBe(b.ref.hash); + }); + + it("marks local profiles", () => { + Profiles.reset(); + Profiles.configure({profile, local: true, path: "/x/kora.local.json"}); + expect(resolveEffectiveProfile(modelsJsonPath).ref.local).toBe(true); + }); + + it("rejects an even number of judge overrides", () => { + expect(() => + resolveEffectiveProfile(modelsJsonPath, {judges: ["judge-x", "judge-y"]}) + ).toThrow(ODD_JUDGES_MESSAGE); + }); + + it("rejects several models for a single-model role", () => { + expect(() => + resolveEffectiveProfile(modelsJsonPath, {user: ["user-x", "judge-x"]}) + ).toThrow(/Role "user" takes exactly one model, got 2/); + }); + + it("rejects an unknown slug", () => { + expect(() => + resolveEffectiveProfile(modelsJsonPath, {user: ["nope"]}) + ).toThrow(/Unknown model "nope"/); + }); + + it("ignores empty override lists", () => { + const effective = resolveEffectiveProfile(modelsJsonPath, {judges: []}); + expect(effective.ref.hash).toBe(profile.hash); + }); +}); + +describe("describeProfileRef", () => { + it("shows markers only when present", () => { + expect(describeProfileRef({id: "kora", version: "1", hash: "h"})).toBe( + "kora@1 (h)" + ); + expect( + describeProfileRef({ + id: "kora", + version: "1", + hash: "h", + local: true, + overrides: ["judges", "user"], + }) + ).toBe("kora@1 (h) [local; overrides: judges,user]"); + }); +}); + +describe("fixtures", () => { + it("build a valid roles object", () => { + expect(makeRoles({user: makeSpec("u")}).user.name).toBe("u"); + }); +}); diff --git a/packages/cli/src/profiles/__tests__/fixtures.ts b/packages/cli/src/profiles/__tests__/fixtures.ts new file mode 100644 index 0000000..5e8d2e0 --- /dev/null +++ b/packages/cli/src/profiles/__tests__/fixtures.ts @@ -0,0 +1,37 @@ +import {ModelSpec} from "@korabench/benchmark"; +import {Profile, ProfileRoles} from "../profile.js"; + +// +// Test fixtures. +// + +export function makeSpec( + name: string, + overrides: Partial = {} +): ModelSpec { + return {name, model: `provider/${name}`, ...overrides}; +} + +export function makeRoles(overrides: Partial = {}): ProfileRoles { + return { + seeds: [makeSpec("seed-a")], + expansion: [makeSpec("expand-a")], + expansionUser: [makeSpec("user-a")], + user: makeSpec("user-a"), + judges: [makeSpec("judge-a")], + ...overrides, + }; +} + +/** A structurally valid profile whose `hash` is correct. */ +export function makeProfile( + overrides: Partial> = {} +): Profile { + const base = { + id: "test", + version: "1", + roles: makeRoles(), + ...overrides, + }; + return {...base, hash: Profile.computeHash(base)}; +} diff --git a/packages/cli/src/profiles/__tests__/loadProfile.test.ts b/packages/cli/src/profiles/__tests__/loadProfile.test.ts new file mode 100644 index 0000000..ff83fe1 --- /dev/null +++ b/packages/cli/src/profiles/__tests__/loadProfile.test.ts @@ -0,0 +1,83 @@ +import {mkdirSync, mkdtempSync, writeFileSync} from "node:fs"; +import {tmpdir} from "node:os"; +import * as path from "node:path"; +import {describe, expect, it} from "vitest"; +import {listProfileNames, loadProfile, profilesDir} from "../loadProfile.js"; +import {makeProfile} from "./fixtures.js"; + +const dir = mkdtempSync(path.join(tmpdir(), "kora-profiles-")); + +function writeJson(name: string, value: unknown): string { + const filePath = path.join(dir, name); + writeFileSync(filePath, JSON.stringify(value)); + return filePath; +} + +writeJson("kora.json", makeProfile({id: "kora"})); +writeJson("other.json", makeProfile({id: "other"})); +writeJson("scratch.local.json", { + ...makeProfile({id: "scratch"}), + hash: "nope", +}); +writeJson("stale.json", {...makeProfile({id: "stale"}), hash: "nope"}); + +describe("profilesDir", () => { + it("sits next to models.json", () => { + expect(profilesDir("/repo/models.json")).toBe("/repo/profiles"); + }); +}); + +describe("listProfileNames", () => { + it("lists committed profiles and hides local ones", () => { + expect(listProfileNames(dir)).toEqual(["kora", "other", "stale"]); + }); + + it("is empty for a missing directory", () => { + expect(listProfileNames(path.join(dir, "absent"))).toEqual([]); + }); +}); + +describe("loadProfile", () => { + it("resolves a committed profile by name", () => { + const loaded = loadProfile("kora", dir); + expect(loaded.profile.id).toBe("kora"); + expect(loaded.local).toBe(false); + expect(loaded.path).toBe(path.join(dir, "kora.json")); + }); + + it("resolves a local profile by name and skips the hash check", () => { + const loaded = loadProfile("scratch.local", dir); + expect(loaded.profile.id).toBe("scratch"); + expect(loaded.local).toBe(true); + }); + + it("verifies the hash of a committed profile", () => { + expect(() => loadProfile("stale", dir)).toThrow(/declares hash "nope"/); + }); + + it("can skip verification explicitly", () => { + expect(loadProfile("stale", dir, {verifyHash: false}).profile.hash).toBe( + "nope" + ); + }); + + it("lists known profiles for an unknown name", () => { + expect(() => loadProfile("nope", dir)).toThrow( + /Unknown profile "nope". Known profiles: kora, other, stale/ + ); + }); + + it("loads a profile from a path", () => { + const nested = path.join(dir, "elsewhere"); + mkdirSync(nested, {recursive: true}); + const filePath = path.join(nested, "custom.json"); + writeFileSync(filePath, JSON.stringify(makeProfile({id: "custom"}))); + expect(loadProfile(filePath, dir).profile.id).toBe("custom"); + }); + + it("reports the path when a file is missing", () => { + expect(() => loadProfile(path.join(dir, "absent.json"), dir)).toThrow( + /Could not read profile from .*absent\.json/ + ); + }); +}); diff --git a/packages/cli/src/profiles/__tests__/profile.test.ts b/packages/cli/src/profiles/__tests__/profile.test.ts new file mode 100644 index 0000000..a6d16d3 --- /dev/null +++ b/packages/cli/src/profiles/__tests__/profile.test.ts @@ -0,0 +1,95 @@ +import {describe, expect, it} from "vitest"; +import {ODD_JUDGES_MESSAGE, Profile, Role} from "../profile.js"; +import {makeProfile, makeRoles, makeSpec} from "./fixtures.js"; + +describe("Profile.parse", () => { + it("accepts a valid profile with a matching hash", () => { + const profile = makeProfile(); + expect(Profile.parse(profile, {verifyHash: true})).toEqual(profile); + }); + + it("rejects an even number of judges", () => { + const profile = makeProfile({ + roles: makeRoles({judges: [makeSpec("j1"), makeSpec("j2")]}), + }); + expect(() => Profile.parse(profile, {verifyHash: true})).toThrow( + ODD_JUDGES_MESSAGE + ); + }); + + it("rejects a duplicate model name within a role", () => { + const profile = makeProfile({ + roles: makeRoles({seeds: [makeSpec("same"), makeSpec("same")]}), + }); + expect(() => Profile.parse(profile, {verifyHash: true})).toThrow( + /role "seeds" lists the same model name more than once: same/ + ); + }); + + it("rejects unknown keys so a typo cannot silently drop a role", () => { + const profile = {...makeProfile(), roles: {...makeRoles(), judge: []}}; + expect(() => Profile.parse(profile, {verifyHash: true})).toThrow(); + }); + + it("reports the expected hash on mismatch", () => { + const profile = {...makeProfile(), hash: "stale"}; + const expected = Profile.computeHash(profile); + expect(() => Profile.parse(profile, {verifyHash: true})).toThrow( + new RegExp( + `declares hash "stale" but its content hashes to "${expected}"` + ) + ); + }); + + it("accepts a stale hash when verification is off", () => { + const profile = {...makeProfile(), hash: "stale"}; + expect(Profile.parse(profile, {verifyHash: false}).hash).toBe("stale"); + }); +}); + +describe("Profile.computeHash", () => { + it("ignores the hash field and key order", () => { + const profile = makeProfile(); + const reordered = { + roles: profile.roles, + version: profile.version, + id: profile.id, + }; + expect(Profile.computeHash(reordered)).toBe(profile.hash); + }); + + it("changes when any role config changes", () => { + const a = makeProfile(); + const b = makeProfile({ + roles: makeRoles({user: makeSpec("user-a", {temperature: 0.1})}), + }); + expect(a.hash).not.toBe(b.hash); + }); + + it("changes when the version changes", () => { + expect(makeProfile({version: "1"}).hash).not.toBe( + makeProfile({version: "2"}).hash + ); + }); +}); + +describe("Profile.effectiveRoles", () => { + it("falls back to user for continueUser", () => { + const roles = makeRoles(); + expect(Profile.effectiveRoles(roles).continueUser).toEqual(roles.user); + }); + + it("keeps an explicit continueUser", () => { + const roles = makeRoles({continueUser: makeSpec("cont")}); + expect(Profile.effectiveRoles(roles).continueUser.name).toBe("cont"); + }); +}); + +describe("Role.specsOf", () => { + it("normalizes single and chain roles to arrays", () => { + const roles = makeRoles(); + expect(Role.specsOf(roles, "user")).toEqual([roles.user]); + expect(Role.specsOf(roles, "seeds")).toEqual(roles.seeds); + expect(Role.specsOf(roles, "continueUser")).toEqual([]); + }); +}); diff --git a/packages/cli/src/profiles/__tests__/profiles.test.ts b/packages/cli/src/profiles/__tests__/profiles.test.ts new file mode 100644 index 0000000..644e295 --- /dev/null +++ b/packages/cli/src/profiles/__tests__/profiles.test.ts @@ -0,0 +1,41 @@ +import {afterEach, describe, expect, it} from "vitest"; +import {LoadedProfile} from "../loadProfile.js"; +import {Profiles} from "../profiles.js"; +import {makeProfile} from "./fixtures.js"; + +afterEach(() => Profiles.reset()); + +function loaded(id: string, local = false): LoadedProfile { + return {profile: makeProfile({id}), local, path: `/x/${id}.json`}; +} + +describe("Profiles", () => { + it("throws before configuration", () => { + expect(() => Profiles.current()).toThrow( + /No evaluation profile configured/ + ); + }); + + it("returns the configured profile", () => { + const a = loaded("a"); + Profiles.configure(a); + expect(Profiles.current()).toBe(a); + }); + + it("is idempotent for identical content", () => { + Profiles.configure(loaded("a")); + expect(() => Profiles.configure(loaded("a"))).not.toThrow(); + }); + + it("refuses a second, different profile", () => { + Profiles.configure(loaded("a")); + expect(() => Profiles.configure(loaded("b"))).toThrow( + /called twice with different profiles \(a@1 then b@1\)/ + ); + }); + + it("treats local and committed copies as different", () => { + Profiles.configure(loaded("a")); + expect(() => Profiles.configure(loaded("a", true))).toThrow(); + }); +}); diff --git a/packages/cli/src/profiles/__tests__/roleModels.test.ts b/packages/cli/src/profiles/__tests__/roleModels.test.ts new file mode 100644 index 0000000..fe10caf --- /dev/null +++ b/packages/cli/src/profiles/__tests__/roleModels.test.ts @@ -0,0 +1,40 @@ +import {describe, expect, it} from "vitest"; +import { + chainLabel, + createChainModel, + createJudgeModels, + createSpecModel, +} from "../roleModels.js"; +import {makeSpec} from "./fixtures.js"; + +describe("createSpecModel", () => { + it("exposes an empty served set before any call", () => { + const model = createSpecModel(makeSpec("a")); + expect([...model.served]).toEqual([]); + }); +}); + +describe("createChainModel", () => { + it("keeps one member per spec, in order", () => { + const chain = createChainModel([makeSpec("a"), makeSpec("b")]); + expect(chain.members.map(m => m.spec.name)).toEqual(["a", "b"]); + }); + + it("returns the single member itself for a one-model chain", () => { + const chain = createChainModel([makeSpec("only")]); + expect(chain.model).toBe(chain.members[0]!.model); + }); +}); + +describe("createJudgeModels", () => { + it("keys judges by spec name", () => { + const judges = createJudgeModels([makeSpec("j1"), makeSpec("j2")]); + expect(Object.keys(judges)).toEqual(["j1", "j2"]); + }); +}); + +describe("chainLabel", () => { + it("joins names with arrows", () => { + expect(chainLabel([makeSpec("a"), makeSpec("b")])).toBe("a → b"); + }); +}); diff --git a/packages/cli/src/profiles/effectiveProfile.ts b/packages/cli/src/profiles/effectiveProfile.ts new file mode 100644 index 0000000..d0df97b --- /dev/null +++ b/packages/cli/src/profiles/effectiveProfile.ts @@ -0,0 +1,125 @@ +import {ModelSpec} from "@korabench/benchmark"; +import * as R from "remeda"; +import {resolveModelConfig} from "../models/modelConfig.js"; +import {EffectiveRoles, Profile, Role} from "./profile.js"; +import {Profiles} from "./profiles.js"; + +// +// Effective profile. +// +// The configured profile, with any per-role CLI override applied. Overrides +// resolve registry slugs into inline specs, so the effective roles are as +// self-describing as a file profile. Any override changes the profile hash: +// results from an overridden run must never look comparable to results from +// the named profile. +// + +export interface ProfileRef { + id: string; + version: string; + hash: string; + /** Loaded from an uncommitted `*.local.json` file. */ + local?: boolean; + /** Roles replaced on the command line. */ + overrides?: Role[]; +} + +export interface EffectiveProfile { + ref: ProfileRef; + roles: EffectiveRoles; +} + +/** Slugs given on the command line, per role. Single roles take one slug. */ +export type RoleOverrides = Partial>; + +const SINGLE_ROLES: ReadonlySet = new Set(["user", "continueUser"]); + +function resolveSpecs( + modelsJsonPath: string, + slugs: readonly string[] +): ModelSpec[] { + return slugs.map(slug => + ModelSpec.fromConfig(slug, resolveModelConfig(modelsJsonPath, slug)) + ); +} + +function applyOverride( + roles: EffectiveRoles, + role: Role, + specs: readonly ModelSpec[] +): EffectiveRoles { + if (SINGLE_ROLES.has(role)) { + if (specs.length !== 1) { + throw new Error( + `Role "${role}" takes exactly one model, got ${specs.length}: ${specs.map(s => s.name).join(", ")}.` + ); + } + return {...roles, [role]: specs[0]!}; + } + return {...roles, [role]: specs}; +} + +function warnOverride( + ref: Pick, + role: Role, + before: readonly ModelSpec[], + after: readonly ModelSpec[] +): void { + const names = (specs: readonly ModelSpec[]) => + specs.map(s => s.name).join(","); + console.error( + `WARNING: command-line ${role} overrides profile ${Profile.label(ref)} ` + + `(${names(before)} -> ${names(after)}). Results will carry an ad-hoc profile hash.` + ); +} + +export function resolveEffectiveProfile( + modelsJsonPath: string, + overrides: RoleOverrides = {} +): EffectiveProfile { + const {profile, local} = Profiles.current(); + const base = Profile.effectiveRoles(profile.roles); + + const overridden = Role.list.filter(role => { + const slugs = overrides[role]; + return slugs !== undefined && slugs.length > 0; + }); + + const roles = overridden.reduce((acc, role) => { + const specs = resolveSpecs(modelsJsonPath, overrides[role]!); + warnOverride(profile, role, Role.specsOf(acc, role), specs); + return applyOverride(acc, role, specs); + }, base); + + Profile.assertValidRoles(roles); + + const hash = + overridden.length === 0 + ? profile.hash + : Profile.computeHash({id: profile.id, version: profile.version, roles}); + + const ref: ProfileRef = { + id: profile.id, + version: profile.version, + hash, + ...(local ? {local} : {}), + ...(overridden.length > 0 ? {overrides: overridden} : {}), + }; + + return {ref, roles}; +} + +/** `id@version (hash)` plus local / override markers. */ +export function describeProfileRef(ref: ProfileRef): string { + const markers = R.pipe( + [ + ref.local ? "local" : undefined, + ref.overrides?.length + ? `overrides: ${ref.overrides.join(",")}` + : undefined, + ], + R.filter(R.isDefined) + ); + const suffix = markers.length > 0 ? ` [${markers.join("; ")}]` : ""; + return `${Profile.label(ref)} (${ref.hash})${suffix}`; +} diff --git a/packages/cli/src/profiles/loadProfile.ts b/packages/cli/src/profiles/loadProfile.ts new file mode 100644 index 0000000..1125c0d --- /dev/null +++ b/packages/cli/src/profiles/loadProfile.ts @@ -0,0 +1,93 @@ +import {existsSync, readdirSync, readFileSync} from "node:fs"; +import * as path from "node:path"; +import {Profile} from "./profile.js"; + +// +// Profile specs. +// +// A spec is either a profile name ("kora", "judge-test.local") resolved inside +// the profiles directory, or a path to a JSON file. Anything that looks like a +// path is treated as one; profile ids may not contain a path separator or end +// in ".json", so the two never overlap. +// +// Local profiles (`*.local.json`) are gitignored scratch files for testing a +// model configuration. They skip the hash check: nobody is expected to keep a +// throwaway file's hash current. +// + +const LOCAL_SUFFIX = ".local.json"; + +function isPath(spec: string): boolean { + return ( + spec.endsWith(".json") || spec.includes("/") || spec.includes(path.sep) + ); +} + +function isLocalPath(filePath: string): boolean { + return filePath.endsWith(LOCAL_SUFFIX); +} + +function readJson(filePath: string): unknown { + try { + return JSON.parse(readFileSync(filePath, "utf-8")); + } catch (error) { + throw new Error( + `Could not read profile from ${filePath}: ${(error as Error).message}`, + {cause: error} + ); + } +} + +// +// API. +// + +/** The profiles directory sits next to `models.json`. */ +export function profilesDir(modelsJsonPath: string): string { + return path.join(path.dirname(modelsJsonPath), "profiles"); +} + +/** Committed profile names in `dir` (local profiles excluded). */ +export function listProfileNames(dir: string): readonly string[] { + if (!existsSync(dir)) return []; + return readdirSync(dir) + .filter(file => file.endsWith(".json") && !isLocalPath(file)) + .map(file => file.slice(0, -".json".length)) + .sort(); +} + +export interface LoadedProfile { + profile: Profile; + /** Loaded from a `*.local.json` file: uncommitted, hash unchecked. */ + local: boolean; + path: string; +} + +export interface LoadProfileOptions { + /** Defaults to `true` for committed profiles and `false` for local ones. */ + verifyHash?: boolean; +} + +export function loadProfile( + spec: string, + dir: string, + options: LoadProfileOptions = {} +): LoadedProfile { + const filePath = isPath(spec) + ? path.resolve(process.cwd(), spec) + : path.join(dir, `${spec}.json`); + + if (!isPath(spec) && !existsSync(filePath)) { + const known = listProfileNames(dir); + throw new Error( + `Unknown profile "${spec}". Known profiles: ${known.length > 0 ? known.join(", ") : "(none)"} (in ${dir}). ` + + `Pass a path to a JSON file, or a ".local" profile stored as ${dir}/${LOCAL_SUFFIX}.` + ); + } + + const local = isLocalPath(filePath); + const profile = Profile.parse(readJson(filePath), { + verifyHash: options.verifyHash ?? !local, + }); + return {profile, local, path: filePath}; +} diff --git a/packages/cli/src/profiles/printProfile.ts b/packages/cli/src/profiles/printProfile.ts new file mode 100644 index 0000000..642040d --- /dev/null +++ b/packages/cli/src/profiles/printProfile.ts @@ -0,0 +1,118 @@ +import {Mechanism, ModelSpec, Packs, RiskTaxonomy} from "@korabench/benchmark"; +import * as R from "remeda"; +import {describeProfileRef, EffectiveProfile} from "./effectiveProfile.js"; +import {Role} from "./profile.js"; +import {createSpecModel} from "./roleModels.js"; + +// +// Reporting. +// + +function formatSpec(spec: ModelSpec): string { + return `${spec.name} ${JSON.stringify(ModelSpec.config(spec))}`; +} + +export function printPacks(): void { + const {taxonomy, behaviors} = Packs.current(); + const stamp = Packs.fingerprint(); + + console.log( + `Taxonomy: ${RiskTaxonomy.label(taxonomy)} (${stamp.taxonomy.hash})` + ); + console.log( + ` ${taxonomy.categories.length} categories, ${RiskTaxonomy.allRisks(taxonomy).length} risks` + ); + console.log( + `Behaviors: ${behaviors.id}@${behaviors.version} (${stamp.behaviors.hash})` + ); + console.log( + ` ${behaviors.behaviors.map(Mechanism.codeOf).join(", ")}` + ); +} + +/** One line per role, full config per model, so the printout is a record. */ +export function printProfile(effective: EffectiveProfile, path?: string): void { + console.log(`Profile: ${describeProfileRef(effective.ref)}`); + if (path) { + console.log(` ${path}`); + } + Role.list.forEach(role => { + const specs = Role.specsOf(effective.roles, role); + specs.forEach((spec, index) => { + const head = index === 0 ? `${role}:`.padEnd(15) : "".padEnd(15); + console.log(` ${head}${formatSpec(spec)}`); + }); + }); + console.log(""); + printPacks(); +} + +// +// Live check. +// + +interface CheckOutcome { + spec: ModelSpec; + ok: boolean; + served?: string; + durationMs: number; + error?: string; +} + +async function checkSpec(spec: ModelSpec): Promise { + const model = createSpecModel(spec, {retry: {maxRetries: 0}}); + const started = Date.now(); + try { + await model.getTextResponse({ + messages: [{role: "user", content: "Reply with the single word OK."}], + maxTokens: 16, + }); + return { + spec, + ok: true, + served: [...model.served].join(","), + durationMs: Date.now() - started, + }; + } catch (error) { + return { + spec, + ok: false, + durationMs: Date.now() - started, + error: error instanceof Error ? error.message : String(error), + }; + } +} + +/** + * Call every distinct model of the profile once. Returns `true` when every + * call succeeded. Identical specs used by several roles are called once. + */ +export async function checkProfileModels( + effective: EffectiveProfile +): Promise { + const specs = R.pipe( + Role.list, + R.flatMap(role => Role.specsOf(effective.roles, role)), + R.uniqueBy(spec => JSON.stringify(spec)) + ); + console.log(`Checking ${specs.length} model(s)...`); + + const outcomes = await Promise.all(specs.map(checkSpec)); + outcomes.forEach(outcome => { + const status = outcome.ok ? "PASS" : "FAIL"; + const detail = outcome.ok + ? `served=${outcome.served}` + : (outcome.error ?? "").slice(0, 200); + console.log( + ` ${status} ${outcome.spec.name.padEnd(28)} ${outcome.spec.model.padEnd(32)} ${String(outcome.durationMs).padStart(6)}ms ${detail}` + ); + }); + + const failed = outcomes.filter(outcome => !outcome.ok).length; + console.log( + failed === 0 + ? `All ${outcomes.length} model(s) answered.` + : `${failed} of ${outcomes.length} model(s) failed.` + ); + return failed === 0; +} diff --git a/packages/cli/src/profiles/profile.ts b/packages/cli/src/profiles/profile.ts new file mode 100644 index 0000000..bcbc793 --- /dev/null +++ b/packages/cli/src/profiles/profile.ts @@ -0,0 +1,157 @@ +import {ModelSpec, PackId, stableJson} from "@korabench/benchmark"; +import {Hash} from "@korabench/core"; +import * as R from "remeda"; +import * as v from "valibot"; + +// +// Runtime model. +// +// An evaluation profile pins the LLM used for every role of the pipeline. +// Configs are inline (not slugs into models.json) so a profile file is a +// complete, self-describing record of what ran. The target model is never part +// of a profile: it is the subject of the evaluation, not the harness. +// + +const VChain = v.pipe(v.array(ModelSpec.io), v.minLength(1)); + +const VProfileRoles = v.strictObject({ + /** Seed generation (`generate-seeds`). Fallback chain. */ + seeds: VChain, + /** Scenario expansion and validation (`expand-scenarios`). Fallback chain. */ + expansion: VChain, + /** First user message during expansion. Fallback chain. */ + expansionUser: VChain, + /** User simulator during `run` (and `reassess` label). */ + user: ModelSpec.io, + /** Concurrent judges (`run`, `reassess`, `continue`). Odd count. */ + judges: VChain, + /** User simulator during `continue`; falls back to `user`. */ + continueUser: v.optional(ModelSpec.io), +}); + +const VProfile = v.strictObject({ + id: PackId.io, + version: v.pipe(v.string(), v.minLength(1)), + /** `computeHash()` of the rest of the file; guarded by a test in CI. */ + hash: v.string(), + roles: VProfileRoles, +}); + +const ROLE_LIST = [ + "seeds", + "expansion", + "expansionUser", + "user", + "judges", + "continueUser", +] as const; + +export const ODD_JUDGES_MESSAGE = + "The current implementation only supports odd numbers of judges. This ensures that the median assessment is always defined. See `aggregateTestAssessments` for reference."; + +// +// API. +// + +function computeHash( + profile: Pick +): string { + return Hash.shortHash( + stableJson({ + id: profile.id, + version: profile.version, + roles: profile.roles, + }) + ); +} + +function label(profile: Pick): string { + return `${profile.id}@${profile.version}`; +} + +function specsOf(roles: ProfileRoles, role: Role): readonly ModelSpec[] { + const value = roles[role]; + if (value === undefined) return []; + return Array.isArray(value) ? value : [value]; +} + +function assertUniqueNames(roles: ProfileRoles): void { + ROLE_LIST.forEach(role => { + const names = specsOf(roles, role).map(spec => spec.name); + const duplicates = R.pipe( + names, + R.groupBy(R.identity()), + R.pickBy(group => group.length > 1), + R.keys() + ); + if (duplicates.length > 0) { + throw new Error( + `Profile role "${role}" lists the same model name more than once: ${duplicates.join(", ")}.` + ); + } + }); +} + +/** Structural checks shared by file profiles and override-derived ones. */ +function assertValidRoles(roles: ProfileRoles): void { + if (roles.judges.length % 2 === 0) { + throw new Error(ODD_JUDGES_MESSAGE); + } + assertUniqueNames(roles); +} + +export interface ParseOptions { + /** Refuse a profile whose declared `hash` does not match its content. */ + verifyHash: boolean; +} + +function parse(data: unknown, options: ParseOptions): Profile { + const profile = v.parse(VProfile, data); + assertValidRoles(profile.roles); + if (options.verifyHash) { + const expected = computeHash(profile); + if (profile.hash !== expected) { + throw new Error( + `Profile "${label(profile)}" declares hash "${profile.hash}" but its content hashes to "${expected}". ` + + `Bump "version" and set "hash" to "${expected}" ` + + `(\`yarn kora --profile profile --print-hash\` prints it).` + ); + } + } + return profile; +} + +/** Every role filled in: `continueUser` defaults to `user`. */ +function effectiveRoles(roles: ProfileRoles): EffectiveRoles { + return { + ...roles, + continueUser: roles.continueUser ?? roles.user, + }; +} + +// +// Exports. +// + +export type Role = (typeof ROLE_LIST)[number]; + +export interface ProfileRoles extends v.InferOutput {} + +/** `ProfileRoles` with every optional role resolved. */ +export type EffectiveRoles = Required; + +export interface Profile extends v.InferOutput {} + +export const Role = { + list: ROLE_LIST, + specsOf, +}; + +export const Profile = { + io: VProfile, + parse, + computeHash, + label, + effectiveRoles, + assertValidRoles, +}; diff --git a/packages/cli/src/profiles/profiles.ts b/packages/cli/src/profiles/profiles.ts new file mode 100644 index 0000000..347561f --- /dev/null +++ b/packages/cli/src/profiles/profiles.ts @@ -0,0 +1,61 @@ +import {LoadedProfile} from "./loadProfile.js"; +import {Profile} from "./profile.js"; + +// +// State. +// +// Process-wide and one-shot, like `Packs.configure()`: the CLI resolves +// `--profile` once per invocation in its preAction hook and every command +// reads it. Commands never read this at module scope (commander defaults must +// stay static strings), only inside their bodies. +// + +let configured: LoadedProfile | undefined; + +// +// API. +// + +function sameProfile(a: LoadedProfile, b: LoadedProfile): boolean { + return a.profile.hash === b.profile.hash && a.local === b.local; +} + +/** + * Set the process-wide profile. Idempotent for an identical profile; a second + * call with different content throws rather than silently re-pointing models + * that may already have been built against the first. + */ +function configure(loaded: LoadedProfile): void { + if (configured) { + if (sameProfile(configured, loaded)) return; + throw new Error( + "Profiles.configure() called twice with different profiles " + + `(${Profile.label(configured.profile)} then ${Profile.label(loaded.profile)}).` + ); + } + configured = loaded; +} + +function current(): LoadedProfile { + if (!configured) { + throw new Error( + "No evaluation profile configured. The CLI configures one from --profile in its preAction hook." + ); + } + return configured; +} + +/** Test-only: drop the process-wide configuration. */ +function reset(): void { + configured = undefined; +} + +// +// Exports. +// + +export const Profiles = { + configure, + current, + reset, +}; diff --git a/packages/cli/src/profiles/roleModels.ts b/packages/cli/src/profiles/roleModels.ts new file mode 100644 index 0000000..fc8f92f --- /dev/null +++ b/packages/cli/src/profiles/roleModels.ts @@ -0,0 +1,104 @@ +import {ModelSpec} from "@korabench/benchmark"; +import {createFallbackModel} from "../models/fallbackModel.js"; +import { + createGatewayModelFromConfig, + GatewayModel, + ModelOptions, +} from "../models/gatewayModel.js"; +import {Model} from "../models/model.js"; + +// +// Models for profile roles. +// +// Every role of a profile is a list of `ModelSpec`s; these helpers turn them +// into the `Model` shapes the commands consume, keeping the spec `name` as the +// label used in logs and result headers. +// + +export interface ChainMember { + spec: ModelSpec; + model: GatewayModel; +} + +export interface ChainModel { + /** Fallback chain over `members`, in order. */ + model: Model; + members: readonly ChainMember[]; +} + +export function createSpecModel( + spec: ModelSpec, + options?: ModelOptions +): GatewayModel { + return createGatewayModelFromConfig( + ModelSpec.config(spec), + spec.name, + options + ); +} + +export function createChainModel( + specs: readonly ModelSpec[], + options?: ModelOptions +): ChainModel { + const members = specs.map(spec => ({ + spec, + model: createSpecModel(spec, options), + })); + return { + model: createFallbackModel( + members.map(({spec, model}) => ({label: spec.name, model})) + ), + members, + }; +} + +/** Judges keyed by spec name: the key is what `runJudges` reports per judge. */ +export function createJudgeModels( + specs: readonly ModelSpec[], + options?: ModelOptions +): Record { + return Object.fromEntries( + specs.map(spec => [spec.name, createSpecModel(spec, options)]) + ); +} + +export function chainLabel(specs: readonly ModelSpec[]): string { + return specs.map(spec => spec.name).join(" → "); +} + +// +// Served model ids. +// + +/** Provider-reported model ids per role, sorted, as written to results. */ +export interface ServedModels { + user?: readonly string[]; + judges: Record; + target?: readonly string[]; +} + +export function servedOf( + model: GatewayModel | undefined +): string[] | undefined { + return model ? [...model.served].sort() : undefined; +} + +export function collectServed(args: { + user?: GatewayModel; + judges: Record; + target?: GatewayModel; +}): ServedModels { + const user = servedOf(args.user); + const target = servedOf(args.target); + return { + ...(user ? {user} : {}), + judges: Object.fromEntries( + Object.entries(args.judges).map(([name, model]) => [ + name, + servedOf(model)!, + ]) + ), + ...(target ? {target} : {}), + }; +} diff --git a/packages/cli/src/shared/packageVersion.ts b/packages/cli/src/shared/packageVersion.ts new file mode 100644 index 0000000..f3b4d9b --- /dev/null +++ b/packages/cli/src/shared/packageVersion.ts @@ -0,0 +1,23 @@ +import {existsSync, readFileSync} from "node:fs"; +import {dirname, join} from "node:path"; +import {fileURLToPath} from "node:url"; + +function nearestPackageJson(from: string): string { + const candidate = join(from, "package.json"); + if (existsSync(candidate)) return candidate; + const parent = dirname(from); + if (parent === from) { + throw new Error(`No package.json above ${from}.`); + } + return nearestPackageJson(parent); +} + +/** + * Version of the `@korabench/cli` package this code belongs to. Resolved by + * walking up from this module, so it works from `build/` and from `src/`. + */ +export function readPackageVersion(): string { + const pkgPath = nearestPackageJson(dirname(fileURLToPath(import.meta.url))); + const pkg = JSON.parse(readFileSync(pkgPath, "utf-8")); + return pkg.version || "0.0.0"; +} diff --git a/packages/cli/src/shared/sha256File.ts b/packages/cli/src/shared/sha256File.ts new file mode 100644 index 0000000..7039273 --- /dev/null +++ b/packages/cli/src/shared/sha256File.ts @@ -0,0 +1,7 @@ +import {createHash} from "node:crypto"; +import * as fs from "node:fs/promises"; + +export async function sha256File(filePath: string): Promise { + const buf = await fs.readFile(filePath); + return createHash("sha256").update(buf).digest("hex"); +} diff --git a/packages/cli/src/stamp/__tests__/buildRunStamp.test.ts b/packages/cli/src/stamp/__tests__/buildRunStamp.test.ts new file mode 100644 index 0000000..f11e0b9 --- /dev/null +++ b/packages/cli/src/stamp/__tests__/buildRunStamp.test.ts @@ -0,0 +1,74 @@ +import {Packs, Prompts, RunStamp} from "@korabench/benchmark"; +import {mkdtempSync, writeFileSync} from "node:fs"; +import {tmpdir} from "node:os"; +import * as path from "node:path"; +import * as v from "valibot"; +import {describe, expect, it} from "vitest"; +import {makeRoles} from "../../profiles/__tests__/fixtures.js"; +import {EffectiveProfile} from "../../profiles/effectiveProfile.js"; +import {Profile} from "../../profiles/profile.js"; +import {buildRunStamp, resolveTargetRef} from "../buildRunStamp.js"; + +const dir = mkdtempSync(path.join(tmpdir(), "kora-stamp-")); +const modelsJsonPath = path.join(dir, "models.json"); +writeFileSync( + modelsJsonPath, + JSON.stringify({"gpt-x": {model: "openai/gpt-x", maxTokens: 10}}) +); +const inputPath = path.join(dir, "input.jsonl"); +writeFileSync(inputPath, "{}\n"); + +const effective: EffectiveProfile = { + ref: {id: "test", version: "1", hash: "h"}, + roles: Profile.effectiveRoles(makeRoles()), +}; + +describe("resolveTargetRef", () => { + it("resolves gateway slugs to a spec", () => { + expect(resolveTargetRef(modelsJsonPath, "gpt-x")).toEqual({ + name: "gpt-x", + model: "openai/gpt-x", + maxTokens: 10, + }); + }); + + it.each([ + ["kora-app-foo", "web-runner"], + ["kora-app-foo-android", "native-runner"], + ["kora-app-foo-ios", "native-runner"], + ["custom-thing", "custom"], + ])("maps %s to a %s reference", (slug, kind) => { + expect(resolveTargetRef(modelsJsonPath, slug)).toEqual({kind, slug}); + }); +}); + +describe("buildRunStamp", () => { + it("records profile, roles, prompts, packs, code and input", async () => { + const stamp = await buildRunStamp({ + effective, + modelsJsonPath, + target: "kora-app-foo", + inputPath, + }); + expect(v.parse(RunStamp.io, stamp)).toEqual(stamp); + expect(stamp.profile).toEqual(effective.ref); + expect(stamp.models.user).toEqual(effective.roles.user); + expect(stamp.models.target).toEqual({ + kind: "web-runner", + slug: "kora-app-foo", + }); + expect(stamp.prompts).toEqual(Prompts.fingerprint()); + expect(stamp.packs).toEqual(Packs.fingerprint()); + expect(stamp.code.version).toMatch(/^\d+\.\d+\.\d+/); + expect(stamp.input).toEqual({ + path: inputPath, + sha256: expect.stringMatching(/^[0-9a-f]{64}$/), + }); + }); + + it("omits target and input when not given", async () => { + const stamp = await buildRunStamp({effective, modelsJsonPath}); + expect("target" in stamp.models).toBe(false); + expect("input" in stamp).toBe(false); + }); +}); diff --git a/packages/cli/src/stamp/__tests__/fixtures.ts b/packages/cli/src/stamp/__tests__/fixtures.ts new file mode 100644 index 0000000..246b75b --- /dev/null +++ b/packages/cli/src/stamp/__tests__/fixtures.ts @@ -0,0 +1,28 @@ +import {ModelSpec, Packs, RunStamp} from "@korabench/benchmark"; + +// +// Test fixtures. +// + +function spec(name: string): ModelSpec { + return {name, model: `provider/${name}`}; +} + +/** A complete stamp under the bundled packs. Override any field. */ +export function makeStamp(overrides: Partial = {}): RunStamp { + return { + profile: {id: "test", version: "1", hash: "profile-hash"}, + models: { + seeds: [spec("seed")], + expansion: [spec("expand")], + expansionUser: [spec("user")], + user: spec("user"), + judges: [spec("judge")], + continueUser: spec("user"), + }, + prompts: {version: "1", hash: "prompts-hash"}, + code: {version: "1.0.0", commit: "abc", dirty: false}, + packs: Packs.fingerprint(), + ...overrides, + }; +} diff --git a/packages/cli/src/stamp/__tests__/gitInfo.test.ts b/packages/cli/src/stamp/__tests__/gitInfo.test.ts new file mode 100644 index 0000000..d6d9284 --- /dev/null +++ b/packages/cli/src/stamp/__tests__/gitInfo.test.ts @@ -0,0 +1,14 @@ +import {describe, expect, it} from "vitest"; +import {readGitInfo} from "../gitInfo.js"; + +describe("readGitInfo", () => { + it("returns a commit sha and dirty flag inside a checkout, or nothing", () => { + const info = readGitInfo(); + if (info.commit === undefined) { + expect(info).toEqual({}); + return; + } + expect(info.commit).toMatch(/^[0-9a-f]{40}$/); + expect(typeof info.dirty).toBe("boolean"); + }); +}); diff --git a/packages/cli/src/stamp/buildRunStamp.ts b/packages/cli/src/stamp/buildRunStamp.ts new file mode 100644 index 0000000..3328e79 --- /dev/null +++ b/packages/cli/src/stamp/buildRunStamp.ts @@ -0,0 +1,60 @@ +import { + ModelSpec, + Packs, + Prompts, + RunStamp, + TargetRef, +} from "@korabench/benchmark"; +import {resolveModelConfig} from "../models/modelConfig.js"; +import {isNativeRunnerSlug} from "../models/nativeRunnerModel.js"; +import {isWebRunnerSlug} from "../models/webRunnerModel.js"; +import {EffectiveProfile} from "../profiles/effectiveProfile.js"; +import {readPackageVersion} from "../shared/packageVersion.js"; +import {sha256File} from "../shared/sha256File.js"; +import {readGitInfo} from "./gitInfo.js"; + +// +// Stamp construction. +// + +export interface BuildRunStampArgs { + effective: EffectiveProfile; + modelsJsonPath: string; + /** Target slug for `run`; resolved to a spec or a runner reference. */ + target?: string; + /** Input corpus, fingerprinted so results name what they were computed on. */ + inputPath?: string; +} + +export function resolveTargetRef( + modelsJsonPath: string, + slug: string +): TargetRef { + if (slug.startsWith("custom-")) return {kind: "custom", slug}; + if (isNativeRunnerSlug(slug)) return {kind: "native-runner", slug}; + if (isWebRunnerSlug(slug)) return {kind: "web-runner", slug}; + return ModelSpec.fromConfig(slug, resolveModelConfig(modelsJsonPath, slug)); +} + +export async function buildRunStamp( + args: BuildRunStampArgs +): Promise { + const {effective, modelsJsonPath, target, inputPath} = args; + const targetRef = + target === undefined + ? {} + : {target: resolveTargetRef(modelsJsonPath, target)}; + const input = + inputPath === undefined + ? {} + : {input: {path: inputPath, sha256: await sha256File(inputPath)}}; + + return { + profile: effective.ref, + models: {...effective.roles, ...targetRef}, + prompts: Prompts.fingerprint(), + code: {version: readPackageVersion(), ...readGitInfo()}, + packs: Packs.fingerprint(), + ...input, + }; +} diff --git a/packages/cli/src/stamp/gitInfo.ts b/packages/cli/src/stamp/gitInfo.ts new file mode 100644 index 0000000..2feb4f5 --- /dev/null +++ b/packages/cli/src/stamp/gitInfo.ts @@ -0,0 +1,28 @@ +import {execFileSync} from "node:child_process"; + +export interface GitInfo { + commit?: string; + dirty?: boolean; +} + +function git(args: readonly string[]): string { + return execFileSync("git", [...args], { + encoding: "utf-8", + stdio: ["ignore", "pipe", "ignore"], + }).trim(); +} + +/** + * Commit and dirty flag of the working tree the CLI runs in. Both are absent + * outside a git checkout (an installed package, a container without git): the + * stamp then still carries the package version. + */ +export function readGitInfo(): GitInfo { + try { + const commit = git(["rev-parse", "HEAD"]); + const dirty = git(["status", "--porcelain"]).length > 0; + return {commit, dirty}; + } catch { + return {}; + } +} diff --git a/profiles/example.local.json.example b/profiles/example.local.json.example new file mode 100644 index 0000000..a905483 --- /dev/null +++ b/profiles/example.local.json.example @@ -0,0 +1,56 @@ +{ + "id": "example", + "version": "1", + "hash": "unchecked-for-local-profiles", + "roles": { + "seeds": [ + { + "name": "gpt-4o", + "model": "openai/gpt-4o" + } + ], + "expansion": [ + { + "name": "gpt-5.2:high", + "model": "openai/gpt-5.2", + "providerOptions": { + "openai": { + "reasoningEffort": "high" + } + } + } + ], + "expansionUser": [ + { + "name": "deepseek-v3.2", + "model": "deepseek/deepseek-v3.2", + "maxTokens": 4000, + "temperature": 1.3 + } + ], + "user": { + "name": "deepseek-v3.2", + "model": "deepseek/deepseek-v3.2", + "maxTokens": 4000, + "temperature": 1.3 + }, + "judges": [ + { + "name": "gpt-5.2:medium:limited", + "model": "openai/gpt-5.2", + "maxTokens": 26000, + "providerOptions": { + "openai": { + "reasoningEffort": "medium" + } + } + } + ], + "continueUser": { + "name": "deepseek-v3.2-temp-1.3", + "model": "deepseek/deepseek-v3.2", + "maxTokens": 4000, + "temperature": 1.3 + } + } +} diff --git a/profiles/kora.json b/profiles/kora.json new file mode 100644 index 0000000..f7660d2 --- /dev/null +++ b/profiles/kora.json @@ -0,0 +1,56 @@ +{ + "id": "kora", + "version": "1", + "hash": "0b7b93d2b3ca55a333ca687b996489d2", + "roles": { + "seeds": [ + { + "name": "gpt-4o", + "model": "openai/gpt-4o" + } + ], + "expansion": [ + { + "name": "gpt-5.2:high", + "model": "openai/gpt-5.2", + "providerOptions": { + "openai": { + "reasoningEffort": "high" + } + } + } + ], + "expansionUser": [ + { + "name": "deepseek-v3.2", + "model": "deepseek/deepseek-v3.2", + "maxTokens": 4000, + "temperature": 1.3 + } + ], + "user": { + "name": "deepseek-v3.2", + "model": "deepseek/deepseek-v3.2", + "maxTokens": 4000, + "temperature": 1.3 + }, + "judges": [ + { + "name": "gpt-5.2:medium:limited", + "model": "openai/gpt-5.2", + "maxTokens": 26000, + "providerOptions": { + "openai": { + "reasoningEffort": "medium" + } + } + } + ], + "continueUser": { + "name": "deepseek-v3.2-temp-1.3", + "model": "deepseek/deepseek-v3.2", + "maxTokens": 4000, + "temperature": 1.3 + } + } +} diff --git a/scripts/README.md b/scripts/README.md index f031d6d..32abd2d 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,6 +8,12 @@ Both scripts import the **built** packages (`packages/*/build/...`), so run `yarn build` (or `yarn tsbuild`) first, and pass the gateway/runner env with `node --env-file=.env`. They read `models.json` from the repo root. +They predate evaluation profiles: models come from `JUDGE` / `USER_MODEL` and +`models.json`, and the temp files they write carry no run stamp. The follow-up +`kora run` accepts such a directory with a warning (`without a stamp`) and +stamps the aggregated result with its own configuration. Adapting the scripts +to load a profile and stamp their records is a follow-up. + ## `manual-rerun.mjs` — collect conversations, human-in-the-loop Drives one scenario at a time: prints the user (child) turn, you paste the app's From e6ff6737ed2cd96e6c9aa1605ad0b02766d40e07 Mon Sep 17 00:00:00 2001 From: Thibaut Fatus Date: Fri, 4 Sep 2026 16:39:48 +0200 Subject: [PATCH 2/5] [feat] operator scripts load the evaluation profile and stamp their results complete-run.mjs and manual-rerun.mjs take judges and the user simulator from the evaluation profile (KORA_PROFILE) like the CLI; JUDGE / USER_MODEL remain as warned overrides. complete-run builds the run stamp, goes through the same resume guard as `kora run` against .kora-run-tmp/stamp.json, and names the target from the cached stamp (or TARGET). manual-rerun records the user spec on each transcript entry. Claude-Session: https://claude.ai/code/session_017SeRjc1yMoa617ZR2bxeE8 --- scripts/README.md | 19 +++++--- scripts/complete-run.mjs | 102 ++++++++++++++++++++++++++++++--------- scripts/manual-rerun.mjs | 30 ++++++++++-- 3 files changed, 117 insertions(+), 34 deletions(-) diff --git a/scripts/README.md b/scripts/README.md index 32abd2d..bad6d06 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -8,11 +8,10 @@ Both scripts import the **built** packages (`packages/*/build/...`), so run `yarn build` (or `yarn tsbuild`) first, and pass the gateway/runner env with `node --env-file=.env`. They read `models.json` from the repo root. -They predate evaluation profiles: models come from `JUDGE` / `USER_MODEL` and -`models.json`, and the temp files they write carry no run stamp. The follow-up -`kora run` accepts such a directory with a warning (`without a stamp`) and -stamps the aggregated result with its own configuration. Adapting the scripts -to load a profile and stamp their records is a follow-up. +Models come from the evaluation profile (`KORA_PROFILE`, default `kora`), +exactly as in the CLI; `JUDGE` / `USER_MODEL` are overrides on top of it and +are warned and stamped like `--judges` / `[user-model]`. See the README's +"Evaluation profiles". ## `manual-rerun.mjs` — collect conversations, human-in-the-loop @@ -26,7 +25,7 @@ Seed `RUN_DIR/manual-reruns.json` with one entry per scenario: `{scenario, messages: [{role: "user", content: }]}`. ```sh -RUN_DIR=data/ node --env-file=.env scripts/manual-rerun.mjs [assistantFile] +RUN_DIR=data/ [USER_MODEL=] node --env-file=.env scripts/manual-rerun.mjs [assistantFile] # 1-based scenario index into manual-reruns.json # [assistantFile] file with the pasted app reply; omit to (re)print the # pending user message @@ -40,7 +39,13 @@ overwrites the matching `RUN_DIR/.kora-run-tmp/.json` (matched by `scenario.seed.id`). Re-run `kora run -o RUN_DIR/results.json` afterwards to cache-aggregate every result into the final `results.json` + `.zip`. +The results are stamped, and the script goes through the same resume guard as +`kora run`: if `.kora-run-tmp/stamp.json` was written under a different +configuration (other judges, prompts or packs) it refuses to proceed. `TARGET` +names the run's target in the stamp; it defaults to the one in the temp dir's +`stamp.json` when present. + ```sh -RUN_DIR=data/ [JUDGE=gpt-5.2:medium:limited] \ +RUN_DIR=data/ [JUDGE=[,…]] [TARGET=] \ node --env-file=.env scripts/complete-run.mjs ``` diff --git a/scripts/complete-run.mjs b/scripts/complete-run.mjs index ce66ef5..533d663 100644 --- a/scripts/complete-run.mjs +++ b/scripts/complete-run.mjs @@ -9,25 +9,72 @@ * (matched by scenario.seed.id) under .kora-run-tmp so a subsequent * `kora run` cache-aggregates every result into the final results.json + .zip. * + * Judges come from the evaluation profile (KORA_PROFILE, default "kora"), + * exactly as in `kora run`; JUDGE is an override, warned and stamped like the + * CLI's --judges. The run stamp is built from the profile and checked against + * the temp dir's stamp.json the same way `kora run` does, so the manual + * completion cannot silently use different judges than the automated part. + * * Prereqts: packages are built (`yarn build`/`tsbuild`); models.json present. * Usage: - * RUN_DIR=data/ [JUDGE=gpt-5.2:medium:limited] \ + * RUN_DIR=data/ [KORA_PROFILE=kora] [JUDGE=[,…]] [TARGET=] \ * node --env-file=.env scripts/complete-run.mjs * (RUN_DIR must contain manual-reruns.json and a .kora-run-tmp/ with the - * other cached results; re-run `kora run -o /results.json` after.) + * other cached results; re-run `kora run -o /results.json` after. + * TARGET is the run's target slug, recorded in the stamp; defaults to the + * one in the temp dir's stamp.json when present.) */ -import {readFileSync, readdirSync, writeFileSync} from "node:fs"; +import {existsSync, readFileSync, writeFileSync} from "node:fs"; import path from "node:path"; -import {kora} from "../packages/benchmark/build/src/index.js"; -import {createGatewayModel} from "../packages/cli/build/src/models/gatewayModel.js"; +import {kora, Stamp} from "../packages/benchmark/build/src/index.js"; +import { + assertResumable, + listCachedFiles, + STAMP_FILE, +} from "../packages/cli/build/src/commands/shared/cacheStamp.js"; +import { + describeProfileRef, + resolveEffectiveProfile, +} from "../packages/cli/build/src/profiles/effectiveProfile.js"; +import { + loadProfile, + profilesDir, +} from "../packages/cli/build/src/profiles/loadProfile.js"; +import {Profiles} from "../packages/cli/build/src/profiles/profiles.js"; +import {createJudgeModels} from "../packages/cli/build/src/profiles/roleModels.js"; +import {buildRunStamp} from "../packages/cli/build/src/stamp/buildRunStamp.js"; const DIR = process.env.RUN_DIR ?? "data/2026-06-10-gemini-104"; const TMP = path.join(DIR, ".kora-run-tmp"); const STORE = path.join(DIR, "manual-reruns.json"); -const JUDGE = process.env.JUDGE ?? "gpt-5.2:medium:limited"; const modelsJsonPath = path.resolve("models.json"); -const judgeModel = createGatewayModel(modelsJsonPath, JUDGE); +Profiles.configure( + loadProfile(process.env.KORA_PROFILE ?? "kora", profilesDir(modelsJsonPath)) +); +const effective = resolveEffectiveProfile(modelsJsonPath, { + judges: process.env.JUDGE?.split(",").map(s => s.trim()), +}); +console.log(`Profile: ${describeProfileRef(effective.ref)}`); + +// The target is not part of the profile; take it from the run's own stamp +// when the automated part left one, else from TARGET. +const cachedStampPath = path.join(TMP, STAMP_FILE); +const cachedTarget = existsSync(cachedStampPath) + ? JSON.parse(readFileSync(cachedStampPath, "utf8"))?.models?.target + : undefined; +const target = process.env.TARGET ?? cachedTarget?.slug ?? cachedTarget?.name; +if (!target) { + console.error( + "WARNING: no TARGET given and no stamp.json in the temp dir; the stamp will not name the target." + ); +} + +const stamp = await buildRunStamp({effective, modelsJsonPath, target}); +await assertResumable(TMP, stamp); +Stamp.configure(stamp); + +const judgeModels = createJudgeModels(effective.roles.judges); // Context: only judgeModels is exercised (the conversation loop is skipped). const ctx = { @@ -37,18 +84,22 @@ const ctx = { getAssistantResponse: async () => { throw new Error("assistant model must not be called"); }, - judgeModels: { - [JUDGE]: { - getResponse: async request => ({ - output: await judgeModel.getStructuredResponse(request), - }), - }, - }, + judgeModels: Object.fromEntries( + Object.entries(judgeModels).map(([name, model]) => [ + name, + { + getResponse: async request => ({ + output: await model.getStructuredResponse(request), + }), + }, + ]) + ), }; -// Map seed.id -> temp filename for the existing 104 results. +// Map seed.id -> temp filename for the existing results. const seedToFile = {}; -for (const f of readdirSync(TMP).filter(n => n.endsWith(".json"))) { +for (const f of await listCachedFiles(TMP)) { + if (!f.endsWith(".json")) continue; const d = JSON.parse(readFileSync(path.join(TMP, f), "utf8")); const id = d?.scenario?.seed?.id; if (id) seedToFile[id] = f; @@ -59,18 +110,23 @@ const store = JSON.parse(readFileSync(STORE, "utf8")); for (const entry of store) { const seedId = entry.scenario.seed.id; const file = seedToFile[seedId]; - if (!file) throw new Error(`No temp file for seed ${seedId} (${entry.title})`); + if (!file) + throw new Error(`No temp file for seed ${seedId} (${entry.title})`); const key = kora.mapScenarioToKeys(entry.scenario, ["default"])[0]; - const testResult = await kora.runTest(ctx, entry.scenario, key, entry.messages); - - writeFileSync( - path.join(TMP, file), - JSON.stringify(testResult, null, 2) + const testResult = await kora.runTest( + ctx, + entry.scenario, + key, + entry.messages ); + writeFileSync(path.join(TMP, file), JSON.stringify(testResult, null, 2)); + const grade = testResult?.assessment?.grade ?? "?"; console.log(`✓ ${entry.title} → ${file} [grade: ${grade}]`); } -console.log(`\nRe-judged ${store.length} scenarios. Now run \`kora run\` to aggregate.`); +console.log( + `\nRe-judged ${store.length} scenarios. Now run \`kora run\` to aggregate.` +); diff --git a/scripts/manual-rerun.mjs b/scripts/manual-rerun.mjs index d5a7ff7..f2bde1c 100644 --- a/scripts/manual-rerun.mjs +++ b/scripts/manual-rerun.mjs @@ -10,9 +10,14 @@ * The transcript store (RUN_DIR/manual-reruns.json) is seeded by the operator, * one entry per scenario: {scenario, messages:[{role:"user",content:first}]}. * + * The user simulator comes from the evaluation profile's `user` role + * (KORA_PROFILE, default "kora"), exactly as in `kora run`; USER_MODEL is an + * override, warned like the CLI's [user-model]. The spec actually used is + * recorded on each store entry (`userModel`) for provenance. + * * Prereqts: packages are built (`yarn build`/`tsbuild`); models.json present. * Usage: - * RUN_DIR=data/ [USER_MODEL=deepseek-v3.2] \ + * RUN_DIR=data/ [KORA_PROFILE=kora] [USER_MODEL=] \ * node --env-file=.env scripts/manual-rerun.mjs [assistantFile] * 1-based scenario index into RUN_DIR/manual-reruns.json * [assistantFile] file with the pasted app reply for the current turn; @@ -24,12 +29,20 @@ import { generateNextUserMessage, RiskCategory, } from "../packages/benchmark/build/src/index.js"; -import {createGatewayModel} from "../packages/cli/build/src/models/gatewayModel.js"; +import { + describeProfileRef, + resolveEffectiveProfile, +} from "../packages/cli/build/src/profiles/effectiveProfile.js"; +import { + loadProfile, + profilesDir, +} from "../packages/cli/build/src/profiles/loadProfile.js"; +import {Profiles} from "../packages/cli/build/src/profiles/profiles.js"; +import {createSpecModel} from "../packages/cli/build/src/profiles/roleModels.js"; const RUN_DIR = process.env.RUN_DIR ?? "data/2026-06-10-gemini-104"; const STORE = `${RUN_DIR}/manual-reruns.json`; const MD = `${RUN_DIR}/manual-reruns.md`; -const USER_MODEL = process.env.USER_MODEL ?? "deepseek-v3.2"; const idx = Number(process.argv[2]); const assistantFile = process.argv[3]; @@ -44,7 +57,16 @@ const risk = RiskCategory.findRisk(category, entry.scenario.seed.riskId); const conversationLength = risk.conversationLength; const modelsJsonPath = path.resolve("models.json"); -const userModel = createGatewayModel(modelsJsonPath, USER_MODEL); +Profiles.configure( + loadProfile(process.env.KORA_PROFILE ?? "kora", profilesDir(modelsJsonPath)) +); +const effective = resolveEffectiveProfile(modelsJsonPath, { + user: process.env.USER_MODEL ? [process.env.USER_MODEL] : undefined, +}); +console.log(`Profile: ${describeProfileRef(effective.ref)}`); +const userSpec = effective.roles.user; +const userModel = createSpecModel(userSpec); +entry.userModel = userSpec; const ctx = { getUserResponse: async request => ({ output: await userModel.getTextResponse(request), From 34f12a1df01b73c58a834446706b4f16df98e988 Mon Sep 17 00:00:00 2001 From: Thibaut Fatus Date: Fri, 4 Sep 2026 17:08:13 +0200 Subject: [PATCH 3/5] [refactor] share the profile model with other harnesses Move the evaluation profile schema, hash and role helpers into the benchmark package next to packs, so kora-infra can build profiles and stamps with the same formula; the CLI keeps loading files. Stamp roles other than `user` and `judges` become optional: a harness records the roles it has. Claude-Session: https://claude.ai/code/session_017SeRjc1yMoa617ZR2bxeE8 --- README.md | 3 +- packages/benchmark/src/index.ts | 1 + packages/benchmark/src/profiles/profile.ts | 163 ++++++++++++++++++++ packages/benchmark/src/stamp/runStamp.ts | 11 +- packages/cli/src/profiles/profile.ts | 167 ++------------------- 5 files changed, 183 insertions(+), 162 deletions(-) create mode 100644 packages/benchmark/src/profiles/profile.ts diff --git a/README.md b/README.md index 38d01fc..15d8fef 100644 --- a/README.md +++ b/README.md @@ -436,7 +436,7 @@ everything that shaped it: | Field | Description | | --------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | | `profile` | `{id, version, hash}` plus `local` and `overrides` when applicable. The hash covers the *effective* roles. | -| `models` | The resolved configuration of every role, and `target` for `run` (a model spec, or `{kind, slug}` for `kora-app-*` / `custom-*` targets) | +| `models` | The resolved configuration of every role the harness has (the CLI fills all six), and `target` for `run` (a model spec, or `{kind, slug}` for `kora-app-*` / `custom-*` targets) | | `prompts` | `{version, hash}` of the prompt templates (`packages/benchmark/src/prompts/promptsFingerprint.ts`, guarded by a test the same way as profiles) | | `code` | `@korabench/cli` version, git `commit` and `dirty` flag when run from a checkout | | `packs` | Taxonomy and behavior pack, as in `packs` | @@ -833,6 +833,7 @@ packages/ data/ Bundled pack: risks.json, behaviors.json, motivations.json (see data/README.md) src/ Core benchmark logic packs/ Pack model, scoping and taxonomy conformance + profiles/ Evaluation profile model (schema, hash) stamp/ Run stamp model and scoping prompts/ Prompt templates for each pipeline stage (+ promptsFingerprint.ts) model/ Domain types (scenario, risk, assessment, etc.) diff --git a/packages/benchmark/src/index.ts b/packages/benchmark/src/index.ts index 7822fc5..5f86d6c 100644 --- a/packages/benchmark/src/index.ts +++ b/packages/benchmark/src/index.ts @@ -38,6 +38,7 @@ export * from "./packs/packs.js"; export * from "./packs/packStamp.js"; export * from "./packs/riskTaxonomy.js"; export * from "./packs/stableJson.js"; +export * from "./profiles/profile.js"; export * from "./prompts/conversationToAssessmentPrompt.js"; export * from "./prompts/conversationToMechanismAssessmentPrompt.js"; export * from "./prompts/promptsFingerprint.js"; diff --git a/packages/benchmark/src/profiles/profile.ts b/packages/benchmark/src/profiles/profile.ts new file mode 100644 index 0000000..f25a14e --- /dev/null +++ b/packages/benchmark/src/profiles/profile.ts @@ -0,0 +1,163 @@ +import {Hash} from "@korabench/core"; +import * as R from "remeda"; +import * as v from "valibot"; +import {ModelSpec} from "../model/modelSpec.js"; +import {PackId} from "../packs/packId.js"; +import {stableJson} from "../packs/stableJson.js"; + +// +// Runtime model. +// +// An evaluation profile pins the LLM used for every role of the pipeline. +// Configs are inline (not slugs into a registry) so a profile file is a +// complete, self-describing record of what ran. The target model is never part +// of a profile: it is the subject of the evaluation, not the harness. +// +// The model lives here, next to packs, so that every harness (the CLI, the +// kora-infra worker) shares one schema and one hash formula. Loading a profile +// from disk is the CLI's business (`packages/cli/src/profiles/loadProfile.ts`). +// + +const VChain = v.pipe(v.array(ModelSpec.io), v.minLength(1)); + +const VProfileRoles = v.strictObject({ + /** Seed generation (`generate-seeds`). Fallback chain. */ + seeds: VChain, + /** Scenario expansion and validation (`expand-scenarios`). Fallback chain. */ + expansion: VChain, + /** First user message during expansion. Fallback chain. */ + expansionUser: VChain, + /** User simulator during `run` (and `reassess` label). */ + user: ModelSpec.io, + /** Concurrent judges (`run`, `reassess`, `continue`). Odd count. */ + judges: VChain, + /** User simulator during `continue`; falls back to `user`. */ + continueUser: v.optional(ModelSpec.io), +}); + +const VProfile = v.strictObject({ + id: PackId.io, + version: v.pipe(v.string(), v.minLength(1)), + /** `computeHash()` of the rest of the file; guarded by a test in CI. */ + hash: v.string(), + roles: VProfileRoles, +}); + +const ROLE_LIST = [ + "seeds", + "expansion", + "expansionUser", + "user", + "judges", + "continueUser", +] as const; + +export const ODD_JUDGES_MESSAGE = + "The current implementation only supports odd numbers of judges. This ensures that the median assessment is always defined. See `aggregateTestAssessments` for reference."; + +// +// API. +// + +function computeHash( + profile: Pick +): string { + return Hash.shortHash( + stableJson({ + id: profile.id, + version: profile.version, + roles: profile.roles, + }) + ); +} + +function label(profile: Pick): string { + return `${profile.id}@${profile.version}`; +} + +function specsOf(roles: ProfileRoles, role: Role): readonly ModelSpec[] { + const value = roles[role]; + if (value === undefined) return []; + return Array.isArray(value) ? value : [value]; +} + +function assertUniqueNames(roles: ProfileRoles): void { + ROLE_LIST.forEach(role => { + const names = specsOf(roles, role).map(spec => spec.name); + const duplicates = R.pipe( + names, + R.groupBy(R.identity()), + R.pickBy(group => group.length > 1), + R.keys() + ); + if (duplicates.length > 0) { + throw new Error( + `Profile role "${role}" lists the same model name more than once: ${duplicates.join(", ")}.` + ); + } + }); +} + +/** Structural checks shared by file profiles and override-derived ones. */ +function assertValidRoles(roles: ProfileRoles): void { + if (roles.judges.length % 2 === 0) { + throw new Error(ODD_JUDGES_MESSAGE); + } + assertUniqueNames(roles); +} + +export interface ParseOptions { + /** Refuse a profile whose declared `hash` does not match its content. */ + verifyHash: boolean; +} + +function parse(data: unknown, options: ParseOptions): Profile { + const profile = v.parse(VProfile, data); + assertValidRoles(profile.roles); + if (options.verifyHash) { + const expected = computeHash(profile); + if (profile.hash !== expected) { + throw new Error( + `Profile "${label(profile)}" declares hash "${profile.hash}" but its content hashes to "${expected}". ` + + `Bump "version" and set "hash" to "${expected}" ` + + `(\`yarn kora --profile profile --print-hash\` prints it).` + ); + } + } + return profile; +} + +/** Every role filled in: `continueUser` defaults to `user`. */ +function effectiveRoles(roles: ProfileRoles): EffectiveRoles { + return { + ...roles, + continueUser: roles.continueUser ?? roles.user, + }; +} + +// +// Exports. +// + +export type Role = (typeof ROLE_LIST)[number]; + +export interface ProfileRoles extends v.InferOutput {} + +/** `ProfileRoles` with every optional role resolved. */ +export type EffectiveRoles = Required; + +export interface Profile extends v.InferOutput {} + +export const Role = { + list: ROLE_LIST, + specsOf, +}; + +export const Profile = { + io: VProfile, + parse, + computeHash, + label, + effectiveRoles, + assertValidRoles, +}; diff --git a/packages/benchmark/src/stamp/runStamp.ts b/packages/benchmark/src/stamp/runStamp.ts index 308bb7e..925317e 100644 --- a/packages/benchmark/src/stamp/runStamp.ts +++ b/packages/benchmark/src/stamp/runStamp.ts @@ -36,13 +36,16 @@ const VTargetRef = v.union([ModelSpec.io, VRunnerTarget]); const VChain = v.array(ModelSpec.io); +// A harness records the roles it has. The CLI fills every one; the kora-infra +// worker only simulates the user and judges, so the corpus-building roles are +// optional. `user` and `judges` are what every evaluation needs. const VStampModels = v.object({ - seeds: VChain, - expansion: VChain, - expansionUser: VChain, + seeds: v.optional(VChain), + expansion: v.optional(VChain), + expansionUser: v.optional(VChain), user: ModelSpec.io, judges: VChain, - continueUser: ModelSpec.io, + continueUser: v.optional(ModelSpec.io), /** The evaluated model; only meaningful for `run`. */ target: v.optional(VTargetRef), }); diff --git a/packages/cli/src/profiles/profile.ts b/packages/cli/src/profiles/profile.ts index bcbc793..f59513f 100644 --- a/packages/cli/src/profiles/profile.ts +++ b/packages/cli/src/profiles/profile.ts @@ -1,157 +1,10 @@ -import {ModelSpec, PackId, stableJson} from "@korabench/benchmark"; -import {Hash} from "@korabench/core"; -import * as R from "remeda"; -import * as v from "valibot"; - -// -// Runtime model. -// -// An evaluation profile pins the LLM used for every role of the pipeline. -// Configs are inline (not slugs into models.json) so a profile file is a -// complete, self-describing record of what ran. The target model is never part -// of a profile: it is the subject of the evaluation, not the harness. -// - -const VChain = v.pipe(v.array(ModelSpec.io), v.minLength(1)); - -const VProfileRoles = v.strictObject({ - /** Seed generation (`generate-seeds`). Fallback chain. */ - seeds: VChain, - /** Scenario expansion and validation (`expand-scenarios`). Fallback chain. */ - expansion: VChain, - /** First user message during expansion. Fallback chain. */ - expansionUser: VChain, - /** User simulator during `run` (and `reassess` label). */ - user: ModelSpec.io, - /** Concurrent judges (`run`, `reassess`, `continue`). Odd count. */ - judges: VChain, - /** User simulator during `continue`; falls back to `user`. */ - continueUser: v.optional(ModelSpec.io), -}); - -const VProfile = v.strictObject({ - id: PackId.io, - version: v.pipe(v.string(), v.minLength(1)), - /** `computeHash()` of the rest of the file; guarded by a test in CI. */ - hash: v.string(), - roles: VProfileRoles, -}); - -const ROLE_LIST = [ - "seeds", - "expansion", - "expansionUser", - "user", - "judges", - "continueUser", -] as const; - -export const ODD_JUDGES_MESSAGE = - "The current implementation only supports odd numbers of judges. This ensures that the median assessment is always defined. See `aggregateTestAssessments` for reference."; - -// -// API. -// - -function computeHash( - profile: Pick -): string { - return Hash.shortHash( - stableJson({ - id: profile.id, - version: profile.version, - roles: profile.roles, - }) - ); -} - -function label(profile: Pick): string { - return `${profile.id}@${profile.version}`; -} - -function specsOf(roles: ProfileRoles, role: Role): readonly ModelSpec[] { - const value = roles[role]; - if (value === undefined) return []; - return Array.isArray(value) ? value : [value]; -} - -function assertUniqueNames(roles: ProfileRoles): void { - ROLE_LIST.forEach(role => { - const names = specsOf(roles, role).map(spec => spec.name); - const duplicates = R.pipe( - names, - R.groupBy(R.identity()), - R.pickBy(group => group.length > 1), - R.keys() - ); - if (duplicates.length > 0) { - throw new Error( - `Profile role "${role}" lists the same model name more than once: ${duplicates.join(", ")}.` - ); - } - }); -} - -/** Structural checks shared by file profiles and override-derived ones. */ -function assertValidRoles(roles: ProfileRoles): void { - if (roles.judges.length % 2 === 0) { - throw new Error(ODD_JUDGES_MESSAGE); - } - assertUniqueNames(roles); -} - -export interface ParseOptions { - /** Refuse a profile whose declared `hash` does not match its content. */ - verifyHash: boolean; -} - -function parse(data: unknown, options: ParseOptions): Profile { - const profile = v.parse(VProfile, data); - assertValidRoles(profile.roles); - if (options.verifyHash) { - const expected = computeHash(profile); - if (profile.hash !== expected) { - throw new Error( - `Profile "${label(profile)}" declares hash "${profile.hash}" but its content hashes to "${expected}". ` + - `Bump "version" and set "hash" to "${expected}" ` + - `(\`yarn kora --profile profile --print-hash\` prints it).` - ); - } - } - return profile; -} - -/** Every role filled in: `continueUser` defaults to `user`. */ -function effectiveRoles(roles: ProfileRoles): EffectiveRoles { - return { - ...roles, - continueUser: roles.continueUser ?? roles.user, - }; -} - -// -// Exports. -// - -export type Role = (typeof ROLE_LIST)[number]; - -export interface ProfileRoles extends v.InferOutput {} - -/** `ProfileRoles` with every optional role resolved. */ -export type EffectiveRoles = Required; - -export interface Profile extends v.InferOutput {} - -export const Role = { - list: ROLE_LIST, - specsOf, -}; - -export const Profile = { - io: VProfile, - parse, - computeHash, - label, - effectiveRoles, - assertValidRoles, -}; +// The profile model lives in the benchmark package so that every harness shares +// one schema and one hash formula; this module keeps the CLI's import paths. +export { + ODD_JUDGES_MESSAGE, + Profile, + Role, + type EffectiveRoles, + type ParseOptions, + type ProfileRoles, +} from "@korabench/benchmark"; From eb3b245debc9c45552c9e8cf63e4a77648356a42 Mon Sep 17 00:00:00 2001 From: Thibaut Fatus Date: Fri, 4 Sep 2026 17:19:15 +0200 Subject: [PATCH 4/5] [fix] type provider options as JSON values so stamps serialize everywhere Claude-Session: https://claude.ai/code/session_017SeRjc1yMoa617ZR2bxeE8 --- packages/benchmark/src/model/modelSpec.ts | 26 ++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/packages/benchmark/src/model/modelSpec.ts b/packages/benchmark/src/model/modelSpec.ts index 38c6214..3782730 100644 --- a/packages/benchmark/src/model/modelSpec.ts +++ b/packages/benchmark/src/model/modelSpec.ts @@ -13,13 +13,37 @@ import * as v from "valibot"; * else is the configuration the gateway actually uses, so a spec is * self-describing: no registry lookup is needed to know what ran. */ +/** + * Provider options are JSON, typed as such rather than `unknown`: a stamp is + * persisted and served to browsers, and serialization-aware frameworks reject + * `unknown` where they accept a JSON value. + */ +type JsonValue = + | string + | number + | boolean + | null + | readonly JsonValue[] + | {readonly [key: string]: JsonValue}; + +const VJsonValue: v.GenericSchema = v.lazy(() => + v.union([ + v.string(), + v.number(), + v.boolean(), + v.null(), + v.array(VJsonValue), + v.record(v.string(), VJsonValue), + ]) +); + const VModelSpec = v.object({ name: v.pipe(v.string(), v.minLength(1)), model: v.string(), maxTokens: v.optional(v.number()), temperature: v.optional(v.number()), providerOptions: v.optional( - v.record(v.string(), v.record(v.string(), v.unknown())) + v.record(v.string(), v.record(v.string(), VJsonValue)) ), }); From fc9ef252a9bf10e85031fa3513d41655d459aa06 Mon Sep 17 00:00:00 2001 From: Thibaut Fatus Date: Fri, 4 Sep 2026 17:25:57 +0200 Subject: [PATCH 5/5] [fix] keep provider option typing flat so serializers can instantiate it Claude-Session: https://claude.ai/code/session_017SeRjc1yMoa617ZR2bxeE8 --- packages/benchmark/src/model/modelSpec.ts | 41 +++++++++-------------- 1 file changed, 16 insertions(+), 25 deletions(-) diff --git a/packages/benchmark/src/model/modelSpec.ts b/packages/benchmark/src/model/modelSpec.ts index 3782730..32bf924 100644 --- a/packages/benchmark/src/model/modelSpec.ts +++ b/packages/benchmark/src/model/modelSpec.ts @@ -5,6 +5,21 @@ import * as v from "valibot"; // Runtime model. // +/** + * Provider options are JSON, typed as such rather than `unknown`: a stamp is + * persisted and served to browsers, and serialization-aware frameworks reject + * `unknown` where they accept a JSON value. Two levels of nesting cover every + * provider option in use (e.g. `deepseek.thinking = {type: "enabled"}`) without + * a recursive type, which those same frameworks fail to instantiate. + */ +const VJsonLeaf = v.union([v.string(), v.number(), v.boolean(), v.null()]); + +const VProviderOption = v.union([ + VJsonLeaf, + v.array(VJsonLeaf), + v.record(v.string(), VJsonLeaf), +]); + /** * A fully resolved LLM configuration plus a display `name`. * @@ -13,37 +28,13 @@ import * as v from "valibot"; * else is the configuration the gateway actually uses, so a spec is * self-describing: no registry lookup is needed to know what ran. */ -/** - * Provider options are JSON, typed as such rather than `unknown`: a stamp is - * persisted and served to browsers, and serialization-aware frameworks reject - * `unknown` where they accept a JSON value. - */ -type JsonValue = - | string - | number - | boolean - | null - | readonly JsonValue[] - | {readonly [key: string]: JsonValue}; - -const VJsonValue: v.GenericSchema = v.lazy(() => - v.union([ - v.string(), - v.number(), - v.boolean(), - v.null(), - v.array(VJsonValue), - v.record(v.string(), VJsonValue), - ]) -); - const VModelSpec = v.object({ name: v.pipe(v.string(), v.minLength(1)), model: v.string(), maxTokens: v.optional(v.number()), temperature: v.optional(v.number()), providerOptions: v.optional( - v.record(v.string(), v.record(v.string(), VJsonValue)) + v.record(v.string(), v.record(v.string(), VProviderOption)) ), });