[feat] evaluation profiles and run stamps

.gitignore

@@ -147,3 +147,6 @@ vite.config.ts.timestamp-*
 
 # Claude
 */settings.local.json
+
+# Local evaluation profiles (see README "Evaluation profiles")
+profiles/*.local.json

EVALUATION_PROCESS.md

@@ -287,6 +287,12 @@ Two quirks worth knowing about the shipped files:
 - **No taxonomy stamp.** These seeds predate packs, so `taxonomyId` and
   `taxonomyVersion` are absent — exactly the case the optional stamp in
   `model/scenarioSeed.ts` allows for.
+- **No run stamp either.** Records written today carry a `stamp` (evaluation
+  profile, prompts fingerprint, packs, code revision; see the README's
+  "Evaluation profiles"). The shipped corpus predates it. A rerun of the
+  commands above uses the `kora` profile with the chains shown as explicit
+  overrides, so its stamp records an ad-hoc profile hash with
+  `overrides: ["seeds"]` / `["expansion", "expansionUser"]`.
 
 ## Dead code
 

packages/benchmark/src/__tests__/runTest.test.ts

@@ -5,6 +5,8 @@ import {kora} from "../kora.js";
 import {InvalidTurnError} from "../model/invalidTurnError.js";
 import {Mechanism} from "../model/mechanism.js";
 import {ScenarioPrompt} from "../model/scenarioKey.js";
+import {makeStamp} from "../stamp/__tests__/fixtures.js";
+import {Stamp} from "../stamp/stamp.js";
 import {createScenario} from "./fixtures.js";
 
 //
@@ -76,6 +78,23 @@ describe("kora.runTest", () => {
   const defaultKey = keys.find(k => k.endsWith(":default"))!;
   const childKey = keys.find(k => k.endsWith(":child"))!;
 
+  it("omits the stamp when none is configured", async () => {
+    const result = await kora.runTest(
+      createTestContext(),
+      scenario,
+      defaultKey
+    );
+    expect("stamp" in result).toBe(false);
+  });
+
+  it("attaches the active stamp", async () => {
+    const stamp = makeStamp();
+    const result = await Stamp.run(stamp, () =>
+      kora.runTest(createTestContext(), scenario, defaultKey)
+    );
+    expect(result.stamp).toBe(stamp);
+  });
+
   it("produces a 3-turn conversation with 6 messages", async () => {
     const context = createTestContext();
 

packages/benchmark/src/index.ts

@@ -12,6 +12,7 @@ export * from "./model/invalidTurnError.js";
 export * from "./model/judgeAssessment.js";
 export * from "./model/mechanism.js";
 export * from "./model/mechanismAssessment.js";
+export * from "./model/modelSpec.js";
 export * from "./model/motivation.js";
 export * from "./model/populationDistribution.js";
 export * from "./model/prompt.js";
@@ -37,6 +38,10 @@ export * from "./packs/packs.js";
 export * from "./packs/packStamp.js";
 export * from "./packs/riskTaxonomy.js";
 export * from "./packs/stableJson.js";
+export * from "./profiles/profile.js";
 export * from "./prompts/conversationToAssessmentPrompt.js";
 export * from "./prompts/conversationToMechanismAssessmentPrompt.js";
+export * from "./prompts/promptsFingerprint.js";
+export * from "./stamp/runStamp.js";
+export * from "./stamp/stamp.js";
 export * from "./validateAssistantTurn.js";

packages/benchmark/src/kora.ts

@@ -55,6 +55,8 @@ import {conversationToNextMessagePrompt} from "./prompts/conversationToNextMessa
 import {riskToScenarioSeedsPrompt} from "./prompts/riskToScenarioSeedsPrompt.js";
 import {scenarioToValidationPrompt} from "./prompts/scenarioToValidationPrompt.js";
 import {seedToScenarioPrompt} from "./prompts/seedToScenarioPrompt.js";
+import {RunStamp} from "./stamp/runStamp.js";
+import {Stamp} from "./stamp/stamp.js";
 import {validateAssistantTurn} from "./validateAssistantTurn.js";
 
 const AGE_BANDS: Record<AgeRange, readonly [number, number]> = {
@@ -71,6 +73,12 @@ function clampAgeToBand(age: number, band: AgeRange): number {
   return rounded;
 }
 
+/** The active run stamp as a spreadable field: present only when configured. */
+function stampField(): {stamp?: RunStamp} {
+  const stamp = Stamp.current();
+  return stamp ? {stamp} : {};
+}
+
 /**
  * Run the judge-assessment step on a pre-existing transcript.
  *
@@ -151,6 +159,7 @@ export async function runJudges(
     mechanismAssessment,
     judgeAssessments,
     packs: Packs.fingerprint(),
+    ...stampField(),
   };
 }
 
@@ -331,6 +340,7 @@ export const kora = Benchmark.new({
             ...s,
             taxonomyId: taxonomy.id,
             taxonomyVersion: taxonomy.version,
+            ...stampField(),
             id: uuid(),
             riskCategoryId: riskCategory.id,
             riskId: risk.id,
@@ -404,6 +414,7 @@ export const kora = Benchmark.new({
         seed,
         firstUserMessage: "",
         ...modelScenario,
+        ...stampField(),
       };
 
       const validationPrompt = scenarioToValidationPrompt(

packages/benchmark/src/model/modelSpec.ts

@@ -0,0 +1,66 @@
+import * as R from "remeda";
+import * as v from "valibot";
+
+//
+// Runtime model.
+//
+
+/**
+ * Provider options are JSON, typed as such rather than `unknown`: a stamp is
+ * persisted and served to browsers, and serialization-aware frameworks reject
+ * `unknown` where they accept a JSON value. Two levels of nesting cover every
+ * provider option in use (e.g. `deepseek.thinking = {type: "enabled"}`) without
+ * a recursive type, which those same frameworks fail to instantiate.
+ */
+const VJsonLeaf = v.union([v.string(), v.number(), v.boolean(), v.null()]);
+
+const VProviderOption = v.union([
+  VJsonLeaf,
+  v.array(VJsonLeaf),
+  v.record(v.string(), VJsonLeaf),
+]);
+
+/**
+ * A fully resolved LLM configuration plus a display `name`.
+ *
+ * `name` is what the CLI prints and what result headers persist in their
+ * `judges` / `user` fields (historically a `models.json` slug). Everything
+ * else is the configuration the gateway actually uses, so a spec is
+ * self-describing: no registry lookup is needed to know what ran.
+ */
+const VModelSpec = v.object({
+  name: v.pipe(v.string(), v.minLength(1)),
+  model: v.string(),
+  maxTokens: v.optional(v.number()),
+  temperature: v.optional(v.number()),
+  providerOptions: v.optional(
+    v.record(v.string(), v.record(v.string(), VProviderOption))
+  ),
+});
+
+//
+// API.
+//
+
+function config(spec: ModelSpec): ModelConfig {
+  return R.omit(spec, ["name"]);
+}
+
+function fromConfig(name: string, config: ModelConfig): ModelSpec {
+  return {name, ...config};
+}
+
+//
+// Exports.
+//
+
+export interface ModelSpec extends v.InferOutput<typeof VModelSpec> {}
+
+/** A `ModelSpec` without its display name: the gateway-facing part. */
+export type ModelConfig = Omit<ModelSpec, "name">;
+
+export const ModelSpec = {
+  io: VModelSpec,
+  config,
+  fromConfig,
+};

packages/benchmark/src/model/scenario.ts

@@ -1,4 +1,5 @@
 import * as v from "valibot";
+import {RunStamp} from "../stamp/runStamp.js";
 import {ScenarioKey} from "./scenarioKey.js";
 import {ScenarioPrompt} from "./scenarioPrompt.js";
 import {ScenarioSeed} from "./scenarioSeed.js";
@@ -79,6 +80,8 @@ const VScenario = v.strictObject({
   ...VModelScenario.entries,
   seed: ScenarioSeed.io,
   firstUserMessage: v.string(),
+  /** Provenance of the expanding run. Optional: older corpora carry none. */
+  stamp: v.optional(RunStamp.io),
 });
 
 //

packages/benchmark/src/model/scenarioSeed.ts

@@ -1,4 +1,5 @@
 import * as v from "valibot";
+import {RunStamp} from "../stamp/runStamp.js";
 import {AgeRange} from "./ageRange.js";
 import {Motivation} from "./motivation.js";
 
@@ -253,6 +254,8 @@ const VScenarioSeed = v.strictObject({
    */
   taxonomyId: v.optional(v.string()),
   taxonomyVersion: v.optional(v.string()),
+  /** Full provenance of the generating run. Optional for the same reason. */
+  stamp: v.optional(RunStamp.io),
   id: v.string(),
   riskCategoryId: v.string(),
   riskId: v.string(),

packages/benchmark/src/model/testResult.ts

@@ -1,6 +1,7 @@
 import {ModelMessage} from "@korabench/core";
 import * as v from "valibot";
 import {PackStamp} from "../packs/packStamp.js";
+import {RunStamp} from "../stamp/runStamp.js";
 import {JudgeAssessment} from "./judgeAssessment.js";
 import {MechanismAssessment} from "./mechanismAssessment.js";
 import {Scenario} from "./scenario.js";
@@ -23,8 +24,9 @@ function buildTestResultSchema(): TestResultSchema {
     assessment: TestAssessment.io,
     mechanismAssessment: MechanismAssessment.io,
     judgeAssessments: v.array(JudgeAssessment.io),
-    // Optional: results written before packs existed carry no stamp.
+    // Optional: results written before packs / stamps existed carry none.
     packs: v.optional(PackStamp.io),
+    stamp: v.optional(RunStamp.io),
   }) as unknown as TestResultSchema;
 }
 
@@ -40,6 +42,7 @@ export interface TestResult {
   mechanismAssessment: MechanismAssessment;
   judgeAssessments: JudgeAssessment[];
   packs?: PackStamp;
+  stamp?: RunStamp;
 }
 
 type TestResultSchema = v.GenericSchema<