diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d0918bc3..ddf6d41b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -41,7 +41,7 @@ Prefer building checks declaratively and returning the list in one place instead ## Adding an experiment -Add a file under `experiments/` for the agent, model, and runtime setup you want to compare. Here you can configure which skills and MCP servers are available. +Add a `*.experiment.ts` file under `experiments//` for the agent, model, and runtime setup you want to compare. Experiment discovery only scans this owner directory depth, so supporting files can live beside experiments or in nested directories. Reuse the base configs exported from `experiments/presets.ts` where they fit. Select the experiment's `suite:` depending on your use case. If this experiment should be part of our published benchmark, assign `suite: ["benchmark"]` and include a corresponding `*-no-skills` variant to compare results with and without skills. You can also assign custom experiment suites for grouping related experiments for other head-to-head comparisons as desired. diff --git a/README.md b/README.md index 52a84349..ceb81932 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,7 @@ Agent-backed runs require the relevant provider key in `.env` (e.g. `OPENAI_API_ ## Concepts - An **eval** is one scenario under `evals///`. It contains the prompt, scorer, and optional starting state for the two environments: `remote/` (the hosted project) and `local/` (the agent's working files). -- An **experiment** is one agent/runtime/model setup under `experiments/.ts`. +- An **experiment** is one agent/runtime/model setup under `experiments//.experiment.ts`. Its ID remains ``. - An **eval suite** is a named set of evals to run together. An eval's suite is its parent folder under `evals/` (`benchmark`, `regression`, `docs`, or `other`). - An **experiment suite** is a named set of experiments with related configurations, for head to head comparisons. - An **agent** is the model driver that receives the eval prompt and calls the configured tools. diff --git a/apps/framework/harness/run-eval.ts b/apps/framework/harness/run-eval.ts index e932b827..14cb6b9e 100644 --- a/apps/framework/harness/run-eval.ts +++ b/apps/framework/harness/run-eval.ts @@ -27,6 +27,7 @@ import { readRepeatedFlag, readSuiteFilters, } from '../lib/cli-args.js'; +import { discoverExperimentFiles } from '../lib/experiment-files.js'; import { bootPlatformBackend } from './platform-backend.js'; import { viteBuild, vitestRun } from './project-runner.js'; import { buildSystemPrompt } from './system-prompt.js'; @@ -101,10 +102,10 @@ const DEBUG = args.has('--debug'); async function loadExperiments() { const dir = join(ROOT, 'experiments'); const out: Array<{ name: string; config: ExperimentConfig }> = []; - for (const f of readdirSync(dir).filter((f) => f.endsWith('.ts'))) { - const mod = await import(pathToFileURL(join(dir, f)).href); + for (const experiment of await discoverExperimentFiles(dir)) { + const mod = await import(pathToFileURL(experiment.path).href); out.push({ - name: f.replace(/\.ts$/, ''), + name: experiment.name, config: mod.default as ExperimentConfig, }); } @@ -396,7 +397,7 @@ async function runOne( if (!exp.localStack) { throw new Error( `eval ${ev.id} has interface: cli but experiment "${expName}" does not configure a local stack runtime. ` + - `Add \`localStack: localStackRuntime()\` (from "@supabase-evals/sandbox") to experiments/${expName}.ts.` + `Add \`localStack: localStackRuntime()\` (from "@supabase-evals/sandbox") to experiments//${expName}.experiment.ts.` ); } // Boots a platform-lite backend seeded from the eval's `remote/` dir so scorers @@ -714,7 +715,7 @@ async function main() { for (const ev of suiteFiltered) { if (ev.mode === 'local-stack' && !config.localStack) { console.log( - `SKIP ${name} x ${ev.id} (no local stack runtime — add \`localStack: localStackRuntime()\` from "@supabase-evals/sandbox" to experiments/${name}.ts)` + `SKIP ${name} x ${ev.id} (no local stack runtime — add \`localStack: localStackRuntime()\` from "@supabase-evals/sandbox" to experiments//${name}.experiment.ts)` ); continue; } diff --git a/apps/framework/lib/experiment-files.test.ts b/apps/framework/lib/experiment-files.test.ts new file mode 100644 index 00000000..be75c307 --- /dev/null +++ b/apps/framework/lib/experiment-files.test.ts @@ -0,0 +1,60 @@ +import { mkdtemp, mkdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; + +import { discoverExperimentFiles } from './experiment-files.js'; + +let root: string | undefined; + +afterEach(async () => { + if (root) await rm(root, { recursive: true, force: true }); + root = undefined; +}); + +describe('discoverExperimentFiles', () => { + it('discovers one owner layer and keeps experiment IDs stable', async () => { + root = await mkdtemp(join(tmpdir(), 'eval-experiments-')); + await mkdir(join(root, 'ai', 'lib'), { recursive: true }); + await mkdir(join(root, '_private')); + await writeFile(join(root, 'presets.ts'), 'export {};'); + await writeFile( + join(root, 'ai', 'model.experiment.ts'), + 'export default {};' + ); + await writeFile(join(root, 'ai', 'support.ts'), 'export {};'); + await writeFile( + join(root, 'ai', 'lib', 'nested.experiment.ts'), + 'export default {};' + ); + await writeFile( + join(root, '_private', 'hidden.experiment.ts'), + 'export default {};' + ); + + expect(await discoverExperimentFiles(root)).toEqual([ + { + name: 'model', + path: join(root, 'ai', 'model.experiment.ts'), + }, + ]); + }); + + it('rejects duplicate experiment IDs across owners', async () => { + root = await mkdtemp(join(tmpdir(), 'eval-experiments-')); + await mkdir(join(root, 'ai')); + await mkdir(join(root, 'docs')); + await writeFile( + join(root, 'ai', 'model.experiment.ts'), + 'export default {};' + ); + await writeFile( + join(root, 'docs', 'model.experiment.ts'), + 'export default {};' + ); + + await expect(discoverExperimentFiles(root)).rejects.toThrow( + 'Duplicate experiment ID "model"' + ); + }); +}); diff --git a/apps/framework/lib/experiment-files.ts b/apps/framework/lib/experiment-files.ts new file mode 100644 index 00000000..8d2ab49b --- /dev/null +++ b/apps/framework/lib/experiment-files.ts @@ -0,0 +1,46 @@ +import { readdir } from 'node:fs/promises'; +import { join } from 'node:path'; + +const EXPERIMENT_SUFFIX = '.experiment.ts'; + +export type ExperimentFile = { + name: string; + path: string; +}; + +export async function discoverExperimentFiles( + experimentsDir: string +): Promise { + const files: ExperimentFile[] = []; + const experimentPaths = new Map(); + + for (const owner of await readdir(experimentsDir, { withFileTypes: true })) { + if ( + !owner.isDirectory() || + owner.name.startsWith('.') || + owner.name.startsWith('_') + ) { + continue; + } + + const ownerDir = join(experimentsDir, owner.name); + for (const entry of await readdir(ownerDir, { withFileTypes: true })) { + if (!entry.isFile() || !entry.name.endsWith(EXPERIMENT_SUFFIX)) continue; + + const name = entry.name.slice(0, -EXPERIMENT_SUFFIX.length); + const path = join(ownerDir, entry.name); + const duplicatePath = experimentPaths.get(name); + + if (duplicatePath) { + throw new Error( + `Duplicate experiment ID "${name}": ${[duplicatePath, path].sort().join(', ')}` + ); + } + + experimentPaths.set(name, path); + files.push({ name, path }); + } + } + + return files.sort((a, b) => a.name.localeCompare(b.name)); +} diff --git a/apps/framework/package.json b/apps/framework/package.json index 7f73e481..770ad8ac 100644 --- a/apps/framework/package.json +++ b/apps/framework/package.json @@ -12,7 +12,7 @@ "typecheck": "tsc --noEmit", "test": "vitest run harness", "test:framework": "node --env-file-if-exists=../../.env --import tsx/esm scripts/smoke-framework.ts", - "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/sample-sets.test.ts", + "test:vercel-runner": "vitest run scripts/run-vercel-evals.test.ts lib/cli-args.test.ts lib/experiment-files.test.ts lib/sample-sets.test.ts", "export-results": "node --import tsx/esm scripts/export-results.ts", "demo:mcp": "node --env-file=../../.env --import tsx/esm scripts/mcp-demo.ts", "demo:executor": "node --env-file=../../.env --import tsx/esm scripts/executor-demo.ts" diff --git a/apps/framework/scripts/export-results.ts b/apps/framework/scripts/export-results.ts index 2c7c28d5..50e8912f 100644 --- a/apps/framework/scripts/export-results.ts +++ b/apps/framework/scripts/export-results.ts @@ -27,6 +27,7 @@ import { readRepeatedFlag, readSuiteFilters, } from '../lib/cli-args.js'; +import { discoverExperimentFiles } from '../lib/experiment-files.js'; import { formatIncompleteSampleSets, splitBySampleSetCompleteness, @@ -55,12 +56,10 @@ async function loadExperimentMetadata(): Promise< Map > { const map = new Map(); - for (const f of (await readdir(EXPERIMENTS_DIR)).filter((f) => - f.endsWith('.ts') - )) { - const mod = await import(pathToFileURL(join(EXPERIMENTS_DIR, f)).href); + for (const experiment of await discoverExperimentFiles(EXPERIMENTS_DIR)) { + const mod = await import(pathToFileURL(experiment.path).href); const config = mod.default as ExperimentConfig; - map.set(f.replace(/\.ts$/, ''), { + map.set(experiment.name, { display: getExperimentDisplayMetadata(config), experimentSuite: config.suite?.[0], }); diff --git a/apps/framework/tsconfig.json b/apps/framework/tsconfig.json index 0b21c008..46ea7676 100644 --- a/apps/framework/tsconfig.json +++ b/apps/framework/tsconfig.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.base.json", "include": [ - "experiments", + "../../experiments", "harness", "shims", "scripts", diff --git a/experiments/ai/claude-code-opus-5-no-skills.experiment.ts b/experiments/ai/claude-code-opus-5-no-skills.experiment.ts new file mode 100644 index 00000000..8a5dcd2a --- /dev/null +++ b/experiments/ai/claude-code-opus-5-no-skills.experiment.ts @@ -0,0 +1,8 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { claudeCodeOpus5 } from '../presets.js'; + +export default defineExperiment({ + ...claudeCodeOpus5, + suite: ['no-skills'], + skills: [], +}); diff --git a/experiments/ai/claude-code-opus-5.experiment.ts b/experiments/ai/claude-code-opus-5.experiment.ts new file mode 100644 index 00000000..1f28798b --- /dev/null +++ b/experiments/ai/claude-code-opus-5.experiment.ts @@ -0,0 +1,7 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { claudeCodeOpus5 } from '../presets.js'; + +export default defineExperiment({ + ...claudeCodeOpus5, + suite: ['benchmark'], +}); diff --git a/experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts b/experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts new file mode 100644 index 00000000..df5f5b70 --- /dev/null +++ b/experiments/ai/claude-code-sonnet-5-no-skills.experiment.ts @@ -0,0 +1,8 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { claudeCodeSonnet5 } from '../presets.js'; + +export default defineExperiment({ + ...claudeCodeSonnet5, + suite: ['no-skills'], + skills: [], +}); diff --git a/experiments/ai/claude-code-sonnet-5.experiment.ts b/experiments/ai/claude-code-sonnet-5.experiment.ts new file mode 100644 index 00000000..da35fac1 --- /dev/null +++ b/experiments/ai/claude-code-sonnet-5.experiment.ts @@ -0,0 +1,7 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { claudeCodeSonnet5 } from '../presets.js'; + +export default defineExperiment({ + ...claudeCodeSonnet5, + suite: ['benchmark'], +}); diff --git a/experiments/ai/codex-gpt-5.6-luna-no-skills.experiment.ts b/experiments/ai/codex-gpt-5.6-luna-no-skills.experiment.ts new file mode 100644 index 00000000..10f09e9c --- /dev/null +++ b/experiments/ai/codex-gpt-5.6-luna-no-skills.experiment.ts @@ -0,0 +1,10 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { codexGpt56Luna } from '../presets.js'; + +export default defineExperiment({ + ...codexGpt56Luna, + suite: ['no-skills', 'regression', 'docs'], + skills: [], + // Evals that override `skills: []` already run under the baseline experiment. + skipEval: (ev) => ev.metadata.skills?.length === 0, +}); diff --git a/experiments/ai/codex-gpt-5.6-luna.experiment.ts b/experiments/ai/codex-gpt-5.6-luna.experiment.ts new file mode 100644 index 00000000..b44c2574 --- /dev/null +++ b/experiments/ai/codex-gpt-5.6-luna.experiment.ts @@ -0,0 +1,7 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { codexGpt56Luna } from '../presets.js'; + +export default defineExperiment({ + ...codexGpt56Luna, + suite: ['benchmark', 'regression'], +}); diff --git a/experiments/ai/codex-gpt-5.6-sol-no-skills.experiment.ts b/experiments/ai/codex-gpt-5.6-sol-no-skills.experiment.ts new file mode 100644 index 00000000..220bbb29 --- /dev/null +++ b/experiments/ai/codex-gpt-5.6-sol-no-skills.experiment.ts @@ -0,0 +1,8 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { codexGpt56Sol } from '../presets.js'; + +export default defineExperiment({ + ...codexGpt56Sol, + suite: ['no-skills'], + skills: [], +}); diff --git a/experiments/ai/codex-gpt-5.6-sol.experiment.ts b/experiments/ai/codex-gpt-5.6-sol.experiment.ts new file mode 100644 index 00000000..e0b74e08 --- /dev/null +++ b/experiments/ai/codex-gpt-5.6-sol.experiment.ts @@ -0,0 +1,7 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { codexGpt56Sol } from '../presets.js'; + +export default defineExperiment({ + ...codexGpt56Sol, + suite: ['benchmark'], +}); diff --git a/experiments/ai/grok-4.6-no-skills.experiment.ts b/experiments/ai/grok-4.6-no-skills.experiment.ts new file mode 100644 index 00000000..427408b4 --- /dev/null +++ b/experiments/ai/grok-4.6-no-skills.experiment.ts @@ -0,0 +1,8 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { grok46 } from '../presets.js'; + +export default defineExperiment({ + ...grok46, + suite: ['no-skills'], + skills: [], +}); diff --git a/experiments/ai/grok-4.6.experiment.ts b/experiments/ai/grok-4.6.experiment.ts new file mode 100644 index 00000000..bd9f547f --- /dev/null +++ b/experiments/ai/grok-4.6.experiment.ts @@ -0,0 +1,7 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { grok46 } from '../presets.js'; + +export default defineExperiment({ + ...grok46, + suite: ['benchmark'], +}); diff --git a/experiments/ai/opencode-kimi-k3-no-skills.experiment.ts b/experiments/ai/opencode-kimi-k3-no-skills.experiment.ts new file mode 100644 index 00000000..081efb96 --- /dev/null +++ b/experiments/ai/opencode-kimi-k3-no-skills.experiment.ts @@ -0,0 +1,8 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { opencodeKimiK3 } from '../presets.js'; + +export default defineExperiment({ + ...opencodeKimiK3, + suite: ['no-skills'], + skills: [], +}); diff --git a/experiments/ai/opencode-kimi-k3.experiment.ts b/experiments/ai/opencode-kimi-k3.experiment.ts new file mode 100644 index 00000000..32f10260 --- /dev/null +++ b/experiments/ai/opencode-kimi-k3.experiment.ts @@ -0,0 +1,7 @@ +import { defineExperiment } from '@supabase-evals/core'; +import { opencodeKimiK3 } from '../presets.js'; + +export default defineExperiment({ + ...opencodeKimiK3, + suite: ['benchmark'], +}); diff --git a/experiments/claude-code-opus-5-no-skills.ts b/experiments/claude-code-opus-5-no-skills.ts deleted file mode 100644 index a93df1d8..00000000 --- a/experiments/claude-code-opus-5-no-skills.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - claudeCodeAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['no-skills'], - agent: claudeCodeAgent({ - model: 'claude-opus-5', - reasoningEffort: 'high', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: [], -}); diff --git a/experiments/claude-code-opus-5.ts b/experiments/claude-code-opus-5.ts deleted file mode 100644 index 0ffe0823..00000000 --- a/experiments/claude-code-opus-5.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - claudeCodeAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['benchmark'], - agent: claudeCodeAgent({ - model: 'claude-opus-5', - reasoningEffort: 'high', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/claude-code-sonnet-5-no-skills.ts b/experiments/claude-code-sonnet-5-no-skills.ts deleted file mode 100644 index 5a266f92..00000000 --- a/experiments/claude-code-sonnet-5-no-skills.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { - claudeCodeAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -// Same as claude-code-sonnet-5 but with no skills, to measure skills' impact. -export default defineExperiment({ - suite: ['no-skills'], - agent: claudeCodeAgent({ - model: 'claude-sonnet-5', - reasoningEffort: 'high', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: [], -}); diff --git a/experiments/claude-code-sonnet-5.ts b/experiments/claude-code-sonnet-5.ts deleted file mode 100644 index e2b01a50..00000000 --- a/experiments/claude-code-sonnet-5.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - claudeCodeAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['benchmark'], - agent: claudeCodeAgent({ - model: 'claude-sonnet-5', - reasoningEffort: 'high', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/codex-gpt-5.6-luna-no-skills.ts b/experiments/codex-gpt-5.6-luna-no-skills.ts deleted file mode 100644 index ef9f97cd..00000000 --- a/experiments/codex-gpt-5.6-luna-no-skills.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { - codexAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['no-skills', 'regression', 'docs'], - agent: codexAgent({ - model: 'gpt-5.6-luna', - reasoningEffort: 'medium', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: [], - // Evals that override `skills: []` already run under the baseline experiment. Skip them from running again here. - skipEval: (ev) => ev.metadata.skills?.length === 0, -}); diff --git a/experiments/codex-gpt-5.6-luna.ts b/experiments/codex-gpt-5.6-luna.ts deleted file mode 100644 index d4e18052..00000000 --- a/experiments/codex-gpt-5.6-luna.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - codexAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['benchmark', 'regression'], - agent: codexAgent({ - model: 'gpt-5.6-luna', - reasoningEffort: 'medium', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/codex-gpt-5.6-sol-no-skills.ts b/experiments/codex-gpt-5.6-sol-no-skills.ts deleted file mode 100644 index 4902cf78..00000000 --- a/experiments/codex-gpt-5.6-sol-no-skills.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - codexAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['no-skills'], - agent: codexAgent({ - model: 'gpt-5.6-sol', - reasoningEffort: 'medium', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: [], -}); diff --git a/experiments/codex-gpt-5.6-sol.ts b/experiments/codex-gpt-5.6-sol.ts deleted file mode 100644 index b3894270..00000000 --- a/experiments/codex-gpt-5.6-sol.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - codexAgent, - defineExperiment, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['benchmark'], - agent: codexAgent({ - model: 'gpt-5.6-sol', - reasoningEffort: 'medium', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/grok-4.6-no-skills.ts b/experiments/grok-4.6-no-skills.ts deleted file mode 100644 index a53d232b..00000000 --- a/experiments/grok-4.6-no-skills.ts +++ /dev/null @@ -1,21 +0,0 @@ -import { - defineExperiment, - grokAgent, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -// Same as grok-4.6 but with no skills, to measure skills' impact. -export default defineExperiment({ - suite: ['no-skills'], - agent: grokAgent({ - model: 'grok-4.6', - reasoningEffort: 'high', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: [], -}); diff --git a/experiments/grok-4.6.ts b/experiments/grok-4.6.ts deleted file mode 100644 index 1f6fc541..00000000 --- a/experiments/grok-4.6.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - defineExperiment, - grokAgent, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['benchmark'], - agent: grokAgent({ - model: 'grok-4.6', - reasoningEffort: 'high', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/opencode-kimi-k3-no-skills.ts b/experiments/opencode-kimi-k3-no-skills.ts deleted file mode 100644 index 0a6d2edd..00000000 --- a/experiments/opencode-kimi-k3-no-skills.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { - defineExperiment, - opencodeAgent, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -// Same as opencode-kimi-k3 but with no skills, to measure skills' impact. -export default defineExperiment({ - suite: ['no-skills'], - agent: opencodeAgent({ - model: 'moonshotai/kimi-k3', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: [], -}); diff --git a/experiments/opencode-kimi-k3.ts b/experiments/opencode-kimi-k3.ts deleted file mode 100644 index be08d45f..00000000 --- a/experiments/opencode-kimi-k3.ts +++ /dev/null @@ -1,19 +0,0 @@ -import { - defineExperiment, - opencodeAgent, - platformLiteRuntime, - supabaseMcpServer, -} from '@supabase-evals/core'; -import { localStackRuntime } from '@supabase-evals/sandbox'; - -export default defineExperiment({ - suite: ['benchmark'], - agent: opencodeAgent({ - model: 'moonshotai/kimi-k3', - }), - runtime: platformLiteRuntime({ - mcpServers: [supabaseMcpServer()], - }), - localStack: localStackRuntime(), - skills: ['supabase', 'supabase-postgres-best-practices'], -}); diff --git a/experiments/presets.ts b/experiments/presets.ts new file mode 100644 index 00000000..5ee335d6 --- /dev/null +++ b/experiments/presets.ts @@ -0,0 +1,82 @@ +import { + claudeCodeAgent, + codexAgent, + grokAgent, + opencodeAgent, + platformLiteRuntime, + supabaseMcpServer, + type ExperimentConfig, +} from '@supabase-evals/core'; +import { localStackRuntime } from '@supabase-evals/sandbox'; + +type ExperimentPreset = Pick< + ExperimentConfig, + 'agent' | 'runtime' | 'localStack' | 'skills' +>; + +const skills = ['supabase', 'supabase-postgres-best-practices']; + +function defaultRuntime() { + return platformLiteRuntime({ + mcpServers: [supabaseMcpServer()], + }); +} + +export const claudeCodeOpus5 = { + agent: claudeCodeAgent({ + model: 'claude-opus-5', + reasoningEffort: 'high', + }), + runtime: defaultRuntime(), + localStack: localStackRuntime(), + skills, +} satisfies ExperimentPreset; + +export const claudeCodeSonnet5 = { + agent: claudeCodeAgent({ + model: 'claude-sonnet-5', + reasoningEffort: 'high', + }), + runtime: defaultRuntime(), + localStack: localStackRuntime(), + skills, +} satisfies ExperimentPreset; + +export const codexGpt56Luna = { + agent: codexAgent({ + model: 'gpt-5.6-luna', + reasoningEffort: 'medium', + }), + runtime: defaultRuntime(), + localStack: localStackRuntime(), + skills, +} satisfies ExperimentPreset; + +export const codexGpt56Sol = { + agent: codexAgent({ + model: 'gpt-5.6-sol', + reasoningEffort: 'medium', + }), + runtime: defaultRuntime(), + localStack: localStackRuntime(), + skills, +} satisfies ExperimentPreset; + +export const grok46 = { + agent: grokAgent({ + model: 'grok-4.6', + reasoningEffort: 'high', + }), + runtime: defaultRuntime(), + localStack: localStackRuntime(), + skills, +} satisfies ExperimentPreset; + +export const opencodeKimiK3 = { + agent: opencodeAgent({ + model: 'moonshotai/kimi-k3', + }), + runtime: defaultRuntime(), + localStack: localStackRuntime(), + skills, +} satisfies ExperimentPreset;