From b13fe663d1bde48dad49ee0701a85fb1d3da6334 Mon Sep 17 00:00:00 2001 From: Arnav Dadarya Date: Sat, 22 Aug 2026 13:42:44 -0700 Subject: [PATCH] feat(core): add few-shot demos and optimizers Predict now accepts demos, and two optimizers turn a labelled trainset into them, so a program improves from data rather than from prompt edits. - Predict accepts `{ lm, demos }` alongside the existing `(signature, lm)` form, plus `withDemos()` / `withLM()` for configured copies. - buildPrompt renders demos as worked examples: labelled `field: value` on the text path, JSON on the structured-output path, so a demo always models the shape the reply is allowed to take. A prompt with no demos is byte-for-byte what it was before. - LabeledFewShot selects k of your own labels with no model calls. - BootstrapFewShot runs the trainset, scores each attempt with a metric, and promotes the runs that passed; an optional teacher model generates demos a cheaper student then imitates. Bounded concurrency, early stop once enough demos are collected, failing rows skipped, progress via callback. - Both are deterministic given a seed. Adds examples/optimizer.ts with an `example:optimizer` script, and docs site section 21. Co-Authored-By: Claude Opus 5 --- .changeset/few-shot-optimizers.md | 26 ++ README.md | 7 + examples/optimizer.ts | 171 +++++++ package.json | 3 +- packages/core/src/index.ts | 16 +- .../core/src/modules/predict-demos.test.ts | 246 ++++++++++ packages/core/src/modules/predict.ts | 114 ++++- .../src/optimizers/bootstrap-few-shot.test.ts | 429 ++++++++++++++++++ .../core/src/optimizers/bootstrap-few-shot.ts | 260 +++++++++++ packages/core/src/optimizers/index.ts | 11 + .../src/optimizers/labeled-few-shot.test.ts | 93 ++++ .../core/src/optimizers/labeled-few-shot.ts | 63 +++ packages/core/src/optimizers/random.test.ts | 96 ++++ packages/core/src/optimizers/random.ts | 69 +++ packages/core/src/optimizers/types.ts | 74 +++ packages/core/src/utils/parsing.ts | 218 ++++++++- site/docs.html | 136 ++++++ 17 files changed, 2021 insertions(+), 11 deletions(-) create mode 100644 .changeset/few-shot-optimizers.md create mode 100644 examples/optimizer.ts create mode 100644 packages/core/src/modules/predict-demos.test.ts create mode 100644 packages/core/src/optimizers/bootstrap-few-shot.test.ts create mode 100644 packages/core/src/optimizers/bootstrap-few-shot.ts create mode 100644 packages/core/src/optimizers/index.ts create mode 100644 packages/core/src/optimizers/labeled-few-shot.test.ts create mode 100644 packages/core/src/optimizers/labeled-few-shot.ts create mode 100644 packages/core/src/optimizers/random.test.ts create mode 100644 packages/core/src/optimizers/random.ts create mode 100644 packages/core/src/optimizers/types.ts diff --git a/.changeset/few-shot-optimizers.md b/.changeset/few-shot-optimizers.md new file mode 100644 index 0000000..026358c --- /dev/null +++ b/.changeset/few-shot-optimizers.md @@ -0,0 +1,26 @@ +--- +'@ts-dspy/core': minor +--- + +Add few-shot demos and optimizers, so a program can improve itself from data +rather than from prompt edits. + +`Predict` now accepts demos — `new Predict(Sig, { demos })`, or `withDemos()` for +a configured copy — and renders them into the prompt as worked examples before +the real input, in the same labelled `field: value` shape the parser reads back. +A prompt built without demos is byte-for-byte what it was before. + +Two optimizers turn a labelled trainset into those demos. `LabeledFewShot` +selects _k_ of your own labels and makes no model calls at all. `BootstrapFewShot` +runs the module over the trainset, scores each attempt with a metric, and +promotes the runs that passed into demos; a `teacher` option generates them with +a stronger model that the cheaper student then imitates, so you pay for the +strong model once, at compile time. + +Both are deterministic given a seed, so a compiled program can be reproduced and +tested. Trainset runs use bounded concurrency, and an example whose attempt +throws is skipped rather than failing the whole compile. Progress is reported +through an optional callback. + +`Predict` also gains `withLM()`, and `renderDemos()` is exported for inspecting +the few-shot text a set of demos produces. diff --git a/README.md b/README.md index aa2eed0..edc98ac 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,13 @@ Model output is **validated at runtime** against the shape you declared. If a field you declared as a number comes back as prose, you get a `ValidationError` naming the field — not a string masquerading as a number. +Programs also **optimize themselves from data**, the part of DSPy that makes it +more than a prompt template library. Give `BootstrapFewShot` a labelled trainset +and a metric and it runs your module, keeps the runs that passed, and puts them +back in the prompt as worked examples — optionally generated by a stronger +teacher model that your cheaper one then imitates. Runnable end to end in +[`examples/optimizer.ts`](examples/optimizer.ts). + ```bash npm install @ts-dspy/core @ts-dspy/openai ``` diff --git a/examples/optimizer.ts b/examples/optimizer.ts new file mode 100644 index 0000000..843bddd --- /dev/null +++ b/examples/optimizer.ts @@ -0,0 +1,171 @@ +/** + * Few-shot demos and optimizers. + * + * A program that improves itself from data: run a labelled trainset, keep the + * runs a metric approved of, and put those runs in the prompt as worked + * examples. Optionally let a stronger model generate the demos that a cheaper + * one then imitates. + * + * export OPENAI_API_KEY="sk-..." + * npm run example:optimizer + */ +import { + Signature, + InputField, + OutputField, + Predict, + Example, + LabeledFewShot, + BootstrapFewShot, + configure, + type Metric, +} from '@ts-dspy/core'; +import { OpenAILM } from '@ts-dspy/openai'; +import { requireEnv, section } from './utils'; + +// --- Signature -------------------------------------------------------------- + +class RouteTicket extends Signature { + static description = + 'Route a support ticket to the team that owns it. ' + + 'Answer with exactly one of: billing, bug, account, feature.'; + + @InputField({ description: 'the ticket text' }) + ticket!: string; + + @OutputField({ description: 'billing, bug, account, or feature' }) + team!: string; +} + +type TicketRouting = { team: string }; + +// --- Trainset --------------------------------------------------------------- + +// Labelled data. `withInputs` marks which fields are the question, so the rest +// is understood to be the answer — an optimizer must never feed the label back +// in as input, or every run would be trivially correct. +const labelled = [ + ['My card was charged twice this month.', 'billing'], + ['The export button does nothing on Safari.', 'bug'], + ['I cannot reset my password, the email never arrives.', 'account'], + ['Could you add dark mode to the dashboard?', 'feature'], + ['Why is my invoice higher than the quoted plan price?', 'billing'], + ['The app crashes when I upload a file over 10 MB.', 'bug'], + ['Please remove my colleague from the workspace.', 'account'], + ['It would help if reports could be scheduled weekly.', 'feature'], + ['I was billed after cancelling my subscription.', 'billing'], + ['Search returns no results even for exact titles.', 'bug'], + ['Can I merge two accounts into one?', 'account'], + ['A Slack integration would save us a lot of copying.', 'feature'], +].map(([ticket, team]) => new Example({ ticket, team }).withInputs('ticket')); + +// Disjoint splits. Scoring on rows the optimizer compiled from would let a demo +// for a ticket appear in the prompt used to classify that same ticket, and the +// comparison against the baseline would mean nothing. +const trainset = labelled.slice(0, 8); +const devset = labelled.slice(8); + +// --- Metric ----------------------------------------------------------------- + +// The metric is the whole specification of "good" — everything the optimizer +// does is downstream of it. This one is exact match on a normalised label. +const routedCorrectly: Metric = (example, prediction) => { + const expected = String(example.get('team')).trim().toLowerCase(); + const actual = String(prediction.get('team') ?? '') + .trim() + .toLowerCase(); + return actual === expected; +}; + +async function accuracy(module: Predict): Promise { + let correct = 0; + for (const example of devset) { + try { + const prediction = await module.forward(example.getInputs()); + if (routedCorrectly(example, prediction)) { + correct += 1; + } + } catch { + // A run that fails is a run that scored nothing. + } + } + return correct / devset.length; +} + +async function main(): Promise { + const apiKey = requireEnv('OPENAI_API_KEY'); + + // A cheap student, and a stronger teacher used only at compile time. + const student = new OpenAILM({ apiKey, model: 'gpt-4.1-mini' }); + const teacher = new OpenAILM({ apiKey, model: 'gpt-4.1' }); + configure({ lm: student }); + + // --- Baseline ----------------------------------------------------------- + section('Baseline (no demos)'); + + const baseline = new Predict(RouteTicket); + console.log(`accuracy: ${(await accuracy(baseline)) * 100}%`); + + // --- LabeledFewShot ----------------------------------------------------- + section('LabeledFewShot'); + + // No model calls at all: it just puts k of your labels in the prompt. Seeded, + // so the same seed always picks the same demos. + const labeled = new LabeledFewShot({ k: 3, seed: 42 }).compile(baseline, { trainset }); + + for (const demo of labeled.getDemos()) { + console.log(` demo: ${demo.get('ticket')} -> ${demo.get('team')}`); + } + console.log(`accuracy: ${(await accuracy(labeled)) * 100}%`); + + // --- BootstrapFewShot --------------------------------------------------- + section('BootstrapFewShot'); + + // Run the trainset through the teacher, keep the runs the metric approved + // of, and attach them to the student. The student ends up imitating work the + // stronger model did, without paying for the stronger model at run time. + const optimizer = new BootstrapFewShot({ + metric: routedCorrectly, + maxBootstrappedDemos: 4, + concurrency: 4, + seed: 42, + teacher, + callOptions: { temperature: 0 }, + // Library code never prints; progress arrives through this callback, and + // the example is what decides to put it on the terminal. + onProgress: (event) => { + const detail = event.status === 'error' ? ` (${String(event.error)})` : ''; + console.log(` [${event.index + 1}/${event.total}] ${event.status}${detail}`); + }, + }); + + const compiled = await optimizer.compile(baseline, { trainset }); + + console.log(`\nbootstrapped ${compiled.getDemos().length} demos:`); + for (const demo of compiled.getDemos()) { + console.log(` ${demo.get('ticket')} -> ${demo.get('team')}`); + } + + console.log(`\naccuracy: ${(await accuracy(compiled)) * 100}%`); + + // --- Using the compiled program ----------------------------------------- + section('Compiled program'); + + // Demos are not magic: they are text placed in front of the real input, in + // whichever shape the reply is expected to take — labelled `field: value` + // for a text provider, JSON for one with native structured output. + const routed = await compiled.forward({ ticket: 'Two invoices arrived this month.' }); + console.log(`team: ${routed.team}`); + + // --- Usage -------------------------------------------------------------- + section('Usage'); + const teacherUsage = teacher.getUsage(); + const studentUsage = student.getUsage(); + console.log(`teacher: ${teacherUsage.requestCount} requests (compile time only)`); + console.log(`student: ${studentUsage.requestCount} requests`); +} + +main().catch((error) => { + console.error(error); + process.exit(1); +}); diff --git a/package.json b/package.json index 35ab370..b1f5223 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,8 @@ "chart:downloads": "node scripts/generate-downloads-chart.js", "example:openai": "tsx examples/basic-usage.ts", "example:gemini": "tsx examples/basic-gemini-example.ts", - "example:anthropic": "tsx examples/basic-anthropic-example.ts" + "example:anthropic": "tsx examples/basic-anthropic-example.ts", + "example:optimizer": "tsx examples/optimizer.ts" }, "devDependencies": { "@changesets/cli": "^2.29.8", diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 6911b30..ea14bd8 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -15,10 +15,24 @@ export type { FieldValidationIssue } from './core/errors'; // Modules export { Predict } from './modules/predict'; +export type { PredictOptions } from './modules/predict'; export { ChainOfThought } from './modules/chain-of-thought'; export { RespAct } from './modules/respact'; export type { ToolFunction, ToolWithDescription, ToolDefinition } from './modules/respact'; +// Optimizers +export { LabeledFewShot, BootstrapFewShot } from './optimizers'; +export type { + Metric, + MetricResult, + DemoModule, + LabeledFewShotOptions, + LabeledFewShotConfig, + BootstrapFewShotOptions, + BootstrapFewShotConfig, + BootstrapProgressEvent, +} from './optimizers'; + // Utilities -export { buildPrompt, parseOutput } from './utils/parsing'; +export { buildPrompt, parseOutput, renderDemos } from './utils/parsing'; export { fieldConfigToZod, buildOutputSchema, buildOutputJsonSchema } from './utils/schema'; diff --git a/packages/core/src/modules/predict-demos.test.ts b/packages/core/src/modules/predict-demos.test.ts new file mode 100644 index 0000000..ca905c9 --- /dev/null +++ b/packages/core/src/modules/predict-demos.test.ts @@ -0,0 +1,246 @@ +import { Predict } from './predict'; +import { ChainOfThought } from './chain-of-thought'; +import { Signature, InputField, OutputField } from '../core/signature'; +import { Example } from '../core/example'; +import { buildPrompt, renderDemos } from '../utils/parsing'; +import { MockLM } from '../test-utils'; + +class QA extends Signature { + static description = 'Answer a question'; + + @InputField({ description: 'the question' }) + question!: string; + + @OutputField({ description: 'the answer' }) + answer!: string; + + @OutputField({ description: 'confidence 0-1', type: 'number' }) + confidence!: number; +} + +const paris = new Example({ + question: 'Capital of France?', + answer: 'Paris', + confidence: 0.99, +}).withInputs('question'); + +const tokyo = new Example({ + question: 'Capital of Japan?', + answer: 'Tokyo', + confidence: 0.98, +}).withInputs('question'); + +describe('demos in the prompt', () => { + it('renders nothing when a module has no demos', () => { + const withoutDemos = buildPrompt(QA, { question: 'Capital of Peru?' }); + const withEmptyDemos = buildPrompt(QA, { question: 'Capital of Peru?' }, []); + + expect(withEmptyDemos).toBe(withoutDemos); + expect(withoutDemos).not.toContain('worked example'); + }); + + it('renders demos in the labelled format the parser reads back', () => { + const block = renderDemos(QA, [paris]); + + expect(block).toContain('question: Capital of France?'); + expect(block).toContain('answer: Paris'); + expect(block).toContain('confidence: 0.99'); + }); + + it('numbers demos and keeps them in the order given', () => { + const block = renderDemos(QA, [paris, tokyo]); + + expect(block).toContain('Here are 2 worked examples of this task:'); + expect(block.indexOf('Example 1:')).toBeLessThan(block.indexOf('Example 2:')); + expect(block.indexOf('Paris')).toBeLessThan(block.indexOf('Tokyo')); + }); + + it('renders demos for a string signature', () => { + const prompt = buildPrompt('question -> answer', { question: 'Capital of Peru?' }, [ + new Example({ question: 'Capital of France?', answer: 'Paris' }), + ]); + + expect(prompt).toContain('question: Capital of France?'); + expect(prompt).toContain('answer: Paris'); + expect(prompt).toContain('question: Capital of Peru?'); + }); + + it('splits a demo by the signature when the example declares no input keys', () => { + const block = renderDemos(QA, [ + new Example({ question: 'Capital of Italy?', answer: 'Rome', confidence: 0.9 }), + ]); + + expect(block).toContain('question: Capital of Italy?'); + expect(block).toContain('answer: Rome'); + }); + + it('places demos after the task description and before the real input', () => { + const prompt = buildPrompt(QA, { question: 'Capital of Peru?' }, [paris]); + + expect(prompt.indexOf('Answer a question')).toBeLessThan(prompt.indexOf('Example 1:')); + expect(prompt.indexOf('Example 1:')).toBeLessThan( + prompt.indexOf('question: Capital of Peru?') + ); + }); + + it('honours a custom input prefix but labels outputs plainly', () => { + class Prefixed extends Signature { + @InputField({ description: 'the text', prefix: 'Text:' }) + text!: string; + + @OutputField({ description: 'the label' }) + label!: string; + } + + const block = renderDemos(Prefixed, [ + new Example({ text: 'hello', label: 'greeting' }).withInputs('text'), + ]); + + expect(block).toContain('Text: hello'); + expect(block).toContain('label: greeting'); + }); + + it('serialises non-string demo values as JSON', () => { + class Tagged extends Signature { + @InputField({ description: 'the text' }) + text!: string; + + @OutputField({ description: 'tags', type: 'string[]' }) + tags!: string[]; + } + + const block = renderDemos(Tagged, [ + new Example({ text: 'hello', tags: ['a', 'b'] }).withInputs('text'), + ]); + + expect(block).toContain('tags: ["a","b"]'); + }); + + it('drops a demo that shares no fields with the signature', () => { + const block = renderDemos(QA, [new Example({ unrelated: 'nothing to teach' })]); + + expect(block).toBe(''); + }); + + it('keeps output fields the signature never declared', () => { + const block = renderDemos(QA, [ + new Example({ + question: 'Capital of France?', + answer: 'Paris', + confidence: 0.99, + reasoning: 'France is in Europe and Paris is its seat of government.', + }).withInputs('question'), + ]); + + expect(block).toContain('reasoning: France is in Europe'); + // Declared fields still lead, in signature order. + expect(block.indexOf('answer: Paris')).toBeLessThan(block.indexOf('reasoning:')); + }); + + it('ignores undeclared fields when the example declares no input keys', () => { + const block = renderDemos(QA, [ + new Example({ question: 'Capital of France?', answer: 'Paris', id: 'row-7' }), + ]); + + expect(block).not.toContain('id:'); + }); + + it('renders demos as JSON for the structured-output path', () => { + const block = renderDemos(QA, [paris], { format: 'json' }); + + expect(block).toContain('input: {"question":"Capital of France?"}'); + expect(block).toContain('output: {"answer":"Paris","confidence":0.99}'); + // The JSON schema, not the demo, dictates the shape on this path. + expect(block).not.toContain('in the same format'); + }); +}); + +describe('Predict demo configuration', () => { + it('sends configured demos to the model', async () => { + const lm = new MockLM({ responses: ['answer: Lima\nconfidence: 0.9'] }); + const predict = new Predict(QA, { lm, demos: [paris] }); + + await predict.forward({ question: 'Capital of Peru?' }); + + expect(lm.lastPrompt()).toContain('answer: Paris'); + }); + + it('keeps the two-argument (signature, lm) form working', async () => { + const lm = new MockLM({ responses: ['answer: Lima\nconfidence: 0.9'] }); + + await new Predict(QA, lm).forward({ question: 'Capital of Peru?' }); + + expect(lm.lastPrompt()).not.toContain('worked example'); + }); + + it('exposes its demos as a copy', () => { + const predict = new Predict(QA, { lm: new MockLM(), demos: [paris] }); + + const demos = predict.getDemos(); + demos.push(tokyo); + + expect(predict.getDemos()).toHaveLength(1); + }); + + it('withDemos returns a new module and leaves the original unchanged', () => { + const original = new Predict(QA, new MockLM()); + + const compiled = original.withDemos([paris, tokyo]); + + expect(original.getDemos()).toHaveLength(0); + expect(compiled.getDemos()).toHaveLength(2); + expect(compiled).not.toBe(original); + }); + + it('withDemos preserves the module subclass', async () => { + const lm = new MockLM({ + responses: ['because Peru', 'answer: Lima\nconfidence: 0.9'], + }); + const compiled = new ChainOfThought(QA, lm).withDemos([paris]); + + const result = await compiled.forward({ question: 'Capital of Peru?' }); + + expect(compiled).toBeInstanceOf(ChainOfThought); + expect(result.reasoning).toBe('because Peru'); + expect(lm.calls[0].messages[0].content).toContain('answer: Paris'); + }); + + it('sends JSON demos to a provider with native structured output', async () => { + const lm = new MockLM({ + structuredResponses: [{ answer: 'Lima', confidence: 0.9 }], + capabilities: { supportsStructuredOutput: true }, + }); + + await new Predict(QA, { lm, demos: [paris] }).forward({ question: 'Capital of Peru?' }); + + expect(lm.structuredCalls[0].prompt).toContain('output: {"answer":"Paris"'); + expect(lm.structuredCalls[0].prompt).not.toContain('answer: Paris'); + }); + + it('rejects an object carrying only half a language model', () => { + const halfALM = { chat: async () => 'answer: Paris' }; + + expect(() => new Predict(QA, halfALM as any)).toThrow(/generate\(\)/); + }); + + it('gives each copy its own demo array', () => { + const original = new Predict(QA, { lm: new MockLM(), demos: [paris] }); + + const copy = original.withLM(new MockLM()); + (copy as any).demos.push(tokyo); + + expect((copy as any).demos).not.toBe((original as any).demos); + expect(original.getDemos()).toHaveLength(1); + }); + + it('withLM swaps the model without touching the original', async () => { + const student = new MockLM({ responses: ['answer: Lima\nconfidence: 0.9'] }); + const teacher = new MockLM({ responses: ['answer: Lima\nconfidence: 1.0'] }); + const predict = new Predict(QA, student); + + await predict.withLM(teacher).forward({ question: 'Capital of Peru?' }); + + expect(teacher.calls).toHaveLength(1); + expect(student.calls).toHaveLength(0); + }); +}); diff --git a/packages/core/src/modules/predict.ts b/packages/core/src/modules/predict.ts index 9c1d682..25b3eff 100644 --- a/packages/core/src/modules/predict.ts +++ b/packages/core/src/modules/predict.ts @@ -1,12 +1,65 @@ import { Module } from '../core/module'; import { Prediction } from '../core/prediction'; import { type Signature } from '../core/signature'; +import { type Example } from '../core/example'; import type { ILanguageModel, LLMCallOptions } from '../types/language-model'; import { parseOutput, buildPrompt } from '../utils/parsing'; import { buildOutputSchema, buildOutputJsonSchema } from '../utils/schema'; import { ValidationError, type FieldValidationIssue } from '../core/errors'; import type { SignatureOutput } from '../types/signature'; +/** Construction options for {@link Predict} and its subclasses. */ +export interface PredictOptions { + /** Language model for this module. Defaults to the configured one. */ + lm?: ILanguageModel; + /** + * Worked examples rendered into the prompt before the real input. Usually + * produced by an optimizer, but hand-written demos work just as well. + */ + demos?: Example[]; +} + +/** + * Work out whether the second constructor argument is a language model or an + * options bag. + * + * `new Predict(Sig, lm)` predates `new Predict(Sig, { demos })` and both must + * keep working, so the two are told apart structurally rather than by a marker + * property a hand-rolled `ILanguageModel` would not have. + * + * An object carrying only one of `generate`/`chat` is rejected loudly rather + * than quietly treated as options: falling through would drop the caller's model + * and silently run against the globally configured one instead, and the only + * symptom would be a surprising bill. + */ +function resolveOptions(value?: ILanguageModel | PredictOptions): PredictOptions { + if (value === undefined || value === null) { + return {}; + } + if (typeof value !== 'object') { + throw new Error( + 'Predict expects a language model or an options object as its second argument.' + ); + } + + const candidate = value as Partial & PredictOptions; + const hasGenerate = typeof candidate.generate === 'function'; + const hasChat = typeof candidate.chat === 'function'; + + if (hasGenerate && hasChat) { + return { lm: candidate as ILanguageModel }; + } + if (hasGenerate || hasChat) { + throw new Error( + `Predict was given an object with ${hasChat ? 'chat()' : 'generate()'} but not ` + + `${hasChat ? 'generate()' : 'chat()'}. Implement ILanguageModel in full, or ` + + 'extend BaseLM, which supplies generate() for you.' + ); + } + + return candidate; +} + /** * Single-shot prediction against a signature. * @@ -25,8 +78,58 @@ export class Predict< TSignature extends typeof Signature = typeof Signature, TOutput extends Record = SignatureOutput, > extends Module { - constructor(signature: TSignature | string, lm?: ILanguageModel) { - super(signature, lm); + /** Worked examples prepended to every prompt this module builds. */ + protected demos: Example[] = []; + + constructor(signature: TSignature | string, lmOrOptions?: ILanguageModel | PredictOptions) { + const options = resolveOptions(lmOrOptions); + + super(signature, options.lm); + this.demos = [...(options.demos ?? [])]; + } + + /** The demos this module renders, as a copy. */ + getDemos(): Example[] { + return [...this.demos]; + } + + /** + * A copy of this module that renders `demos`. + * + * Returns a new module rather than mutating this one: an optimizer hands + * back a compiled program while leaving the student it was given untouched, + * so the same student can be compiled twice and compared. + */ + withDemos(demos: Example[]): this { + return this.cloneWith({ demos: [...demos] }); + } + + /** + * A copy of this module that calls `lm`. + * + * This is what makes a teacher model possible: bootstrap the demos with a + * stronger model, then attach them to the cheaper student. + */ + withLM(lm: ILanguageModel): this { + return this.cloneWith({ lm }); + } + + /** + * Shallow-copy this module, preserving its concrete subclass, with some + * fields replaced — so `ChainOfThought.withDemos()` returns a + * `ChainOfThought`. + * + * Only own enumerable properties are carried over, which covers ordinary + * public fields but not `#private` ones; a subclass using those should + * override `withDemos`/`withLM` with its own copy constructor. + */ + private cloneWith(patch: { demos?: Example[]; lm?: ILanguageModel }): this { + const clone = Object.create(Object.getPrototypeOf(this)) as this; + Object.assign(clone, this, patch); + // Never share the demo array with the module we copied from, or pushing + // to one module's demos would silently alter another's. + (clone as Predict).demos = [...(clone as Predict).demos]; + return clone; } async forward( @@ -104,7 +207,12 @@ export class Predict< } protected buildPrompt(inputs: Record): string { - return buildPrompt(this.requireSignature(), inputs); + // Demos must demonstrate the shape the reply will actually take. A + // provider with native structured output has its decoding constrained to + // JSON, so labelled `field: value` demos would be modelling a format the + // model is not permitted to emit. + const format = this.lm.getCapabilities().supportsStructuredOutput ? 'json' : 'labelled'; + return buildPrompt(this.requireSignature(), inputs, this.demos, { format }); } protected parseOutput(rawOutput: string): Record { diff --git a/packages/core/src/optimizers/bootstrap-few-shot.test.ts b/packages/core/src/optimizers/bootstrap-few-shot.test.ts new file mode 100644 index 0000000..989121d --- /dev/null +++ b/packages/core/src/optimizers/bootstrap-few-shot.test.ts @@ -0,0 +1,429 @@ +import { BootstrapFewShot, type BootstrapProgressEvent } from './bootstrap-few-shot'; +import { Example } from '../core/example'; +import { Prediction } from '../core/prediction'; +import { Predict } from '../modules/predict'; +import { Signature, InputField, OutputField } from '../core/signature'; +import type { ILanguageModel } from '../types/language-model'; +import { MockLM } from '../test-utils'; + +class QA extends Signature { + @InputField({ description: 'the question' }) + question!: string; + + @OutputField({ description: 'the answer' }) + answer!: string; +} + +/** + * A module whose reply is a pure function of its input, so a test can decide + * which trainset rows pass without depending on the order they are attempted in. + */ +class FakeModule { + demos: Example[] = []; + /** Shared with every copy on purpose, so a test can see what the copies did. */ + readonly runs: Array<{ inputs: Record; model?: string }> = []; + lm?: ILanguageModel; + + constructor(private readonly reply: (inputs: Record) => Record) {} + + async forward(inputs: Record): Promise { + this.runs.push({ inputs, model: this.lm?.getModelName() }); + return new Prediction(this.reply(inputs)); + } + + withDemos(demos: Example[]): this { + return this.cloneWith({ demos: [...demos] }); + } + + withLM(lm: ILanguageModel): this { + return this.cloneWith({ lm }); + } + + private cloneWith(patch: Record): this { + const clone = Object.create(Object.getPrototypeOf(this)) as this; + Object.assign(clone, this, patch); + return clone; + } +} + +const exactMatch = (example: Example, prediction: Prediction) => + example.get('answer') === prediction.get('answer'); + +function trainset(size: number): Example[] { + return Array.from({ length: size }, (_, i) => + new Example({ question: `q${i}`, answer: `a${i}` }).withInputs('question') + ); +} + +/** Answer `q` correctly only for the listed indices. */ +function replyCorrectlyFor(indices: number[]) { + return (inputs: Record) => { + const index = Number(String(inputs.question).slice(1)); + return { answer: indices.includes(index) ? `a${index}` : 'wrong' }; + }; +} + +function demoQuestions(demos: Example[]): string[] { + return demos.map((demo) => demo.get('question')); +} + +describe('BootstrapFewShot', () => { + it('promotes only the runs whose metric passes', async () => { + const student = new FakeModule(replyCorrectlyFor([1, 3])); + + const compiled = await new BootstrapFewShot({ metric: exactMatch }).compile(student, { + trainset: trainset(5), + }); + + expect(demoQuestions(compiled.demos).sort()).toEqual(['q1', 'q3']); + }); + + it('builds demos from what the model produced, paired with the example inputs', async () => { + const student = new FakeModule(() => ({ answer: 'a0', note: 'extra field' })); + + const compiled = await new BootstrapFewShot({ metric: exactMatch }).compile(student, { + trainset: [new Example({ question: 'q0', answer: 'a0' }).withInputs('question')], + }); + + expect(compiled.demos[0].getInputs()).toEqual({ question: 'q0' }); + expect(compiled.demos[0].getOutputs()).toEqual({ answer: 'a0', note: 'extra field' }); + }); + + it('skips an example whose attempt throws instead of aborting the run', async () => { + const student = new FakeModule((inputs) => { + if (inputs.question === 'q2') { + throw new Error('provider exploded'); + } + return { answer: `a${String(inputs.question).slice(1)}` }; + }); + + const compiled = await new BootstrapFewShot({ metric: exactMatch }).compile(student, { + trainset: trainset(4), + }); + + expect(demoQuestions(compiled.demos).sort()).toEqual(['q0', 'q1', 'q3']); + }); + + it('skips an example whose metric throws', async () => { + const student = new FakeModule((inputs) => ({ + answer: `a${String(inputs.question).slice(1)}`, + })); + const metric = (example: Example, prediction: Prediction) => { + if (example.get('question') === 'q1') { + throw new Error('metric exploded'); + } + return exactMatch(example, prediction); + }; + + const compiled = await new BootstrapFewShot({ metric }).compile(student, { + trainset: trainset(3), + }); + + expect(demoQuestions(compiled.demos).sort()).toEqual(['q0', 'q2']); + }); + + it('produces the same demos, in the same order, for the same seed', async () => { + const data = trainset(12); + const optimizer = () => + new BootstrapFewShot({ metric: exactMatch, seed: 99, maxBootstrappedDemos: 4 }); + + const first = await optimizer().compile( + new FakeModule(replyCorrectlyFor([0, 2, 4, 6, 8, 10])), + { + trainset: data, + } + ); + const second = await optimizer().compile( + new FakeModule(replyCorrectlyFor([0, 2, 4, 6, 8, 10])), + { trainset: data } + ); + + expect(demoQuestions(first.demos)).toEqual(demoQuestions(second.demos)); + }); + + it('caps demos at maxBootstrappedDemos', async () => { + const student = new FakeModule((inputs) => ({ + answer: `a${String(inputs.question).slice(1)}`, + })); + + const compiled = await new BootstrapFewShot({ + metric: exactMatch, + maxBootstrappedDemos: 2, + }).compile(student, { trainset: trainset(8) }); + + expect(compiled.demos).toHaveLength(2); + }); + + it('treats a numeric score below the threshold as a failure', async () => { + const student = new FakeModule(() => ({ answer: 'anything' })); + + const compiled = await new BootstrapFewShot({ + metric: () => 0.4, + threshold: 0.5, + }).compile(student, { trainset: trainset(3) }); + + expect(compiled.demos).toHaveLength(0); + }); + + it('treats a numeric score at the threshold as a pass', async () => { + const student = new FakeModule(() => ({ answer: 'anything' })); + + const compiled = await new BootstrapFewShot({ + metric: () => 0.5, + threshold: 0.5, + }).compile(student, { trainset: trainset(3) }); + + expect(compiled.demos).toHaveLength(3); + }); + + it('awaits an async metric', async () => { + const student = new FakeModule(() => ({ answer: 'a0' })); + + const compiled = await new BootstrapFewShot({ + metric: async (example, prediction) => exactMatch(example, prediction), + }).compile(student, { + trainset: [new Example({ question: 'q0', answer: 'a0' }).withInputs('question')], + }); + + expect(compiled.demos).toHaveLength(1); + }); + + it('tops up with labelled examples when bootstrapping came up short', async () => { + const student = new FakeModule(replyCorrectlyFor([0])); + + const compiled = await new BootstrapFewShot({ + metric: exactMatch, + maxBootstrappedDemos: 3, + maxLabeledDemos: 2, + }).compile(student, { trainset: trainset(5) }); + + expect(compiled.demos).toHaveLength(3); + expect(demoQuestions(compiled.demos)).toContain('q0'); + // The example that produced the bootstrapped demo is not repeated. + expect(demoQuestions(compiled.demos).filter((q) => q === 'q0')).toHaveLength(1); + }); + + it('adds no labelled demos by default', async () => { + const student = new FakeModule(() => ({ answer: 'wrong' })); + + const compiled = await new BootstrapFewShot({ metric: exactMatch }).compile(student, { + trainset: trainset(5), + }); + + expect(compiled.demos).toHaveLength(0); + }); + + it('reports the outcome of every trainset example', async () => { + const student = new FakeModule(replyCorrectlyFor([0])); + const events: BootstrapProgressEvent[] = []; + + await new BootstrapFewShot({ + metric: exactMatch, + onProgress: (event) => events.push(event), + }).compile(student, { trainset: trainset(3) }); + + expect(events).toHaveLength(3); + expect(events.filter((e) => e.status === 'passed')).toHaveLength(1); + expect(events.filter((e) => e.status === 'failed')).toHaveLength(2); + expect(events.every((e) => e.total === 3)).toBe(true); + }); + + it('reports a thrown attempt as an error event', async () => { + const student = new FakeModule(() => { + throw new Error('provider exploded'); + }); + const events: BootstrapProgressEvent[] = []; + + await new BootstrapFewShot({ + metric: exactMatch, + onProgress: (event) => events.push(event), + }).compile(student, { trainset: trainset(1) }); + + expect(events[0].status).toBe('error'); + expect((events[0].error as Error).message).toBe('provider exploded'); + }); + + it('generates demos with the teacher and returns the student carrying them', async () => { + const teacher = new MockLM(); + const student = new FakeModule((inputs) => ({ + answer: `a${String(inputs.question).slice(1)}`, + })); + + const compiled = await new BootstrapFewShot({ metric: exactMatch, teacher }).compile( + student, + { trainset: trainset(2) } + ); + + // Every trainset run went through the teacher's model. + expect(student.runs.map((run) => run.model)).toEqual(['mock-model', 'mock-model']); + // The compiled module is the student, still on the student's own model. + expect(compiled.lm).toBeUndefined(); + expect(compiled.demos).toHaveLength(2); + }); + + it('rejects a teacher when the module cannot swap its model', async () => { + const student: any = { + forward: async () => new Prediction({}), + withDemos: () => student, + }; + + await expect( + new BootstrapFewShot({ metric: exactMatch, teacher: new MockLM() }).compile( + student as any, + { trainset: trainset(1) } + ) + ).rejects.toThrow(/withLM/); + }); + + it('runs at most `concurrency` attempts at a time', async () => { + let inFlight = 0; + let peak = 0; + const student = new FakeModule(() => ({ answer: 'x' })) as any; + student.forward = async (inputs: Record) => { + inFlight += 1; + peak = Math.max(peak, inFlight); + await new Promise((resolve) => setTimeout(resolve, 1)); + inFlight -= 1; + return new Prediction({ answer: `a${String(inputs.question).slice(1)}` }); + }; + + await new BootstrapFewShot({ metric: exactMatch, concurrency: 2 }).compile(student, { + trainset: trainset(8), + }); + + expect(peak).toBeLessThanOrEqual(2); + expect(peak).toBeGreaterThan(1); + }); + + it('names the offending example when the trainset declares no input fields', async () => { + const student = new FakeModule(() => ({ answer: 'a0' })); + + await expect( + new BootstrapFewShot({ metric: exactMatch }).compile(student, { + trainset: [new Example({ question: 'q0', answer: 'a0' })], + }) + ).rejects.toThrow(/withInputs/); + }); + + it('accepts inputKeys instead of withInputs on every example', async () => { + const student = new FakeModule(() => ({ answer: 'a0' })); + + const compiled = await new BootstrapFewShot({ + metric: exactMatch, + inputKeys: ['question'], + }).compile(student, { trainset: [new Example({ question: 'q0', answer: 'a0' })] }); + + expect(compiled.demos).toHaveLength(1); + expect(student.runs[0].inputs).toEqual({ question: 'q0' }); + }); + + it('requires a metric', () => { + expect(() => new BootstrapFewShot({} as any)).toThrow(/metric/); + }); + + it('names a field that inputKeys asks for but the example lacks', async () => { + const student = new FakeModule(() => ({ answer: 'a0' })); + + await expect( + new BootstrapFewShot({ metric: exactMatch, inputKeys: ['qeustion'] }).compile( + student, + { trainset: [new Example({ question: 'q0', answer: 'a0' })] } + ) + ).rejects.toThrow(/qeustion/); + }); + + it('stops running the trainset once it has enough demos', async () => { + const student = new FakeModule((inputs) => ({ + answer: `a${String(inputs.question).slice(1)}`, + })); + + await new BootstrapFewShot({ + metric: exactMatch, + maxBootstrappedDemos: 2, + concurrency: 2, + }).compile(student, { trainset: trainset(50) }); + + // One batch of two suffices; the other 48 rows are never paid for. + expect(student.runs).toHaveLength(2); + }); + + it('makes no model calls at all for a labels-only compile', async () => { + const student = new FakeModule(() => ({ answer: 'a0' })); + + const compiled = await new BootstrapFewShot({ + metric: exactMatch, + maxBootstrappedDemos: 0, + maxLabeledDemos: 3, + }).compile(student, { trainset: trainset(6) }); + + expect(student.runs).toHaveLength(0); + expect(compiled.demos).toHaveLength(3); + }); + + it('carries on when the progress callback throws', async () => { + const student = new FakeModule((inputs) => ({ + answer: `a${String(inputs.question).slice(1)}`, + })); + + const compiled = await new BootstrapFewShot({ + metric: exactMatch, + onProgress: () => { + throw new Error('logging exploded'); + }, + }).compile(student, { trainset: trainset(4) }); + + expect(compiled.demos).toHaveLength(4); + }); + + it('compiles an empty trainset into a module with no demos', async () => { + const student = new FakeModule(() => ({ answer: 'a0' })); + + const compiled = await new BootstrapFewShot({ metric: exactMatch }).compile(student, { + trainset: [], + }); + + expect(compiled.demos).toEqual([]); + }); + + it('puts the bootstrapped demos into the compiled module prompt', async () => { + const lm = new MockLM({ + responses: ['answer: Paris', 'answer: from the compiled run'], + }); + const optimizer = new BootstrapFewShot({ + metric: (example, prediction) => example.get('answer') === prediction.get('answer'), + }); + + const compiled = await optimizer.compile(new Predict(QA, lm), { + trainset: [ + new Example({ question: 'Capital of France?', answer: 'Paris' }).withInputs( + 'question' + ), + ], + }); + await compiled.forward({ question: 'Capital of Peru?' }); + + expect(lm.lastPrompt()).toContain('question: Capital of France?'); + expect(lm.lastPrompt()).toContain('answer: Paris'); + }); + + it('leaves the compiled module unchanged when nothing passed', async () => { + const lm = new MockLM({ + responses: ['answer: Berlin', 'answer: from the compiled run'], + }); + const optimizer = new BootstrapFewShot({ + metric: (example, prediction) => example.get('answer') === prediction.get('answer'), + }); + + const compiled = await optimizer.compile(new Predict(QA, lm), { + trainset: [ + new Example({ question: 'Capital of France?', answer: 'Paris' }).withInputs( + 'question' + ), + ], + }); + await compiled.forward({ question: 'Capital of Peru?' }); + + expect(compiled.getDemos()).toHaveLength(0); + expect(lm.lastPrompt()).not.toContain('worked example'); + }); +}); diff --git a/packages/core/src/optimizers/bootstrap-few-shot.ts b/packages/core/src/optimizers/bootstrap-few-shot.ts new file mode 100644 index 0000000..116a859 --- /dev/null +++ b/packages/core/src/optimizers/bootstrap-few-shot.ts @@ -0,0 +1,260 @@ +import { Example } from '../core/example'; +import { type Prediction } from '../core/prediction'; +import type { ILanguageModel, LLMCallOptions } from '../types/language-model'; +import { createRng, mapWithConcurrency, shuffled } from './random'; +import { type DemoModule, type Metric, type MetricResult, resolveExampleInputs } from './types'; + +/** One trainset example's outcome, reported as the run proceeds. */ +export interface BootstrapProgressEvent { + /** Position in the shuffled trainset. */ + index: number; + /** Size of the trainset. */ + total: number; + /** `passed` and `failed` mean the metric ran; `error` means the attempt threw. */ + status: 'passed' | 'failed' | 'error'; + /** The metric's verdict, when the metric ran. */ + score?: MetricResult; + /** Whatever the attempt threw, when it threw. */ + error?: unknown; +} + +export interface BootstrapFewShotOptions { + /** Judges each attempt. Only the attempts that pass become demos. */ + metric: Metric; + /** + * Cap on demos promoted from successful runs. Default 4. The run stops once + * this many are collected, so a large trainset costs no more than it has to. + */ + maxBootstrappedDemos?: number; + /** + * Plain labelled examples added alongside the bootstrapped ones, capped at + * this many. Default 0 — self-generated demos are the point, so padding with + * labels is opt-in. Set `maxBootstrappedDemos: 0` alongside it for a + * labels-only compile that makes no model calls at all. + */ + maxLabeledDemos?: number; + /** + * Trainset examples attempted at once, and the size of the batch the run + * stops between. Default 4. + */ + concurrency?: number; + /** Seed for the trainset shuffle. The same seed always yields the same demos. */ + seed?: number; + /** + * Score at which a numeric metric counts as a pass. Default 0.5, which + * treats an exact-match 0/1 metric the obvious way while still giving a + * partial-credit metric a sensible midpoint. Booleans ignore this. + */ + threshold?: number; + /** + * A stronger model used to generate the demos. The compiled module is still + * the student, on the student's own model — it just imitates work the + * teacher did. This is the technique that makes bootstrapping worth the + * trouble: you pay for the strong model once, at compile time. + */ + teacher?: ILanguageModel; + /** Input field names, when the trainset examples do not declare their own. */ + inputKeys?: string[]; + /** + * Called once per attempted trainset example. Events arrive in completion + * order, which with concurrency above 1 is not trainset order; the demos + * themselves are deterministic regardless. Fewer than `total` events arrive + * when the run collects enough demos and stops early. Anything this callback + * throws is swallowed — reporting progress must not lose a compile. + */ + onProgress?: (event: BootstrapProgressEvent) => void; + /** Call options for the trainset runs, e.g. a temperature of 0. */ + callOptions?: LLMCallOptions; +} + +export interface BootstrapFewShotConfig { + trainset: Example[]; +} + +/** A successful attempt, kept with its position so ordering stays deterministic. */ +interface Candidate { + index: number; + demo: Example; +} + +/** + * Teach a module from its own successes. + * + * Runs the student over a labelled trainset, scores each attempt with a metric, + * and promotes the runs that passed into demos on the returned module. The + * program improves from data rather than from prompt edits, which is the whole + * idea it inherits from DSPy. + * + * ```ts + * const compiled = await new BootstrapFewShot({ + * metric: (example, prediction) => example.get('answer') === prediction.get('answer'), + * maxBootstrappedDemos: 3, + * teacher: strongLM, + * seed: 42, + * }).compile(new Predict(QA, cheapLM), { trainset }); + * ``` + * + * A trainset example whose attempt throws — a provider error, a reply that fails + * validation — is skipped, not fatal: one bad row must not throw away every demo + * already paid for. + */ +export class BootstrapFewShot { + private readonly metric: Metric; + private readonly maxBootstrappedDemos: number; + private readonly maxLabeledDemos: number; + private readonly concurrency: number; + private readonly seed: number; + private readonly threshold: number; + private readonly teacher?: ILanguageModel; + private readonly inputKeys?: string[]; + private readonly onProgress?: (event: BootstrapProgressEvent) => void; + private readonly callOptions?: LLMCallOptions; + + constructor(options: BootstrapFewShotOptions) { + if (typeof options?.metric !== 'function') { + throw new Error('BootstrapFewShot requires a metric function.'); + } + + this.metric = options.metric; + this.maxBootstrappedDemos = options.maxBootstrappedDemos ?? 4; + this.maxLabeledDemos = options.maxLabeledDemos ?? 0; + this.concurrency = options.concurrency ?? 4; + this.seed = options.seed ?? 0; + this.threshold = options.threshold ?? 0.5; + this.teacher = options.teacher; + this.inputKeys = options.inputKeys; + this.onProgress = options.onProgress; + this.callOptions = options.callOptions; + } + + /** Run the trainset and return a copy of `student` carrying the demos it earned. */ + async compile( + student: M, + config: BootstrapFewShotConfig + ): Promise { + const ordered = shuffled(config.trainset, createRng(this.seed)); + + // Resolve every input split up front. A trainset that never declared its + // inputs is a configuration mistake, and it should say so once rather + // than look like every single example happened to fail. + const inputsByIndex = ordered.map((example, index) => + resolveExampleInputs(example, this.inputKeys, index) + ); + + const runner = this.teacherModule(student); + const total = ordered.length; + const batchSize = Math.max(1, Math.trunc(this.concurrency) || 1); + + // Run in batches and stop once enough demos are in hand. A 500-row + // trainset should not cost 500 teacher calls to keep four demos. The + // batch boundary is what keeps this deterministic: every index below the + // boundary has finished, so the successes collected so far really are + // the first ones in trainset order. + const bootstrapped: Candidate[] = []; + for (let start = 0; start < ordered.length; start += batchSize) { + if (bootstrapped.length >= this.maxBootstrappedDemos) { + break; + } + + const batch = ordered.slice(start, start + batchSize); + const results = await mapWithConcurrency( + batch, + this.concurrency, + async (example, offset) => { + const index = start + offset; + try { + const inputs = inputsByIndex[index]; + const prediction = await runner.forward(inputs, this.callOptions); + const score = await this.metric(example, prediction); + + if (!this.passes(score)) { + this.report({ index, total, status: 'failed', score }); + return null; + } + + this.report({ index, total, status: 'passed', score }); + return { index, demo: this.toDemo(inputs, prediction) }; + } catch (error) { + this.report({ index, total, status: 'error', error }); + return null; + } + } + ); + + for (const result of results) { + if (result && bootstrapped.length < this.maxBootstrappedDemos) { + bootstrapped.push(result); + } + } + } + + return student.withDemos([ + ...bootstrapped.map((c) => c.demo), + ...this.labeledDemos(ordered, bootstrapped), + ]); + } + + /** + * Plain labelled examples to show alongside the bootstrapped ones. Examples + * already used as a bootstrapped demo are excluded, so the model does not + * see the same item twice. + */ + private labeledDemos(ordered: Example[], bootstrapped: Candidate[]): Example[] { + if (this.maxLabeledDemos <= 0) { + return []; + } + + const used = new Set(bootstrapped.map((c) => c.index)); + return ordered.filter((_, index) => !used.has(index)).slice(0, this.maxLabeledDemos); + } + + /** The module that generates the demos: the teacher's, when one was given. */ + private teacherModule(student: M): M { + if (!this.teacher) { + return student; + } + if (typeof student.withLM !== 'function') { + throw new Error( + 'A teacher model was supplied, but this module has no withLM() method. ' + + 'Implement withLM(lm) on the module, or drop the teacher option.' + ); + } + return student.withLM(this.teacher); + } + + private passes(score: MetricResult): boolean { + if (typeof score === 'boolean') { + return score; + } + // NaN fails this comparison, which is the right answer for a metric that + // could not produce a number. + return score >= this.threshold; + } + + /** + * Turn a successful run into a demo: the example's inputs, paired with what + * the model actually produced. The prediction is used rather than the label + * so the demo shows a complete, self-consistent piece of work in the model's + * own voice — including any reasoning field the label never had. + */ + private toDemo(inputs: Record, prediction: Prediction): Example { + const inputKeys = Object.keys(inputs); + return new Example({ ...inputs, ...prediction.toObject() }).withInputs(...inputKeys); + } + + /** + * A progress callback is an observer, not a participant. If it throws, the + * throw is dropped: letting it escape would be caught as an attempt failure, + * and a callback that throws every time would abandon the whole compile. + */ + private report(event: BootstrapProgressEvent): void { + if (!this.onProgress) { + return; + } + try { + this.onProgress(event); + } catch { + // Deliberately ignored. + } + } +} diff --git a/packages/core/src/optimizers/index.ts b/packages/core/src/optimizers/index.ts new file mode 100644 index 0000000..0d6feb1 --- /dev/null +++ b/packages/core/src/optimizers/index.ts @@ -0,0 +1,11 @@ +export { LabeledFewShot } from './labeled-few-shot'; +export type { LabeledFewShotOptions, LabeledFewShotConfig } from './labeled-few-shot'; + +export { BootstrapFewShot } from './bootstrap-few-shot'; +export type { + BootstrapFewShotOptions, + BootstrapFewShotConfig, + BootstrapProgressEvent, +} from './bootstrap-few-shot'; + +export type { Metric, MetricResult, DemoModule } from './types'; diff --git a/packages/core/src/optimizers/labeled-few-shot.test.ts b/packages/core/src/optimizers/labeled-few-shot.test.ts new file mode 100644 index 0000000..5532b8e --- /dev/null +++ b/packages/core/src/optimizers/labeled-few-shot.test.ts @@ -0,0 +1,93 @@ +import { LabeledFewShot } from './labeled-few-shot'; +import { Example } from '../core/example'; +import { Predict } from '../modules/predict'; +import { Signature, InputField, OutputField } from '../core/signature'; +import { MockLM } from '../test-utils'; + +class QA extends Signature { + @InputField({ description: 'the question' }) + question!: string; + + @OutputField({ description: 'the answer' }) + answer!: string; +} + +function trainset(size: number): Example[] { + return Array.from({ length: size }, (_, i) => + new Example({ question: `q${i}`, answer: `a${i}` }).withInputs('question') + ); +} + +function answers(examples: Example[]): string[] { + return examples.map((example) => example.get('answer')); +} + +describe('LabeledFewShot', () => { + it('selects k demos from the trainset', () => { + const selected = new LabeledFewShot({ k: 3 }).select(trainset(10)); + + expect(selected).toHaveLength(3); + }); + + it('selects the same demos for the same seed', () => { + const data = trainset(10); + + const first = new LabeledFewShot({ k: 3, seed: 42 }).select(data); + const second = new LabeledFewShot({ k: 3, seed: 42 }).select(data); + + expect(answers(first)).toEqual(answers(second)); + }); + + it('selects a different order for a different seed', () => { + const data = trainset(20); + + const first = new LabeledFewShot({ k: 5, seed: 1 }).select(data); + const second = new LabeledFewShot({ k: 5, seed: 2 }).select(data); + + expect(answers(first)).not.toEqual(answers(second)); + }); + + it('returns the whole trainset when k exceeds it', () => { + const selected = new LabeledFewShot({ k: 10 }).select(trainset(3)); + + expect(selected).toHaveLength(3); + }); + + it('selects nothing for k of zero', () => { + expect(new LabeledFewShot({ k: 0 }).select(trainset(5))).toEqual([]); + }); + + it('leaves the trainset it was given in its original order', () => { + const data = trainset(10); + + new LabeledFewShot({ k: 4, seed: 3 }).select(data); + + expect(answers(data)).toEqual(answers(trainset(10))); + }); + + it('compiles a module that renders the selected demos', async () => { + const lm = new MockLM({ responses: ['answer: compiled'] }); + const compiled = new LabeledFewShot({ k: 2, seed: 7 }).compile(new Predict(QA, lm), { + trainset: trainset(6), + }); + + await compiled.forward({ question: 'live question' }); + + const prompt = lm.lastPrompt(); + expect(prompt).toContain('Here are 2 worked examples of this task:'); + for (const demo of compiled.getDemos()) { + expect(prompt).toContain(`answer: ${demo.get('answer')}`); + } + }); + + it('leaves the student it compiled untouched', () => { + const student = new Predict(QA, new MockLM()); + + const compiled = new LabeledFewShot({ k: 2 }).compile(student, { + trainset: trainset(5), + }); + + expect(student.getDemos()).toHaveLength(0); + expect(compiled.getDemos()).toHaveLength(2); + }); +}); diff --git a/packages/core/src/optimizers/labeled-few-shot.ts b/packages/core/src/optimizers/labeled-few-shot.ts new file mode 100644 index 0000000..1e36905 --- /dev/null +++ b/packages/core/src/optimizers/labeled-few-shot.ts @@ -0,0 +1,63 @@ +import { type Example } from '../core/example'; +import { createRng, shuffled } from './random'; +import { type DemoModule } from './types'; + +export interface LabeledFewShotOptions { + /** How many demos to select. Default 4. */ + k?: number; + /** Seed for the selection shuffle. The same seed always selects the same demos. */ + seed?: number; +} + +/** What a compile call needs: the labelled data to select from. */ +export interface LabeledFewShotConfig { + trainset: Example[]; +} + +/** + * The simplest optimizer: put *k* of your labelled examples in the prompt. + * + * No model calls, so compiling is free and instant. It is often a surprisingly + * strong baseline, and it is the thing to try before reaching for + * {@link BootstrapFewShot} — if hand-labelled demos already get you where you + * need to be, there is nothing to bootstrap. + * + * ```ts + * const trainset = [ + * new Example({ question: 'Capital of France?', answer: 'Paris' }).withInputs('question'), + * new Example({ question: 'Capital of Japan?', answer: 'Tokyo' }).withInputs('question'), + * ]; + * + * const compiled = new LabeledFewShot({ k: 2, seed: 7 }).compile(new Predict(QA), { trainset }); + * ``` + */ +export class LabeledFewShot { + private readonly k: number; + private readonly seed: number; + + constructor(options: LabeledFewShotOptions = {}) { + this.k = options.k ?? 4; + this.seed = options.seed ?? 0; + } + + /** + * Select demos and return a configured copy of `student`. + * + * Synchronous: selection touches no model, and making callers `await` a + * pure array shuffle would only obscure that. + */ + compile(student: M, config: LabeledFewShotConfig): M { + return student.withDemos(this.select(config.trainset)); + } + + /** + * The demos this optimizer would select, without compiling anything. + * Exposed so a caller can inspect or diff a selection. + */ + select(trainset: Example[]): Example[] { + if (this.k <= 0 || trainset.length === 0) { + return []; + } + return shuffled(trainset, createRng(this.seed)).slice(0, this.k); + } +} diff --git a/packages/core/src/optimizers/random.test.ts b/packages/core/src/optimizers/random.test.ts new file mode 100644 index 0000000..7737cd9 --- /dev/null +++ b/packages/core/src/optimizers/random.test.ts @@ -0,0 +1,96 @@ +import { createRng, mapWithConcurrency, shuffled } from './random'; + +describe('createRng', () => { + it('produces the same sequence for the same seed', () => { + const first = Array.from({ length: 5 }, createRng(7)); + const second = Array.from({ length: 5 }, createRng(7)); + + expect(first).toEqual(second); + }); + + it('produces a different sequence for a different seed', () => { + expect(createRng(1)()).not.toBe(createRng(2)()); + }); + + it('stays within [0, 1)', () => { + const rng = createRng(-13); + + for (let i = 0; i < 500; i++) { + const value = rng(); + expect(value).toBeGreaterThanOrEqual(0); + expect(value).toBeLessThan(1); + } + }); +}); + +describe('shuffled', () => { + it('leaves the input array alone', () => { + const items = [1, 2, 3, 4, 5]; + + shuffled(items, createRng(3)); + + expect(items).toEqual([1, 2, 3, 4, 5]); + }); + + it('keeps every element exactly once', () => { + const items = [1, 2, 3, 4, 5, 6, 7, 8]; + + const result = shuffled(items, createRng(11)); + + expect([...result].sort((a, b) => a - b)).toEqual(items); + }); + + it('orders identically for the same seed', () => { + const items = ['a', 'b', 'c', 'd', 'e', 'f']; + + expect(shuffled(items, createRng(5))).toEqual(shuffled(items, createRng(5))); + }); +}); + +describe('mapWithConcurrency', () => { + it('returns results in input order, not completion order', async () => { + const delays = [30, 1, 20, 2]; + + const results = await mapWithConcurrency(delays, 4, async (delay, index) => { + await new Promise((resolve) => setTimeout(resolve, delay)); + return index; + }); + + expect(results).toEqual([0, 1, 2, 3]); + }); + + it('runs no more than the limit at once', async () => { + let inFlight = 0; + let peak = 0; + + await mapWithConcurrency( + Array.from({ length: 10 }, (_, i) => i), + 3, + async () => { + inFlight += 1; + peak = Math.max(peak, inFlight); + await new Promise((resolve) => setTimeout(resolve, 1)); + inFlight -= 1; + return null; + } + ); + + expect(peak).toBe(3); + }); + + it('handles an empty list', async () => { + expect(await mapWithConcurrency([], 4, async () => 1)).toEqual([]); + }); + + it('treats a limit below one as one', async () => { + const order: number[] = []; + + await mapWithConcurrency([1, 2, 3], 0, async (item) => { + order.push(item); + await new Promise((resolve) => setTimeout(resolve, 1)); + return item; + }); + + expect(order).toEqual([1, 2, 3]); + }); +}); diff --git a/packages/core/src/optimizers/random.ts b/packages/core/src/optimizers/random.ts new file mode 100644 index 0000000..52c61e0 --- /dev/null +++ b/packages/core/src/optimizers/random.ts @@ -0,0 +1,69 @@ +/** + * Deterministic randomness for optimizers. + * + * `Math.random()` cannot be seeded, so a compile that used it would pick a + * different set of demos on every run — nobody could reproduce a result or write + * a stable test for one. These are small, self-contained, and identical across + * platforms and Node versions. + */ + +/** A seeded pseudo-random source producing values in `[0, 1)`. */ +export type Rng = () => number; + +/** + * mulberry32: a 32-bit PRNG that is short, fast, and good enough for shuffling. + * Not cryptographically secure, and not meant to be. + */ +export function createRng(seed: number): Rng { + // Coerce to a 32-bit integer so any finite seed, including a negative or + // fractional one, produces a usable state. + let state = Math.trunc(seed) | 0; + return function next(): number { + state = (state + 0x6d2b79f5) | 0; + let t = state; + t = Math.imul(t ^ (t >>> 15), t | 1); + t ^= t + Math.imul(t ^ (t >>> 7), t | 61); + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** Fisher-Yates over a copy: the input array is left alone. */ +export function shuffled(items: readonly T[], rng: Rng): T[] { + const result = [...items]; + for (let i = result.length - 1; i > 0; i--) { + const j = Math.floor(rng() * (i + 1)); + [result[i], result[j]] = [result[j], result[i]]; + } + return result; +} + +/** + * Map over items with at most `limit` in flight, returning results in input + * order regardless of the order they completed in. + * + * Input order matters: an optimizer that kept whichever demos finished first + * would be at the mercy of network timing, which is exactly the nondeterminism + * the seed exists to remove. + */ +export async function mapWithConcurrency( + items: readonly T[], + limit: number, + worker: (item: T, index: number) => Promise +): Promise { + const results = new Array(items.length); + if (items.length === 0) { + return results; + } + + const workers = Math.max(1, Math.min(Math.trunc(limit) || 1, items.length)); + let cursor = 0; + + const runners = Array.from({ length: workers }, async () => { + for (let index = cursor++; index < items.length; index = cursor++) { + results[index] = await worker(items[index], index); + } + }); + + await Promise.all(runners); + return results; +} diff --git a/packages/core/src/optimizers/types.ts b/packages/core/src/optimizers/types.ts new file mode 100644 index 0000000..53edcaf --- /dev/null +++ b/packages/core/src/optimizers/types.ts @@ -0,0 +1,74 @@ +import { type Example } from '../core/example'; +import { type Prediction } from '../core/prediction'; +import type { ILanguageModel, LLMCallOptions } from '../types/language-model'; + +/** What a metric may return: a score, or a straight pass/fail. */ +export type MetricResult = number | boolean; + +/** + * Judge one prediction against the labelled example it came from. + * + * Return `true`/`false` for a pass/fail metric, or a number for partial credit — + * a numeric score counts as a pass when it meets the optimizer's `threshold`. + */ +export type Metric = ( + example: Example, + prediction: Prediction +) => MetricResult | Promise; + +/** + * The slice of a module an optimizer needs: run it, and produce a configured + * copy of it. + * + * Deliberately structural rather than `Predict`-typed, so a hand-written module + * can be optimized as long as it can run and can accept demos. `withLM` is + * optional and only required when a teacher model is supplied. + */ +export interface DemoModule { + forward(inputs: Record, options?: LLMCallOptions): Promise; + withDemos(demos: Example[]): this; + withLM?(lm: ILanguageModel): this; +} + +/** + * Resolve the input half of a labelled example. + * + * Optimizers have to call the module with inputs alone — feeding the labels back + * in would make every run trivially correct — so the split has to be known. + * `withInputs()` on the example is the usual way to declare it; `inputKeys` on + * the optimizer covers a trainset built without it. + */ +export function resolveExampleInputs( + example: Example, + inputKeys: string[] | undefined, + position: number +): Record { + if (inputKeys && inputKeys.length > 0) { + const data = example.toObject(); + const picked: Record = {}; + for (const key of inputKeys) { + // A typo here would call the module with no input at all: the prompt + // builder skips undefined values, the model would answer noise, the + // metric would reject every row, and the whole trainset would be + // paid for to produce nothing. Say so instead. + if (!Object.prototype.hasOwnProperty.call(data, key)) { + throw new Error( + `Trainset example at index ${position} has no field "${key}". ` + + `It has: ${Object.keys(data).join(', ') || '(nothing)'}.` + ); + } + picked[key] = data[key]; + } + return picked; + } + + try { + return example.getInputs(); + } catch { + throw new Error( + `Trainset example at index ${position} does not declare its input fields. ` + + 'Call example.withInputs(...keys) when building the trainset, or pass ' + + 'inputKeys to the optimizer.' + ); + } +} diff --git a/packages/core/src/utils/parsing.ts b/packages/core/src/utils/parsing.ts index ca80da0..17352ee 100644 --- a/packages/core/src/utils/parsing.ts +++ b/packages/core/src/utils/parsing.ts @@ -1,21 +1,54 @@ import { Signature } from '../core/signature'; +import { type Example } from '../core/example'; import { ValidationError, type FieldValidationIssue } from '../core/errors'; import { buildOutputSchema, getOutputFieldConfigs } from './schema'; +/** + * How demos are written into the prompt. + * + * `labelled` mirrors the `field: value` text {@link parseOutput} reads back, and + * suits a provider answering in plain text. `json` suits a provider whose + * decoding is constrained to a JSON schema, where labelled examples would be + * demonstrating a shape the model is not allowed to emit. + */ +export type DemoFormat = 'labelled' | 'json'; + +export interface RenderDemosOptions { + /** Defaults to `labelled`. */ + format?: DemoFormat; +} + +/** + * Render a prompt for one call. + * + * `demos` are worked examples shown before the real input, so the model can see + * the task performed correctly before attempting it. They render in the shape + * the reply is expected to take, which is what makes them teach the output + * format rather than merely illustrate the task. With no demos the output is + * byte-for-byte what it was before few-shot support existed. + */ export function buildPrompt( signature: typeof Signature | string, - inputs: Record + inputs: Record, + demos: Example[] = [], + options: RenderDemosOptions = {} ): string { + const demoBlock = renderDemos(signature, demos, options); + if (typeof signature === 'string') { - return buildPromptFromString(signature, inputs); + return buildPromptFromString(signature, inputs, demoBlock); } - return buildPromptFromClass(signature, inputs); + return buildPromptFromClass(signature, inputs, demoBlock); } -function buildPromptFromString(signatureStr: string, inputs: Record): string { +function buildPromptFromString( + signatureStr: string, + inputs: Record, + demoBlock = '' +): string { const parsed = Signature.parseStringSignature(signatureStr); - let prompt = ''; + let prompt = demoBlock; for (const inputKey of parsed.inputs) { if (inputs[inputKey] !== undefined) { @@ -39,7 +72,8 @@ function buildPromptFromString(signatureStr: string, inputs: Record function buildPromptFromClass( signatureClass: typeof Signature, - inputs: Record + inputs: Record, + demoBlock = '' ): string { const inputFields = signatureClass.getInputFields(); const outputFields = signatureClass.getOutputFields(); @@ -50,6 +84,10 @@ function buildPromptFromClass( prompt += `${signatureClass.description}\n\n`; } + // After the task description, before the real input: the model reads what + // the task is, then sees it done, then does it. + prompt += demoBlock; + Object.entries(inputFields).forEach(([key, config]) => { if (inputs[key] !== undefined) { const prefix = config.prefix || `${key}:`; @@ -66,6 +104,174 @@ function buildPromptFromClass( return prompt.trim(); } +/** + * Render worked examples as a prompt preamble. + * + * Exported so a caller can inspect exactly what few-shot text a set of demos + * produces — useful when tuning a prompt by hand. Returns an empty string when + * there is nothing to show, so callers can concatenate unconditionally. + */ +export function renderDemos( + signature: typeof Signature | string, + demos: Example[] = [], + options: RenderDemosOptions = {} +): string { + if (demos.length === 0) { + return ''; + } + + const format = options.format ?? 'labelled'; + const { inputs: inputNames, outputs: outputNames } = signatureFieldNames(signature); + const inputFields = typeof signature === 'string' ? {} : signature.getInputFields(); + + const blocks: string[] = []; + for (const demo of demos) { + const { inputs, outputs } = splitDemo(demo, inputNames, outputNames); + + // A demo sharing no fields with the signature teaches nothing, so skip + // it rather than emitting an empty numbered block. + if (Object.keys(inputs).length === 0 && Object.keys(outputs).length === 0) { + continue; + } + + const body = + format === 'json' + ? renderJsonDemo(inputs, outputs) + : renderLabelledDemo(inputs, outputs, inputFields); + blocks.push(`Example ${blocks.length + 1}:\n${body}`); + } + + if (blocks.length === 0) { + return ''; + } + + const verb = blocks.length === 1 ? 'is' : 'are'; + const noun = blocks.length === 1 ? 'example' : 'examples'; + // The labelled form is the only one that can promise "the same format": on + // the JSON path the schema instruction, not the demo, dictates the shape. + const trailer = + format === 'json' + ? 'Now complete the next one.' + : 'Now complete the next one in the same format.'; + + return ( + `Here ${verb} ${blocks.length} worked ${noun} of this task:\n\n` + + `${blocks.join('\n\n')}\n\n` + + `${trailer}\n\n` + ); +} + +function renderLabelledDemo( + inputs: Record, + outputs: Record, + inputFields: Record +): string { + const lines: string[] = []; + + for (const [key, value] of Object.entries(inputs)) { + const prefix = inputFields[key]?.prefix || `${key}:`; + lines.push(`${prefix} ${formatDemoValue(value)}`); + } + // Output labels stay plain `key: value` even when the input side uses a + // custom prefix: that is the shape parseOutput reads back, and a demo + // teaching any other shape would teach the model to break the parser. + for (const [key, value] of Object.entries(outputs)) { + lines.push(`${key}: ${formatDemoValue(value)}`); + } + + return lines.join('\n'); +} + +function renderJsonDemo(inputs: Record, outputs: Record): string { + return `input: ${JSON.stringify(inputs)}\noutput: ${JSON.stringify(outputs)}`; +} + +/** A signature's declared field names, in declaration order. */ +function signatureFieldNames(signature: typeof Signature | string): { + inputs: string[]; + outputs: string[]; +} { + if (typeof signature === 'string') { + const parsed = Signature.parseStringSignature(signature); + return { inputs: parsed.inputs, outputs: parsed.outputs }; + } + return { + inputs: Object.keys(signature.getInputFields()), + outputs: Object.keys(signature.getOutputFields()), + }; +} + +/** + * Split one demo into its input half and its output half. + * + * An `Example` that has been through `withInputs()` already knows its own split, + * so honour it. One that has not is split by the signature instead, which is why + * `new Example({ question, answer })` works as a demo without extra ceremony. + * + * Declared fields lead, in signature order, so demos stay stable and match the + * shape of the real call. An example that declared its own split may also carry + * output fields the signature never declared, and those follow — `reasoning` on + * a bootstrapped `ChainOfThought` demo is exactly that, and dropping it would + * throw away the most valuable part of the trace. Where no split was declared + * there is no way to tell a stray key from an input, so only declared fields + * render. + */ +function splitDemo( + demo: Example, + inputNames: string[], + outputNames: string[] +): { inputs: Record; outputs: Record } { + let inputSource: Record; + let outputSource: Record; + let declaredOwnSplit: boolean; + + try { + inputSource = demo.getInputs(); + outputSource = demo.getOutputs(); + declaredOwnSplit = true; + } catch { + // No explicit input keys: let the signature decide which side is which. + const data = demo.toObject(); + inputSource = data; + outputSource = data; + declaredOwnSplit = false; + } + + const extras = declaredOwnSplit + ? Object.keys(outputSource).filter((key) => !outputNames.includes(key)) + : []; + + return { + inputs: pickInOrder(inputSource, inputNames), + outputs: pickInOrder(outputSource, [...outputNames, ...extras]), + }; +} + +function pickInOrder(source: Record, names: string[]): Record { + const picked: Record = {}; + for (const name of names) { + const value = source[name]; + if (value !== undefined && value !== null) { + picked[name] = value; + } + } + return picked; +} + +/** Render a demo value the way {@link parseOutput} would read it back. */ +function formatDemoValue(value: unknown): string { + if (typeof value === 'string') { + return value; + } + if (value instanceof Date) { + return value.toISOString(); + } + if (typeof value === 'object') { + return JSON.stringify(value); + } + return String(value); +} + /** * Parse and validate a model's raw text output against a signature. * diff --git a/site/docs.html b/site/docs.html index e78ae13..02c2422 100644 --- a/site/docs.html +++ b/site/docs.html @@ -64,6 +64,7 @@

Everything the
library does.

  • Testing
  • API reference
  • Migrating to 0.5
    • +
  • Few-shot & optimizers
    • @@ -591,6 +592,141 @@

    Migrating to 0.5

    +
    +

    21

    +

    Few-shot and optimizers

    +

    + Everything above writes prompts by hand. An optimizer writes them from data + instead: give it labelled examples and a metric, and it works out which + worked examples belong in the prompt. This is the idea TS-DSPy takes from + DSPy — a program that improves itself, rather than a prompt you keep + editing. +

    + +

    Demos

    +

    + A demo is an Example rendered into the prompt before the real + input, showing the task performed correctly. Demos render in whichever + shape the reply is expected to take — labelled + field: value for a provider answering in text, JSON for one + whose decoding is constrained to a schema — so they teach the output + format as well as the task. +

    + 
    import { Example, Predict } from '@ts-dspy/core'
+
+const demos = [
+  new Example({ ticket: 'My card was charged twice.', team: 'billing' }).withInputs('ticket'),
+  new Example({ ticket: 'Export does nothing on Safari.', team: 'bug' }).withInputs('ticket'),
+]
+
+const router = new Predict(RouteTicket, { demos })
    +

    + withInputs() marks which fields are the question; the rest is + the answer. Omit it and the signature decides the split instead. The + original two-argument form, new Predict(Sig, lm), still works + — pass { lm, demos } when you want both. + withDemos() returns a configured copy, leaving the + module you called it on alone. +

    + +

    LabeledFewShot

    +

    + Selects k of your own labelled examples. No model calls, so + compiling is instant and free — try this before bootstrapping. +

    + 
    import { LabeledFewShot } from '@ts-dspy/core'
+
+const compiled = new LabeledFewShot({ k: 3, seed: 42 })
+  .compile(new Predict(RouteTicket), { trainset })
    + +

    BootstrapFewShot

    +

    + Runs the module over the trainset, scores every attempt with your metric, + and promotes the runs that passed into demos. The program learns from its + own successes. +

    + 
    import { BootstrapFewShot } from '@ts-dspy/core'
+
+const optimizer = new BootstrapFewShot({
+  metric: (example, prediction) => example.get('team') === prediction.get('team'),
+  maxBootstrappedDemos: 4,
+  teacher: strongLM,        // generates the demos, at compile time only
+  concurrency: 4,
+  seed: 42,
+  onProgress: (event) => log(`${event.index + 1}/${event.total} ${event.status}`),
+})
+
+const compiled = await optimizer.compile(new Predict(RouteTicket, cheapLM), { trainset })
    +

    + With a teacher, a stronger model does the trainset runs and the + cheaper student ends up imitating its work. You pay for the strong model + once, when compiling — never at run time. Without one, the student + bootstraps from itself. +

    +

    + The run stops as soon as it has maxBootstrappedDemos, so a + trainset of five hundred rows does not cost five hundred calls to keep + four demos. Setting maxBootstrappedDemos: 0 alongside + maxLabeledDemos compiles from labels alone and makes no model + calls whatsoever. +

    + +

    Metrics

    +

    + A metric is (example, prediction) => number | boolean, and may + be async. Booleans are taken at face value; a number counts as a pass when + it reaches threshold, which defaults to 0.5. The + metric is the entire definition of “good” — everything the + optimizer does follows from it. +

    + +

    Options

    +
    + + + + + + + + + + + + + + +
    OptionDefaultMeaning
    metricrequiredJudges each attempt.
    maxBootstrappedDemos4Demos to promote. The run stops once it has this many.
    maxLabeledDemos0Plain labels shown alongside the bootstrapped ones.
    concurrency4Trainset examples attempted at once.
    seed0Seeds the trainset shuffle.
    threshold0.5Score at which a numeric metric passes.
    teacher—Stronger model used to generate demos.
    inputKeys—Input fields, when examples declare none.
    onProgress—Per-example outcome callback.
    callOptions—Call options for the trainset runs.
    +
    + +
    + Deterministic +

    + Both optimizers are seeded, so the same seed and the same trainset + compile to the same demos — a result you can reproduce, diff, and + write a stable test against. Progress events arrive in completion order, + which with concurrency above 1 is not trainset order; the demos are + deterministic regardless. +

    +
    + +
    + Failures are skipped +

    + A trainset example whose attempt throws — a provider error, a reply + that fails validation — is reported as an error event + and skipped. One bad row must not throw away every demo already paid for. +

    +
    + +

    + A full run, including a baseline to compare against, is in + examples/optimizer.ts: +

    + 
    export OPENAI_API_KEY="sk-..."
+npm run example:optimizer
    +
    +