feat(core): few-shot demos and optimizers (LabeledFewShot, BootstrapFewShot)

.changeset/few-shot-optimizers.md

@@ -0,0 +1,26 @@
+---
+'@ts-dspy/core': minor
+---
+
+Add few-shot demos and optimizers, so a program can improve itself from data
+rather than from prompt edits.
+
+`Predict` now accepts demos — `new Predict(Sig, { demos })`, or `withDemos()` for
+a configured copy — and renders them into the prompt as worked examples before
+the real input, in the same labelled `field: value` shape the parser reads back.
+A prompt built without demos is byte-for-byte what it was before.
+
+Two optimizers turn a labelled trainset into those demos. `LabeledFewShot`
+selects _k_ of your own labels and makes no model calls at all. `BootstrapFewShot`
+runs the module over the trainset, scores each attempt with a metric, and
+promotes the runs that passed into demos; a `teacher` option generates them with
+a stronger model that the cheaper student then imitates, so you pay for the
+strong model once, at compile time.
+
+Both are deterministic given a seed, so a compiled program can be reproduced and
+tested. Trainset runs use bounded concurrency, and an example whose attempt
+throws is skipped rather than failing the whole compile. Progress is reported
+through an optional callback.
+
+`Predict` also gains `withLM()`, and `renderDemos()` is exported for inspecting
+the few-shot text a set of demos produces.

README.md

@@ -12,6 +12,13 @@ Model output is **validated at runtime** against the shape you declared. If a
 field you declared as a number comes back as prose, you get a `ValidationError`
 naming the field — not a string masquerading as a number.
 
+Programs also **optimize themselves from data**, the part of DSPy that makes it
+more than a prompt template library. Give `BootstrapFewShot` a labelled trainset
+and a metric and it runs your module, keeps the runs that passed, and puts them
+back in the prompt as worked examples — optionally generated by a stronger
+teacher model that your cheaper one then imitates. Runnable end to end in
+[`examples/optimizer.ts`](examples/optimizer.ts).
+
 ```bash
 npm install @ts-dspy/core @ts-dspy/openai
 ```

examples/optimizer.ts

@@ -0,0 +1,171 @@
+/**
+ * Few-shot demos and optimizers.
+ *
+ * A program that improves itself from data: run a labelled trainset, keep the
+ * runs a metric approved of, and put those runs in the prompt as worked
+ * examples. Optionally let a stronger model generate the demos that a cheaper
+ * one then imitates.
+ *
+ *   export OPENAI_API_KEY="sk-..."
+ *   npm run example:optimizer
+ */
+import {
+    Signature,
+    InputField,
+    OutputField,
+    Predict,
+    Example,
+    LabeledFewShot,
+    BootstrapFewShot,
+    configure,
+    type Metric,
+} from '@ts-dspy/core';
+import { OpenAILM } from '@ts-dspy/openai';
+import { requireEnv, section } from './utils';
+
+// --- Signature --------------------------------------------------------------
+
+class RouteTicket extends Signature {
+    static description =
+        'Route a support ticket to the team that owns it. ' +
+        'Answer with exactly one of: billing, bug, account, feature.';
+
+    @InputField({ description: 'the ticket text' })
+    ticket!: string;
+
+    @OutputField({ description: 'billing, bug, account, or feature' })
+    team!: string;
+}
+
+type TicketRouting = { team: string };
+
+// --- Trainset ---------------------------------------------------------------
+
+// Labelled data. `withInputs` marks which fields are the question, so the rest
+// is understood to be the answer — an optimizer must never feed the label back
+// in as input, or every run would be trivially correct.
+const labelled = [
+    ['My card was charged twice this month.', 'billing'],
+    ['The export button does nothing on Safari.', 'bug'],
+    ['I cannot reset my password, the email never arrives.', 'account'],
+    ['Could you add dark mode to the dashboard?', 'feature'],
+    ['Why is my invoice higher than the quoted plan price?', 'billing'],
+    ['The app crashes when I upload a file over 10 MB.', 'bug'],
+    ['Please remove my colleague from the workspace.', 'account'],
+    ['It would help if reports could be scheduled weekly.', 'feature'],
+    ['I was billed after cancelling my subscription.', 'billing'],
+    ['Search returns no results even for exact titles.', 'bug'],
+    ['Can I merge two accounts into one?', 'account'],
+    ['A Slack integration would save us a lot of copying.', 'feature'],
+].map(([ticket, team]) => new Example({ ticket, team }).withInputs('ticket'));
+
+// Disjoint splits. Scoring on rows the optimizer compiled from would let a demo
+// for a ticket appear in the prompt used to classify that same ticket, and the
+// comparison against the baseline would mean nothing.
+const trainset = labelled.slice(0, 8);
+const devset = labelled.slice(8);
+
+// --- Metric -----------------------------------------------------------------
+
+// The metric is the whole specification of "good" — everything the optimizer
+// does is downstream of it. This one is exact match on a normalised label.
+const routedCorrectly: Metric = (example, prediction) => {
+    const expected = String(example.get('team')).trim().toLowerCase();
+    const actual = String(prediction.get('team') ?? '')
+        .trim()
+        .toLowerCase();
+    return actual === expected;
+};
+
+async function accuracy(module: Predict<typeof RouteTicket, TicketRouting>): Promise<number> {
+    let correct = 0;
+    for (const example of devset) {
+        try {
+            const prediction = await module.forward(example.getInputs());
+            if (routedCorrectly(example, prediction)) {
+                correct += 1;
+            }
+        } catch {
+            // A run that fails is a run that scored nothing.
+        }
+    }
+    return correct / devset.length;
+}
+
+async function main(): Promise<void> {
+    const apiKey = requireEnv('OPENAI_API_KEY');
+
+    // A cheap student, and a stronger teacher used only at compile time.
+    const student = new OpenAILM({ apiKey, model: 'gpt-4.1-mini' });
+    const teacher = new OpenAILM({ apiKey, model: 'gpt-4.1' });
+    configure({ lm: student });
+
+    // --- Baseline -----------------------------------------------------------
+    section('Baseline (no demos)');
+
+    const baseline = new Predict<typeof RouteTicket, TicketRouting>(RouteTicket);
+    console.log(`accuracy: ${(await accuracy(baseline)) * 100}%`);
+
+    // --- LabeledFewShot -----------------------------------------------------
+    section('LabeledFewShot');
+
+    // No model calls at all: it just puts k of your labels in the prompt. Seeded,
+    // so the same seed always picks the same demos.
+    const labeled = new LabeledFewShot({ k: 3, seed: 42 }).compile(baseline, { trainset });
+
+    for (const demo of labeled.getDemos()) {
+        console.log(`  demo: ${demo.get('ticket')} -> ${demo.get('team')}`);
+    }
+    console.log(`accuracy: ${(await accuracy(labeled)) * 100}%`);
+
+    // --- BootstrapFewShot ---------------------------------------------------
+    section('BootstrapFewShot');
+
+    // Run the trainset through the teacher, keep the runs the metric approved
+    // of, and attach them to the student. The student ends up imitating work the
+    // stronger model did, without paying for the stronger model at run time.
+    const optimizer = new BootstrapFewShot({
+        metric: routedCorrectly,
+        maxBootstrappedDemos: 4,
+        concurrency: 4,
+        seed: 42,
+        teacher,
+        callOptions: { temperature: 0 },
+        // Library code never prints; progress arrives through this callback, and
+        // the example is what decides to put it on the terminal.
+        onProgress: (event) => {
+            const detail = event.status === 'error' ? ` (${String(event.error)})` : '';
+            console.log(`  [${event.index + 1}/${event.total}] ${event.status}${detail}`);
+        },
+    });
+
+    const compiled = await optimizer.compile(baseline, { trainset });
+
+    console.log(`\nbootstrapped ${compiled.getDemos().length} demos:`);
+    for (const demo of compiled.getDemos()) {
+        console.log(`  ${demo.get('ticket')} -> ${demo.get('team')}`);
+    }
+
+    console.log(`\naccuracy: ${(await accuracy(compiled)) * 100}%`);
+
+    // --- Using the compiled program -----------------------------------------
+    section('Compiled program');
+
+    // Demos are not magic: they are text placed in front of the real input, in
+    // whichever shape the reply is expected to take — labelled `field: value`
+    // for a text provider, JSON for one with native structured output.
+    const routed = await compiled.forward({ ticket: 'Two invoices arrived this month.' });
+    console.log(`team: ${routed.team}`);
+
+    // --- Usage --------------------------------------------------------------
+    section('Usage');
+    const teacherUsage = teacher.getUsage();
+    const studentUsage = student.getUsage();
+    console.log(`teacher: ${teacherUsage.requestCount} requests (compile time only)`);
+    console.log(`student: ${studentUsage.requestCount} requests`);
+}
+
+main().catch((error) => {
+    console.error(error);
+    process.exit(1);
+});

package.json

@@ -29,7 +29,8 @@
     "example:gemini": "tsx examples/basic-gemini-example.ts",
     "example:anthropic": "tsx examples/basic-anthropic-example.ts",
     "example:zod": "tsx examples/zod-signature.ts",
-    "example:ollama": "tsx examples/ollama-local.ts"
+    "example:ollama": "tsx examples/ollama-local.ts",
+    "example:optimizer": "tsx examples/optimizer.ts"
   },
   "devDependencies": {
     "@changesets/cli": "^2.29.8",

packages/core/src/index.ts

@@ -66,16 +66,30 @@ export type { RepairFormat } from './core/repair';
 
 // Modules
 export { Predict } from './modules/predict';
+export type { PredictOptions } from './modules/predict';
 export type { StreamOptions, PredictionStream, PartialOutput } from './modules/predict';
 export { ChainOfThought } from './modules/chain-of-thought';
 export { RespAct } from './modules/respact';
 export type { ToolFunction, ToolWithDescription, ToolDefinition } from './modules/respact';
 
+// Optimizers
+export { LabeledFewShot, BootstrapFewShot } from './optimizers';
+export type {
+    Metric,
+    MetricResult,
+    DemoModule,
+    LabeledFewShotOptions,
+    LabeledFewShotConfig,
+    BootstrapFewShotOptions,
+    BootstrapFewShotConfig,
+    BootstrapProgressEvent,
+} from './optimizers';
+
 // Evaluation
 export * from './evaluate';
 
 // Utilities
-export { buildPrompt, parseOutput } from './utils/parsing';
+export { buildPrompt, parseOutput, renderDemos } from './utils/parsing';
 export { fieldConfigToZod, buildOutputSchema, buildOutputJsonSchema } from './utils/schema';
 export { parsePartialJson } from './utils/partial-json';
 export type { PartialJsonOptions } from './utils/partial-json';