feat(orchestrator): add GET /metrics endpoint with task and step counters

docs/development.md

@@ -180,6 +180,56 @@ curl -s -X POST http://localhost:3000/api/tasks/preview \
   -d '{"prompt": "summarise yesterday Stellar DEX volume", "budget": 1.0}' | jq .
 ```
 
+### GET /metrics
+
+Operational counters for the orchestrator process — task throughput, step
+outcomes, USDC released, and step-latency percentiles. Intended for a status
+dashboard or an alerting rule; no authentication, no user address required.
+
+The response is **plain JSON** (not Prometheus text exposition). The shape is
+stable — fields may be added, but existing ones keep their names and meaning.
+
+```json
+{
+  "uptime_seconds": 3612,
+  "tasks": { "total": 42, "active": 1, "completed": 38, "failed": 2, "interrupted": 1 },
+  "steps": { "executed": 126, "failed": 4, "timed_out": 2 },
+  "usdc_released_total": 2.34,
+  "step_duration_ms": { "count": 126, "p50_ms": 840, "p95_ms": 15000, "max_ms": 21400 },
+  "memory": { "rss_bytes": 91234304, "heap_used_bytes": 42118400 }
+}
+```
+
+Field notes:
+
+| Field | Meaning |
+|---|---|
+| `tasks.total` | Tasks submitted, including those seeded from the activity log |
+| `tasks.active` | In flight **in this process** right now |
+| `tasks.interrupted` | Were in flight when a previous process exited — seeded at startup, never incremented at runtime |
+| `steps.executed` | Step attempts that finished, successfully or not |
+| `steps.failed` | Subset of `executed` that failed |
+| `steps.timed_out` | Subset of `failed` whose error text reads as a timeout |
+| `usdc_released_total` | USDC released from the vault to the orchestrator wallet |
+| `step_duration_ms` | Percentiles over the most recent 1024 step attempts (fixed-size ring, so memory is bounded); `null` until the first step runs |
+
+Counters are per-process and dependency-free (`packages/orchestrator/src/metrics.ts`
+— no metrics library). On startup they are seeded from `data/activity-log.json`
+and `data/task-results.json` so a redeploy doesn't zero the totals. Because
+activity events are only written for tasks that carry a `user_address`,
+anonymous tasks contribute to live counters but are not restored across a
+restart.
+
+`tasks.total` is not guaranteed to equal `active + completed + failed +
+interrupted`: a task interrupted by a restart is counted in `total` when it
+starts and in `interrupted` only after the *next* startup reads the log.
+
+**Example curl**
+
+```bash
+curl -s http://localhost:3000/metrics | jq .
+```
+
 ## Testing
 
 Unit tests use [Vitest](https://vitest.dev/) and are colocated with the code
@@ -193,6 +243,9 @@ to verify in isolation:
 - `packages/orchestrator/src/validator.test.ts` — execution plan validation.
 - `packages/orchestrator/src/server.preview.test.ts` — `/api/tasks/preview`
   endpoint (happy path, no-agents 503, infeasible 422).
+- `packages/orchestrator/src/metrics.test.ts` — counter transitions, timeout
+  classification, percentile math, ring-buffer bounding, and startup seeding.
+- `packages/orchestrator/src/server.metrics.test.ts` — `/metrics` response shape.
 
 Run the full suite with `npm test`, or scope to a package with
 `npm test -w packages/registry`.

packages/orchestrator/src/executor.ts

@@ -27,6 +27,7 @@ import { makeX402Payment } from './x402-client.js';
 import { makeMPPPayment } from './mpp-client.js';
 import { rateResponse } from './rater.js';
 import { releasePayment, VAULT_ACTIVE } from './agent-vault-client.js';
+import { stepExecuted, stepFailed, usdcReleased } from './metrics.js';
 
 // ── Types ────────────────────────────────────────────────────────────────────
 
@@ -279,6 +280,7 @@ export class PlanExecutor extends EventEmitter {
         }
 
         releaseHash = typeof released === 'string' ? released : null;
+        usdcReleased(amountUsdc);
         // Wrap emit in try/catch — a serialization error must never kill a step
         try {
           this.emit('budget_released', {
@@ -322,6 +324,7 @@ export class PlanExecutor extends EventEmitter {
       }
 
       const latency_ms = Date.now() - stepStart;
+      stepExecuted(latency_ms);
       const quality_rating = await rateResponse(step.action, output);
 
       const result: StepResult = {
@@ -383,7 +386,12 @@ export class PlanExecutor extends EventEmitter {
     }
   }
 
+  /**
+   * Build the StepResult for a failed step. Every failure path routes through
+   * here, so this is also where the failure is counted.
+   */
   private makeFailedResult(step: ExecutionStep, error: string, latency_ms: number): StepResult {
+    stepFailed(error, latency_ms);
     return {
       step_id: step.step_id,
       agent_id: step.agent_id,

packages/orchestrator/src/metrics.test.ts

@@ -0,0 +1,273 @@
+import { describe, it, expect, beforeEach } from 'vitest';
+import {
+  getMetrics,
+  resetMetrics,
+  seedMetrics,
+  stepExecuted,
+  stepFailed,
+  taskCompleted,
+  taskFailed,
+  taskStarted,
+  usdcReleased,
+  isTimeoutError,
+} from './metrics.js';
+
+beforeEach(() => {
+  resetMetrics();
+});
+
+describe('task counters', () => {
+  it('starts at zero', () => {
+    const m = getMetrics();
+    expect(m.tasks).toEqual({ total: 0, active: 0, completed: 0, failed: 0, interrupted: 0 });
+    expect(m.steps).toEqual({ executed: 0, failed: 0, timed_out: 0 });
+    expect(m.usdc_released_total).toBe(0);
+  });
+
+  it('moves a task from active to completed', () => {
+    taskStarted('t1');
+    expect(getMetrics().tasks).toMatchObject({ total: 1, active: 1, completed: 0 });
+
+    taskCompleted('t1');
+    expect(getMetrics().tasks).toMatchObject({ total: 1, active: 0, completed: 1, failed: 0 });
+  });
+
+  it('moves a failed task out of active, not into both', () => {
+    taskStarted('t1');
+    taskFailed('t1');
+
+    const m = getMetrics();
+    expect(m.tasks.active).toBe(0);
+    expect(m.tasks.failed).toBe(1);
+    expect(m.tasks.completed).toBe(0);
+  });
+
+  it('is idempotent on repeated terminal transitions', () => {
+    taskStarted('t1');
+    taskFailed('t1');
+    taskFailed('t1');
+    taskCompleted('t1');
+
+    const m = getMetrics();
+    expect(m.tasks).toMatchObject({ total: 1, active: 0, failed: 1, completed: 0 });
+  });
+
+  it('ignores a terminal transition for a task that never started', () => {
+    taskCompleted('never-seen');
+    expect(getMetrics().tasks).toMatchObject({ total: 0, completed: 0 });
+  });
+
+  it('does not double-count a repeated start', () => {
+    taskStarted('t1');
+    taskStarted('t1');
+    expect(getMetrics().tasks).toMatchObject({ total: 1, active: 1 });
+  });
+
+  it('tracks concurrent tasks independently', () => {
+    taskStarted('a');
+    taskStarted('b');
+    taskStarted('c');
+    expect(getMetrics().tasks.active).toBe(3);
+
+    taskCompleted('b');
+    const m = getMetrics();
+    expect(m.tasks).toMatchObject({ total: 3, active: 2, completed: 1 });
+  });
+
+  it('keeps counts correct when terminal transitions interleave out of order', async () => {
+    taskStarted('a');
+    taskStarted('b');
+
+    await Promise.all([
+      (async () => {
+        await Promise.resolve();
+        taskFailed('b');
+      })(),
+      (async () => {
+        taskCompleted('a');
+      })(),
+    ]);
+
+    expect(getMetrics().tasks).toMatchObject({ total: 2, active: 0, completed: 1, failed: 1 });
+  });
+});
+
+describe('step counters', () => {
+  it('counts successes toward executed only', () => {
+    stepExecuted(100);
+    stepExecuted(200);
+    expect(getMetrics().steps).toEqual({ executed: 2, failed: 0, timed_out: 0 });
+  });
+
+  it('counts failures toward executed and failed', () => {
+    stepExecuted(100);
+    stepFailed('Agent health check failed: http://x/health', 50);
+    expect(getMetrics().steps).toEqual({ executed: 2, failed: 1, timed_out: 0 });
+  });
+
+  it('classifies timeout errors as a subset of failures', () => {
+    stepFailed('The operation was aborted due to timeout', 15000);
+    stepFailed('Request timed out', 15000);
+    stepFailed('Agent not found: agent-x', 1);
+
+    expect(getMetrics().steps).toEqual({ executed: 3, failed: 3, timed_out: 2 });
+  });
+
+  it('recognises timeout phrasings without false positives', () => {
+    expect(isTimeoutError('TimeoutError: signal timed out')).toBe(true);
+    expect(isTimeoutError('operation was aborted')).toBe(true);
+    expect(isTimeoutError('timed-out waiting for agent')).toBe(true);
+    expect(isTimeoutError('Vault release failed for step 2')).toBe(false);
+    expect(isTimeoutError('')).toBe(false);
+    expect(isTimeoutError(null)).toBe(false);
+  });
+});
+
+describe('usdc released', () => {
+  it('accumulates released payments', () => {
+    usdcReleased(0.02);
+    usdcReleased(0.03);
+    usdcReleased(0.02);
+    expect(getMetrics().usdc_released_total).toBe(0.07);
+  });
+
+  it('ignores non-positive and non-finite amounts', () => {
+    usdcReleased(0);
+    usdcReleased(-1);
+    usdcReleased(Number.NaN);
+    expect(getMetrics().usdc_released_total).toBe(0);
+  });
+});
+
+describe('step duration summary', () => {
+  it('reports nulls when no steps have run', () => {
+    expect(getMetrics().step_duration_ms).toEqual({
+      count: 0,
+      p50_ms: null,
+      p95_ms: null,
+      max_ms: null,
+    });
+  });
+
+  it('computes p50, p95 and max over recorded samples', () => {
+    for (let i = 1; i <= 100; i++) stepExecuted(i);
+
+    const summary = getMetrics().step_duration_ms;
+    expect(summary.count).toBe(100);
+    expect(summary.p50_ms).toBe(50);
+    expect(summary.p95_ms).toBe(95);
+    expect(summary.max_ms).toBe(100);
+  });
+
+  it('includes failed-step durations', () => {
+    stepExecuted(10);
+    stepFailed('boom', 90);
+    expect(getMetrics().step_duration_ms).toMatchObject({ count: 2, max_ms: 90 });
+  });
+
+  it('bounds memory at the ring capacity while executed keeps counting', () => {
+    for (let i = 0; i < 5000; i++) stepExecuted(i);
+
+    const m = getMetrics();
+    expect(m.steps.executed).toBe(5000);
+    expect(m.step_duration_ms.count).toBe(1024);
+    // The ring retains the most recent samples, so early values are gone
+    expect(m.step_duration_ms.max_ms).toBe(4999);
+    expect(m.step_duration_ms.p50_ms).toBeGreaterThan(4000);
+  });
+
+  it('ignores negative durations', () => {
+    stepExecuted(-5);
+    expect(getMetrics().step_duration_ms.count).toBe(0);
+  });
+});
+
+describe('uptime and memory', () => {
+  it('reports a non-negative uptime and real memory readings', () => {
+    const m = getMetrics();
+    expect(m.uptime_seconds).toBeGreaterThanOrEqual(0);
+    expect(m.memory.rss_bytes).toBeGreaterThan(0);
+    expect(m.memory.heap_used_bytes).toBeGreaterThan(0);
+  });
+});
+
+describe('seedMetrics', () => {
+  it('restores task totals and spend from the activity pulse', () => {
+    seedMetrics({
+      pulse: {
+        total_tasks: 10,
+        total_completed: 7,
+        total_failed: 2,
+        active_tasks: 1,
+        total_spent_usdc: 0.35,
+      },
+    });
+
+    const m = getMetrics();
+    expect(m.tasks).toEqual({
+      total: 10,
+      active: 0,
+      completed: 7,
+      failed: 2,
+      interrupted: 1,
+    });
+    expect(m.usdc_released_total).toBe(0.35);
+  });
+
+  it('counts tasks in flight at shutdown as interrupted, never active', () => {
+    seedMetrics({
+      pulse: {
+        total_tasks: 3,
+        total_completed: 1,
+        total_failed: 0,
+        active_tasks: 2,
+        total_spent_usdc: 0,
+      },
+    });
+
+    expect(getMetrics().tasks.active).toBe(0);
+    expect(getMetrics().tasks.interrupted).toBe(2);
+  });
+
+  it('seeds step counters and durations from stored task results', () => {
+    seedMetrics({
+      taskResults: [
+        {
+          steps: [
+            { success: true, error: null, latency_ms: 100 },
+            { success: false, error: 'signal timed out', latency_ms: 15000 },
+          ],
+        },
+        {
+          steps: [{ success: false, error: 'Agent not found: x', latency_ms: 5 }],
+        },
+      ],
+    });
+
+    const m = getMetrics();
+    expect(m.steps).toEqual({ executed: 3, failed: 2, timed_out: 1 });
+    expect(m.step_duration_ms).toMatchObject({ count: 3, max_ms: 15000 });
+  });
+
+  it('adds to live counters rather than replacing them', () => {
+    taskStarted('live');
+    taskCompleted('live');
+
+    seedMetrics({
+      pulse: {
+        total_tasks: 4,
+        total_completed: 4,
+        total_failed: 0,
+        active_tasks: 0,
+        total_spent_usdc: 0.1,
+      },
+    });
+
+    expect(getMetrics().tasks).toMatchObject({ total: 5, completed: 5 });
+  });
+
+  it('tolerates an empty seed', () => {
+    expect(() => seedMetrics({})).not.toThrow();
+    expect(getMetrics().tasks.total).toBe(0);
+  });
+});