📈 Add Rudder-specific usage telemetry

bin/rudder-prompt-hook.ts

@@ -3,6 +3,11 @@
 import { join } from 'node:path';
 import { closeDb } from '../src/db/client.ts';
 import { parseAgentPromptSource, recordPromptHookEvent } from '../src/prompt-hook.ts';
+import {
+  captureRudderUsageEvent,
+  rudderUsageEvents,
+  type RudderUsageEvent,
+} from '../src/rudder-telemetry.ts';
 import { captureException, shutdown } from '../src/telemetry.ts';
 
 type AgentSource = 'claude-code' | 'codex' | 'cursor';
@@ -27,6 +32,16 @@ function hookContext(args: string[]): HookContext {
   return { source: parseAgentPromptSource(sourceArgument(args)) };
 }
 
+function rudderUsageEventArgument(args: string[]): RudderUsageEvent | null {
+  if (args[0] !== '--rudder-event') return null;
+  if (args.length !== 2 || !rudderUsageEvents.includes(args[1] as RudderUsageEvent)) {
+    throw new TypeError(
+      `usage: rudder-prompt-hook --rudder-event <${rudderUsageEvents.join('|')}>`
+    );
+  }
+  return args[1] as RudderUsageEvent;
+}
+
 async function readStdin(): Promise<string> {
   const chunks: Buffer[] = [];
   for await (const chunk of process.stdin) {
@@ -36,13 +51,19 @@ async function readStdin(): Promise<string> {
 }
 
 try {
-  const context = hookContext(process.argv.slice(2));
-  if (context.root) {
-    process.env.RUDDER_MIGRATIONS_PATH ||= join(context.root, 'dist', 'drizzle');
-  }
+  const args = process.argv.slice(2);
+  const usageEvent = rudderUsageEventArgument(args);
   const input = await readStdin();
   const payload: unknown = JSON.parse(input);
-  recordPromptHookEvent(context.source, payload);
+  if (usageEvent) {
+    captureRudderUsageEvent(usageEvent, payload);
+  } else {
+    const context = hookContext(args);
+    if (context.root) {
+      process.env.RUDDER_MIGRATIONS_PATH ||= join(context.root, 'dist', 'drizzle');
+    }
+    recordPromptHookEvent(context.source, payload);
+  }
 } catch (error) {
   // Prompt capture is optional metadata. A hook failure must not interrupt the host agent.
   try {

package.json

@@ -48,7 +48,7 @@
     "build": "rm -rf dist && esbuild bin/rudder-prompt-hook.ts --bundle --platform=node --format=esm --target=node24 --outfile=dist/rudder-prompt-hook.mjs && cp -R drizzle dist/drizzle",
     "pretest": "npm run build",
     "test": "node --test",
-    "test:coverage": "npm run build && c8 node --test && diff-cover coverage/lcov.info --fail-under=80 --show-uncovered --include-untracked",
+    "test:coverage": "npm run build && c8 node --test && diff-cover coverage/lcov.info --fail-under=90 --show-uncovered --include-untracked",
     "prepack": "npm run build",
     "prepublishOnly": "npm run typecheck && npm test"
   },

skills/rudder/SKILL.md

@@ -61,11 +61,21 @@ Coverage is loop control, never a source of test intent.
   implementation, improve coverage, or exercise a defensive case.
 - After the first test pass, if coverage is below the target, stop editing tests.
   Select one uncovered behavior.
+  Increment the run's question counter and best-effort record the question before showing it:
+
+  ```text
+  node <skill-directory>/scripts/telemetry.mjs question-asked \
+    --cwd <repository-root> \
+    --run-id <rudder-run-id> \
+    --question-number <question-counter>
+  ```
+
+  Do not pass the question or any answer text to the helper.
   Ask one concrete question about the expected behavior.
 - Do not write the next test until the user answers.
   Repository code may help frame the question, but it cannot supply the answer.
-- After each answer, rerun `scripts/context.mjs`, require a captured prompt
-  record for the answer, and queue only the expectation that answer authorizes.
+- After each answer, rerun `scripts/context.mjs` with `--phase refresh`, `--run-id <rudder-run-id>`, and `--base <resolved-base-ref>`.
+  Require a captured prompt record for the answer, and queue only the expectation that answer authorizes.
 - Complete the red-green cycle for every authorized expectation in its owning agent before integrating the batch.
   Use the queue answered rewrites guidance to set up owning agents.
   Do not measure coverage while a rewrite is pending.
@@ -123,15 +133,18 @@ For every new or changed expectation:
    Determine the requested coverage target.
    Prefer the repository's configured coverage threshold.
    Ask for a target only when neither the request nor repository provides one.
-2. Run `scripts/context.mjs` relative to this file with the repository working
-   directory:
+2. Run `scripts/context.mjs` relative to this file with the repository working directory:
 
    ```text
    node <skill-directory>/scripts/context.mjs \
      --cwd <repository-root> \
+     --phase start \
      [--base <target-ref>]
    ```
 
+   Retain the returned `rudderRunId` and `baseRef` for every later helper call in this run.
+   Use that returned `baseRef` as `<resolved-base-ref>` in every later context, backup, and completion helper call.
+   Initialize the run's question counter to zero.
 3. Inspect the returned merge base, changed paths, and captured prompts.
    Inspect repository instructions, the production diff, and existing tests.
    Inspect the native test and coverage configuration.
@@ -156,7 +169,8 @@ For every new or changed expectation:
    ```text
    node <skill-directory>/scripts/backup-tests.mjs \
      --cwd <repository-root> \
-     --base <target-ref> \
+     --base <resolved-base-ref> \
+     --run-id <rudder-run-id> \
      --path <test-path> \
      [--path <test-path> ...]
    ```
@@ -192,6 +206,22 @@ For every new or changed expectation:
     Continue asking independent questions until the batch must join.
     After joining, run the combined suites and coverage before selecting another uncovered behavior.
     Continue until the target passes or the user tells you to stop the flow.
+12. Before the final report, record the verified Rudder outcome:
+
+    ```text
+    node <skill-directory>/scripts/telemetry.mjs complete \
+      --cwd <repository-root> \
+      --base <resolved-base-ref> \
+      --run-id <rudder-run-id> \
+      --status <completed|stopped|blocked> \
+      --tests-passed <yes|no|unknown> \
+      --coverage-target-met <yes|no|unknown> \
+      --questions-asked <question-counter>
+    ```
+
+    Use `completed` when the workflow reaches its normal report, `stopped` when it ends by user choice or missing intent, and `blocked` only for an external blocker.
+    Set test and coverage values only from command output already observed during this run.
+    Telemetry is best-effort; do not change the workflow result if this helper is unavailable.
 
 Report the requirements derived from intent and all files changed.
 Report commands run, coverage, unanswered ambiguities, and the backup location.

skills/rudder/scripts/backup-tests.mjs

@@ -4,6 +4,10 @@ import { execFileSync, spawnSync } from 'node:child_process';
 import { cpSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs';
 import { homedir } from 'node:os';
 import { dirname, join, relative, resolve } from 'node:path';
+import {
+  captureRudderTelemetry,
+  repositoryKey,
+} from './telemetry.mjs';
 
 function argumentValue(args, name, required = false) {
   const index = args.indexOf(name);
@@ -62,6 +66,7 @@ function safeRelativePath(root, path) {
 function main() {
   const args = process.argv.slice(2);
   const cwd = argumentValue(args, '--cwd', true);
+  const runId = argumentValue(args, '--run-id', true);
   const root = git(cwd, ['rev-parse', '--show-toplevel']);
   const baseRef = argumentValue(args, '--base') ?? 'HEAD';
   if (!git(root, ['rev-parse', '--verify', '--quiet', baseRef], true)) {
@@ -120,6 +125,24 @@ function main() {
     encoding: 'utf8',
     mode: 0o600,
   });
+  try {
+    const branch = git(
+      root,
+      ['symbolic-ref', '--quiet', '--short', 'HEAD'],
+      true
+    );
+    if (branch) {
+      captureRudderTelemetry('test-backup-created', {
+        repository: repositoryKey(root, branch),
+        branch,
+        runId,
+        approvedTestPathCount: paths.length,
+        copiedUntrackedTestPathCount: copiedUntrackedPaths.length,
+      });
+    }
+  } catch {
+    // Telemetry must not affect backup creation or its recovery metadata.
+  }
   process.stdout.write(
     `${JSON.stringify(
       { backupDirectory, metadataPath, ...metadata },

skills/rudder/scripts/context.mjs

@@ -1,11 +1,17 @@
 #!/usr/bin/env node
 
 import { execFileSync, spawnSync } from 'node:child_process';
-import { createHash } from 'node:crypto';
+import { randomUUID } from 'node:crypto';
 import { existsSync, realpathSync } from 'node:fs';
 import { homedir } from 'node:os';
-import { basename, join, resolve } from 'node:path';
+import { join } from 'node:path';
 import { DatabaseSync } from 'node:sqlite';
+import {
+  captureRudderTelemetry,
+  isTestPath,
+  repositoryKey,
+  testDiffLineCounts,
+} from './telemetry.mjs';
 
 function argumentValue(args, name) {
   const index = args.indexOf(name);
@@ -36,52 +42,6 @@ function gitNullList(cwd, args) {
   return output.split('\0').filter(Boolean);
 }
 
-function strippedRepositoryPath(path) {
-  return path.replace(/^\/+|\/+$/gu, '').replace(/\.git$/u, '');
-}
-
-function normalizeRepository(repository) {
-  const value = repository.trim();
-  const scp = /^(?:[^@/]+@)?([^:/]+):(.+)$/u.exec(value);
-  if (scp && !value.includes('://')) {
-    return `${scp[1].toLowerCase()}/${strippedRepositoryPath(scp[2])}`;
-  }
-
-  try {
-    const url = new URL(value);
-    if (url.protocol !== 'file:') {
-      return `${url.host.toLowerCase()}/${strippedRepositoryPath(
-        decodeURIComponent(url.pathname)
-      )}`;
-    }
-  } catch {
-    // Treat non-URL values as local paths.
-  }
-  return strippedRepositoryPath(value);
-}
-
-function repositoryKey(root, branch) {
-  const branchRemote = git(
-    root,
-    ['config', '--get', `branch.${branch}.remote`],
-    true
-  );
-  const remoteNames = [
-    branchRemote && branchRemote !== '.' ? branchRemote : null,
-    'origin',
-    ...((git(root, ['remote'], true) ?? '').split('\n').filter(Boolean)),
-  ].filter(Boolean);
-
-  for (const remoteName of new Set(remoteNames)) {
-    const remote = git(root, ['remote', 'get-url', remoteName], true);
-    if (remote) return normalizeRepository(remote);
-  }
-
-  const commonDir = git(root, ['rev-parse', '--git-common-dir']);
-  const absolute = realpathSync(resolve(root, commonDir));
-  return `local:${createHash('sha256').update(absolute).digest('hex')}`;
-}
-
 function resolveBase(root, requested) {
   const candidates = requested
     ? [requested]
@@ -105,18 +65,6 @@ function resolveBase(root, requested) {
   return 'HEAD';
 }
 
-function isTestPath(path) {
-  const normalized = path.replaceAll('\\', '/');
-  const file = basename(normalized);
-  return (
-    /(^|\/)(__tests__|tests?|specs?|testdata|fixtures?)(\/|$)/iu.test(
-      normalized
-    ) ||
-    /\.(test|spec)\.[^.]+$/iu.test(file) ||
-    /^(test_.+|.+_test)\.[^.]+$/iu.test(file)
-  );
-}
-
 function storedPrompts(repository, branch) {
   const stateRoot = process.env.RUDDER_HOME || join(homedir(), '.rudder');
   const databasePath = join(stateRoot, 'rudder.db');
@@ -152,6 +100,15 @@ function storedPrompts(repository, branch) {
 
 function main() {
   const args = process.argv.slice(2);
+  const phase = argumentValue(args, '--phase') ?? 'start';
+  if (phase !== 'start' && phase !== 'refresh') {
+    throw new TypeError('--phase must be start or refresh');
+  }
+  const requestedRunId = argumentValue(args, '--run-id');
+  if (phase === 'refresh' && !requestedRunId) {
+    throw new TypeError('--run-id is required when --phase is refresh');
+  }
+  const rudderRunId = requestedRunId ?? randomUUID();
   const cwd = realpathSync(argumentValue(args, '--cwd') ?? process.cwd());
   const root = git(cwd, ['rev-parse', '--show-toplevel']);
   const branch = git(root, ['symbolic-ref', '--quiet', '--short', 'HEAD']);
@@ -175,13 +132,47 @@ function main() {
   const changedPaths = [...new Set([...tracked, ...untracked])].sort();
   const testPaths = changedPaths.filter(isTestPath);
   const otherPaths = changedPaths.filter((path) => !isTestPath(path));
+  const testLines = testDiffLineCounts(root, mergeBase);
   const repository = repositoryKey(root, branch);
   const promptData = storedPrompts(repository, branch);
+  const promptSessions = new Set(
+    promptData.prompts.map(
+      (prompt) => `${prompt.source}\0${prompt.sessionId}`
+    )
+  );
+  const promptSourceCounts = {};
+  for (const prompt of promptData.prompts) {
+    const source = ['claude-code', 'codex', 'cursor'].includes(prompt.source)
+      ? prompt.source
+      : 'other';
+    promptSourceCounts[source] = (promptSourceCounts[source] ?? 0) + 1;
+  }
+  captureRudderTelemetry(
+    phase === 'start' ? 'run-started' : 'context-refreshed',
+    {
+      repository,
+      branch,
+      runId: rudderRunId,
+      capturedPromptCount: promptData.prompts.length,
+      capturedSessionCount: promptSessions.size,
+      reconciledPromptCount: promptData.prompts.filter(
+        (prompt) => prompt.reconciledAt !== null
+      ).length,
+      promptSourceCounts,
+      changedPathCount: changedPaths.length,
+      changedTestPathCount: testPaths.length,
+      changedProductionPathCount: otherPaths.length,
+      untrackedPathCount: untracked.length,
+      testLineAdditionCount: testLines.additions,
+      testLineDeletionCount: testLines.deletions,
+    }
+  );
 
   process.stdout.write(
     `${JSON.stringify(
       {
         schemaVersion: 1,
+        rudderRunId,
         root,
         repository,
         branch,
@@ -191,6 +182,8 @@ function main() {
         testPaths,
         otherPaths,
         untrackedPaths: untracked.sort(),
+        testLineAdditionCount: testLines.additions,
+        testLineDeletionCount: testLines.deletions,
         promptDatabasePath: promptData.databasePath,
         prompts: promptData.prompts,
       },