diff --git a/bin/rudder-prompt-hook.ts b/bin/rudder-prompt-hook.ts index 0fdf938..725ea83 100755 --- a/bin/rudder-prompt-hook.ts +++ b/bin/rudder-prompt-hook.ts @@ -3,6 +3,11 @@ import { join } from 'node:path'; import { closeDb } from '../src/db/client.ts'; import { parseAgentPromptSource, recordPromptHookEvent } from '../src/prompt-hook.ts'; +import { + captureRudderUsageEvent, + rudderUsageEvents, + type RudderUsageEvent, +} from '../src/rudder-telemetry.ts'; import { captureException, shutdown } from '../src/telemetry.ts'; type AgentSource = 'claude-code' | 'codex' | 'cursor'; @@ -27,6 +32,16 @@ function hookContext(args: string[]): HookContext { return { source: parseAgentPromptSource(sourceArgument(args)) }; } +function rudderUsageEventArgument(args: string[]): RudderUsageEvent | null { + if (args[0] !== '--rudder-event') return null; + if (args.length !== 2 || !rudderUsageEvents.includes(args[1] as RudderUsageEvent)) { + throw new TypeError( + `usage: rudder-prompt-hook --rudder-event <${rudderUsageEvents.join('|')}>` + ); + } + return args[1] as RudderUsageEvent; +} + async function readStdin(): Promise { const chunks: Buffer[] = []; for await (const chunk of process.stdin) { @@ -36,13 +51,19 @@ async function readStdin(): Promise { } try { - const context = hookContext(process.argv.slice(2)); - if (context.root) { - process.env.RUDDER_MIGRATIONS_PATH ||= join(context.root, 'dist', 'drizzle'); - } + const args = process.argv.slice(2); + const usageEvent = rudderUsageEventArgument(args); const input = await readStdin(); const payload: unknown = JSON.parse(input); - recordPromptHookEvent(context.source, payload); + if (usageEvent) { + captureRudderUsageEvent(usageEvent, payload); + } else { + const context = hookContext(args); + if (context.root) { + process.env.RUDDER_MIGRATIONS_PATH ||= join(context.root, 'dist', 'drizzle'); + } + recordPromptHookEvent(context.source, payload); + } } catch (error) { // Prompt capture is optional metadata. A hook failure must not interrupt the host agent. try { diff --git a/package.json b/package.json index 9ce2a66..b795b40 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,7 @@ "build": "rm -rf dist && esbuild bin/rudder-prompt-hook.ts --bundle --platform=node --format=esm --target=node24 --outfile=dist/rudder-prompt-hook.mjs && cp -R drizzle dist/drizzle", "pretest": "npm run build", "test": "node --test", - "test:coverage": "npm run build && c8 node --test && diff-cover coverage/lcov.info --fail-under=80 --show-uncovered --include-untracked", + "test:coverage": "npm run build && c8 node --test && diff-cover coverage/lcov.info --fail-under=90 --show-uncovered --include-untracked", "prepack": "npm run build", "prepublishOnly": "npm run typecheck && npm test" }, diff --git a/skills/rudder/SKILL.md b/skills/rudder/SKILL.md index ef0e5d4..a973e70 100644 --- a/skills/rudder/SKILL.md +++ b/skills/rudder/SKILL.md @@ -61,11 +61,21 @@ Coverage is loop control, never a source of test intent. implementation, improve coverage, or exercise a defensive case. - After the first test pass, if coverage is below the target, stop editing tests. Select one uncovered behavior. + Increment the run's question counter and best-effort record the question before showing it: + + ```text + node /scripts/telemetry.mjs question-asked \ + --cwd \ + --run-id \ + --question-number + ``` + + Do not pass the question or any answer text to the helper. Ask one concrete question about the expected behavior. - Do not write the next test until the user answers. Repository code may help frame the question, but it cannot supply the answer. -- After each answer, rerun `scripts/context.mjs`, require a captured prompt - record for the answer, and queue only the expectation that answer authorizes. +- After each answer, rerun `scripts/context.mjs` with `--phase refresh`, `--run-id `, and `--base `. + Require a captured prompt record for the answer, and queue only the expectation that answer authorizes. - Complete the red-green cycle for every authorized expectation in its owning agent before integrating the batch. Use the queue answered rewrites guidance to set up owning agents. Do not measure coverage while a rewrite is pending. @@ -123,15 +133,18 @@ For every new or changed expectation: Determine the requested coverage target. Prefer the repository's configured coverage threshold. Ask for a target only when neither the request nor repository provides one. -2. Run `scripts/context.mjs` relative to this file with the repository working - directory: +2. Run `scripts/context.mjs` relative to this file with the repository working directory: ```text node /scripts/context.mjs \ --cwd \ + --phase start \ [--base ] ``` + Retain the returned `rudderRunId` and `baseRef` for every later helper call in this run. + Use that returned `baseRef` as `` in every later context, backup, and completion helper call. + Initialize the run's question counter to zero. 3. Inspect the returned merge base, changed paths, and captured prompts. Inspect repository instructions, the production diff, and existing tests. Inspect the native test and coverage configuration. @@ -156,7 +169,8 @@ For every new or changed expectation: ```text node /scripts/backup-tests.mjs \ --cwd \ - --base \ + --base \ + --run-id \ --path \ [--path ...] ``` @@ -192,6 +206,22 @@ For every new or changed expectation: Continue asking independent questions until the batch must join. After joining, run the combined suites and coverage before selecting another uncovered behavior. Continue until the target passes or the user tells you to stop the flow. +12. Before the final report, record the verified Rudder outcome: + + ```text + node /scripts/telemetry.mjs complete \ + --cwd \ + --base \ + --run-id \ + --status \ + --tests-passed \ + --coverage-target-met \ + --questions-asked + ``` + + Use `completed` when the workflow reaches its normal report, `stopped` when it ends by user choice or missing intent, and `blocked` only for an external blocker. + Set test and coverage values only from command output already observed during this run. + Telemetry is best-effort; do not change the workflow result if this helper is unavailable. Report the requirements derived from intent and all files changed. Report commands run, coverage, unanswered ambiguities, and the backup location. diff --git a/skills/rudder/scripts/backup-tests.mjs b/skills/rudder/scripts/backup-tests.mjs index 3c3e725..5e58785 100644 --- a/skills/rudder/scripts/backup-tests.mjs +++ b/skills/rudder/scripts/backup-tests.mjs @@ -4,6 +4,10 @@ import { execFileSync, spawnSync } from 'node:child_process'; import { cpSync, mkdirSync, mkdtempSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join, relative, resolve } from 'node:path'; +import { + captureRudderTelemetry, + repositoryKey, +} from './telemetry.mjs'; function argumentValue(args, name, required = false) { const index = args.indexOf(name); @@ -62,6 +66,7 @@ function safeRelativePath(root, path) { function main() { const args = process.argv.slice(2); const cwd = argumentValue(args, '--cwd', true); + const runId = argumentValue(args, '--run-id', true); const root = git(cwd, ['rev-parse', '--show-toplevel']); const baseRef = argumentValue(args, '--base') ?? 'HEAD'; if (!git(root, ['rev-parse', '--verify', '--quiet', baseRef], true)) { @@ -120,6 +125,24 @@ function main() { encoding: 'utf8', mode: 0o600, }); + try { + const branch = git( + root, + ['symbolic-ref', '--quiet', '--short', 'HEAD'], + true + ); + if (branch) { + captureRudderTelemetry('test-backup-created', { + repository: repositoryKey(root, branch), + branch, + runId, + approvedTestPathCount: paths.length, + copiedUntrackedTestPathCount: copiedUntrackedPaths.length, + }); + } + } catch { + // Telemetry must not affect backup creation or its recovery metadata. + } process.stdout.write( `${JSON.stringify( { backupDirectory, metadataPath, ...metadata }, diff --git a/skills/rudder/scripts/context.mjs b/skills/rudder/scripts/context.mjs index 75f7dc9..c7e7918 100644 --- a/skills/rudder/scripts/context.mjs +++ b/skills/rudder/scripts/context.mjs @@ -1,11 +1,17 @@ #!/usr/bin/env node import { execFileSync, spawnSync } from 'node:child_process'; -import { createHash } from 'node:crypto'; +import { randomUUID } from 'node:crypto'; import { existsSync, realpathSync } from 'node:fs'; import { homedir } from 'node:os'; -import { basename, join, resolve } from 'node:path'; +import { join } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; +import { + captureRudderTelemetry, + isTestPath, + repositoryKey, + testDiffLineCounts, +} from './telemetry.mjs'; function argumentValue(args, name) { const index = args.indexOf(name); @@ -36,52 +42,6 @@ function gitNullList(cwd, args) { return output.split('\0').filter(Boolean); } -function strippedRepositoryPath(path) { - return path.replace(/^\/+|\/+$/gu, '').replace(/\.git$/u, ''); -} - -function normalizeRepository(repository) { - const value = repository.trim(); - const scp = /^(?:[^@/]+@)?([^:/]+):(.+)$/u.exec(value); - if (scp && !value.includes('://')) { - return `${scp[1].toLowerCase()}/${strippedRepositoryPath(scp[2])}`; - } - - try { - const url = new URL(value); - if (url.protocol !== 'file:') { - return `${url.host.toLowerCase()}/${strippedRepositoryPath( - decodeURIComponent(url.pathname) - )}`; - } - } catch { - // Treat non-URL values as local paths. - } - return strippedRepositoryPath(value); -} - -function repositoryKey(root, branch) { - const branchRemote = git( - root, - ['config', '--get', `branch.${branch}.remote`], - true - ); - const remoteNames = [ - branchRemote && branchRemote !== '.' ? branchRemote : null, - 'origin', - ...((git(root, ['remote'], true) ?? '').split('\n').filter(Boolean)), - ].filter(Boolean); - - for (const remoteName of new Set(remoteNames)) { - const remote = git(root, ['remote', 'get-url', remoteName], true); - if (remote) return normalizeRepository(remote); - } - - const commonDir = git(root, ['rev-parse', '--git-common-dir']); - const absolute = realpathSync(resolve(root, commonDir)); - return `local:${createHash('sha256').update(absolute).digest('hex')}`; -} - function resolveBase(root, requested) { const candidates = requested ? [requested] @@ -105,18 +65,6 @@ function resolveBase(root, requested) { return 'HEAD'; } -function isTestPath(path) { - const normalized = path.replaceAll('\\', '/'); - const file = basename(normalized); - return ( - /(^|\/)(__tests__|tests?|specs?|testdata|fixtures?)(\/|$)/iu.test( - normalized - ) || - /\.(test|spec)\.[^.]+$/iu.test(file) || - /^(test_.+|.+_test)\.[^.]+$/iu.test(file) - ); -} - function storedPrompts(repository, branch) { const stateRoot = process.env.RUDDER_HOME || join(homedir(), '.rudder'); const databasePath = join(stateRoot, 'rudder.db'); @@ -152,6 +100,15 @@ function storedPrompts(repository, branch) { function main() { const args = process.argv.slice(2); + const phase = argumentValue(args, '--phase') ?? 'start'; + if (phase !== 'start' && phase !== 'refresh') { + throw new TypeError('--phase must be start or refresh'); + } + const requestedRunId = argumentValue(args, '--run-id'); + if (phase === 'refresh' && !requestedRunId) { + throw new TypeError('--run-id is required when --phase is refresh'); + } + const rudderRunId = requestedRunId ?? randomUUID(); const cwd = realpathSync(argumentValue(args, '--cwd') ?? process.cwd()); const root = git(cwd, ['rev-parse', '--show-toplevel']); const branch = git(root, ['symbolic-ref', '--quiet', '--short', 'HEAD']); @@ -175,13 +132,47 @@ function main() { const changedPaths = [...new Set([...tracked, ...untracked])].sort(); const testPaths = changedPaths.filter(isTestPath); const otherPaths = changedPaths.filter((path) => !isTestPath(path)); + const testLines = testDiffLineCounts(root, mergeBase); const repository = repositoryKey(root, branch); const promptData = storedPrompts(repository, branch); + const promptSessions = new Set( + promptData.prompts.map( + (prompt) => `${prompt.source}\0${prompt.sessionId}` + ) + ); + const promptSourceCounts = {}; + for (const prompt of promptData.prompts) { + const source = ['claude-code', 'codex', 'cursor'].includes(prompt.source) + ? prompt.source + : 'other'; + promptSourceCounts[source] = (promptSourceCounts[source] ?? 0) + 1; + } + captureRudderTelemetry( + phase === 'start' ? 'run-started' : 'context-refreshed', + { + repository, + branch, + runId: rudderRunId, + capturedPromptCount: promptData.prompts.length, + capturedSessionCount: promptSessions.size, + reconciledPromptCount: promptData.prompts.filter( + (prompt) => prompt.reconciledAt !== null + ).length, + promptSourceCounts, + changedPathCount: changedPaths.length, + changedTestPathCount: testPaths.length, + changedProductionPathCount: otherPaths.length, + untrackedPathCount: untracked.length, + testLineAdditionCount: testLines.additions, + testLineDeletionCount: testLines.deletions, + } + ); process.stdout.write( `${JSON.stringify( { schemaVersion: 1, + rudderRunId, root, repository, branch, @@ -191,6 +182,8 @@ function main() { testPaths, otherPaths, untrackedPaths: untracked.sort(), + testLineAdditionCount: testLines.additions, + testLineDeletionCount: testLines.deletions, promptDatabasePath: promptData.databasePath, prompts: promptData.prompts, }, diff --git a/skills/rudder/scripts/telemetry.mjs b/skills/rudder/scripts/telemetry.mjs new file mode 100644 index 0000000..a5619de --- /dev/null +++ b/skills/rudder/scripts/telemetry.mjs @@ -0,0 +1,361 @@ +#!/usr/bin/env node + +import { execFileSync, spawn, spawnSync } from 'node:child_process'; +import { createHash } from 'node:crypto'; +import { existsSync, readFileSync, realpathSync } from 'node:fs'; +import { basename, join, relative, resolve } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +function git(cwd, args, optional = false) { + const result = spawnSync('git', ['-C', cwd, ...args], { + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + if (result.status === 0) return result.stdout.trim(); + if (optional) return null; + throw new Error( + result.stderr.trim() || `git ${args.join(' ')} exited with ${result.status}` + ); +} + +function gitNullList(cwd, args) { + return execFileSync('git', ['-C', cwd, ...args], { + encoding: 'utf8', + }) + .split('\0') + .filter(Boolean); +} + +function strippedRepositoryPath(path) { + return path.replace(/^\/+|\/+$/gu, '').replace(/\.git$/u, ''); +} + +function normalizeRepository(repository) { + const value = repository.trim(); + const scp = /^(?:[^@/]+@)?([^:/]+):(.+)$/u.exec(value); + if (scp && !value.includes('://')) { + return `${scp[1].toLowerCase()}/${strippedRepositoryPath(scp[2])}`; + } + + try { + const url = new URL(value); + if (url.protocol !== 'file:') { + return `${url.host.toLowerCase()}/${strippedRepositoryPath( + decodeURIComponent(url.pathname) + )}`; + } + } catch { + // Treat non-URL values as local paths. + } + return strippedRepositoryPath(value); +} + +export function repositoryKey(root, branch) { + const branchRemote = git( + root, + ['config', '--get', `branch.${branch}.remote`], + true + ); + const remoteNames = [ + branchRemote && branchRemote !== '.' ? branchRemote : null, + 'origin', + ...((git(root, ['remote'], true) ?? '').split('\n').filter(Boolean)), + ].filter(Boolean); + + for (const remoteName of new Set(remoteNames)) { + const remote = git(root, ['remote', 'get-url', remoteName], true); + if (remote) return normalizeRepository(remote); + } + + const commonDir = git(root, ['rev-parse', '--git-common-dir']); + const absolute = realpathSync(resolve(root, commonDir)); + return `local:${createHash('sha256').update(absolute).digest('hex')}`; +} + +export function isTestPath(path) { + const normalized = path.replaceAll('\\', '/'); + const file = basename(normalized); + return ( + /(^|\/)(__tests__|tests?|specs?|testdata|fixtures?)(\/|$)/iu.test( + normalized + ) || + /\.(test|spec)\.[^.]+$/iu.test(file) || + /^(test_.+|.+_test)\.[^.]+$/iu.test(file) + ); +} + +function textLineCount(path) { + try { + const content = readFileSync(path); + if (content.length === 0 || content.includes(0)) return 0; + let lines = 0; + for (const byte of content) { + if (byte === 10) lines += 1; + } + return lines + (content.at(-1) === 10 ? 0 : 1); + } catch { + return 0; + } +} + +export function testDiffLineCounts(root, mergeBase) { + const output = execFileSync( + 'git', + ['-C', root, 'diff', '--numstat', '-z', mergeBase, '--'], + { encoding: 'utf8' } + ); + const fields = output.split('\0'); + let additions = 0; + let deletions = 0; + for (let index = 0; index < fields.length; ) { + const field = fields[index++]; + if (!field) continue; + const match = /^(\d+|-)\t(\d+|-)\t(.*)$/su.exec(field); + if (!match) continue; + + let paths; + if (match[3]) { + paths = [match[3]]; + } else { + paths = [fields[index++] ?? '', fields[index++] ?? '']; + } + if (!paths.some(isTestPath)) continue; + if (match[1] !== '-') additions += Number(match[1]); + if (match[2] !== '-') deletions += Number(match[2]); + } + + const untracked = gitNullList(root, [ + 'ls-files', + '--others', + '--exclude-standard', + '-z', + ]); + for (const path of untracked.filter(isTestPath)) { + additions += textLineCount(join(root, path)); + } + return { additions, deletions }; +} + +function host() { + if (process.env.PLUGIN_ROOT) return 'codex'; + if (process.env.CLAUDE_PLUGIN_ROOT) return 'claude-code'; + return 'unknown'; +} + +export function captureRudderTelemetry(event, payload) { + const executable = fileURLToPath( + new URL('../../../dist/rudder-prompt-hook.mjs', import.meta.url) + ); + if (!existsSync(executable)) return false; + + try { + const input = JSON.stringify({ ...payload, host: host() }); + const telemetry = spawn( + process.execPath, + [executable, '--rudder-event', event], + { + env: process.env, + detached: true, + stdio: ['pipe', 'ignore', 'ignore'], + } + ); + telemetry.once('error', () => { + // Telemetry dispatch failures must not interrupt the Rudder workflow. + }); + telemetry.stdin.once('error', () => { + // The child may exit before reading its best-effort telemetry payload. + }); + telemetry.stdin.end(input); + telemetry.stdin.unref(); + telemetry.unref(); + return true; + } catch { + return false; + } +} + +function argumentValue(args, name, required = false) { + const index = args.indexOf(name); + if (index === -1) { + if (required) throw new TypeError(`${name} is required`); + return null; + } + const value = args[index + 1]; + if (!value || value.startsWith('--')) { + throw new TypeError(`${name} requires a value`); + } + return value; +} + +function argumentChoice(args, name, choices) { + const value = argumentValue(args, name, true); + if (!choices.includes(value)) { + throw new TypeError(`${name} must be one of: ${choices.join(', ')}`); + } + return value; +} + +function argumentCount(args, name, positive = false) { + const value = argumentValue(args, name, true); + if (!/^(0|[1-9]\d*)$/u.test(value)) { + throw new TypeError(`${name} must be a non-negative integer`); + } + const result = Number(value); + if (!Number.isSafeInteger(result)) { + throw new TypeError(`${name} must be a non-negative integer`); + } + if (positive && result === 0) { + throw new TypeError(`${name} must be a positive integer`); + } + return result; +} + +function safeRelativePath(root, path) { + const absolute = resolve(root, path); + const normalized = relative(root, absolute); + if ( + !normalized || + normalized === '..' || + normalized.startsWith( + `..${process.platform === 'win32' ? '\\' : '/'}` + ) + ) { + return null; + } + return normalized; +} + +function promptBackedTestCount(root, testPaths) { + const tag = + /^\s*(?:\/\/|#|--|;|\/\*+|\*)\s*(?:claude-code|codex|cursor)\/[^/\s]+\/[^/\s]+/gmu; + let count = 0; + for (const path of testPaths) { + const safePath = safeRelativePath(root, path); + if (!safePath) continue; + try { + count += readFileSync(join(root, safePath), 'utf8').match(tag)?.length ?? 0; + } catch { + // Deleted, binary, and unreadable test paths contribute no tags. + } + } + return count; +} + +function activeRun(args) { + const cwd = realpathSync(argumentValue(args, '--cwd', true)); + const root = git(cwd, ['rev-parse', '--show-toplevel']); + const branch = git(root, ['symbolic-ref', '--quiet', '--short', 'HEAD']); + if (!branch) throw new Error('Rudder requires an attached Git branch'); + return { + root, + branch, + repository: repositoryKey(root, branch), + runId: argumentValue(args, '--run-id', true), + }; +} + +function recordQuestion(args) { + const run = activeRun(args); + const questionNumber = argumentCount(args, '--question-number', true); + const dispatched = captureRudderTelemetry('question-asked', { + repository: run.repository, + branch: run.branch, + runId: run.runId, + questionNumber, + }); + return { schemaVersion: 1, telemetryDispatched: dispatched, questionNumber }; +} + +function finishRun(args) { + const run = activeRun(args); + const { root } = run; + const baseRef = argumentValue(args, '--base', true); + const mergeBase = git(root, ['merge-base', 'HEAD', baseRef]); + const tracked = gitNullList(root, [ + 'diff', + '--name-only', + '-z', + mergeBase, + '--', + ]); + const untracked = gitNullList(root, [ + 'ls-files', + '--others', + '--exclude-standard', + '-z', + ]); + const changedPaths = [...new Set([...tracked, ...untracked])].sort(); + const testPaths = changedPaths.filter(isTestPath); + const productionPaths = changedPaths.filter((path) => !isTestPath(path)); + const testLines = testDiffLineCounts(root, mergeBase); + const result = { + status: argumentChoice(args, '--status', [ + 'completed', + 'stopped', + 'blocked', + ]), + testsPassed: argumentChoice(args, '--tests-passed', [ + 'yes', + 'no', + 'unknown', + ]), + coverageTargetMet: argumentChoice(args, '--coverage-target-met', [ + 'yes', + 'no', + 'unknown', + ]), + changedPathCount: changedPaths.length, + changedTestPathCount: testPaths.length, + changedProductionPathCount: productionPaths.length, + promptBackedTestCount: promptBackedTestCount(root, testPaths), + finalTestLineAdditionCount: testLines.additions, + finalTestLineDeletionCount: testLines.deletions, + questionsAskedCount: argumentCount(args, '--questions-asked'), + }; + const dispatched = captureRudderTelemetry('run-finished', { + repository: run.repository, + branch: run.branch, + runId: run.runId, + status: result.status, + testsPassed: result.testsPassed, + coverageTargetMet: result.coverageTargetMet, + changedPathCount: result.changedPathCount, + changedTestPathCount: result.changedTestPathCount, + changedProductionPathCount: result.changedProductionPathCount, + promptBackedTestCount: result.promptBackedTestCount, + testLineAdditionCount: result.finalTestLineAdditionCount, + testLineDeletionCount: result.finalTestLineDeletionCount, + questionsAskedCount: result.questionsAskedCount, + }); + return { schemaVersion: 1, telemetryDispatched: dispatched, ...result }; +} + +function main() { + const [command, ...args] = process.argv.slice(2); + switch (command) { + case 'question-asked': + process.stdout.write(`${JSON.stringify(recordQuestion(args), null, 2)}\n`); + return; + case 'complete': + process.stdout.write(`${JSON.stringify(finishRun(args), null, 2)}\n`); + return; + default: + throw new TypeError( + 'usage: telemetry.mjs --cwd --run-id [command options]' + ); + } +} + +const entrypoint = process.argv[1] + ? pathToFileURL(process.argv[1]).href + : null; +if (entrypoint === import.meta.url) { + try { + main(); + } catch (error) { + process.stderr.write( + `${error instanceof Error ? error.message : String(error)}\n` + ); + process.exitCode = 1; + } +} diff --git a/src/prompt-hook.ts b/src/prompt-hook.ts index 723e6a3..5c37521 100644 --- a/src/prompt-hook.ts +++ b/src/prompt-hook.ts @@ -1,9 +1,13 @@ import { reconcilePromptBranch, recordPromptBranch, + promptsForSession, type PromptBranchRow, } from './prompt-tagger.ts'; -import { capture } from './telemetry.ts'; +import { + capture, + type TelemetryPropertiesFactory, +} from './telemetry.ts'; import { readPreviousAgentOutput } from './transcript.ts'; export const agentPromptSources = ['claude-code', 'codex', 'cursor'] as const; @@ -30,7 +34,7 @@ export class PromptHookPayloadError extends TypeError { function captureHookEvent( event: string, - properties: Record + properties: TelemetryPropertiesFactory ): void { try { capture(event, properties); @@ -39,6 +43,24 @@ function captureHookEvent( } } +function targetBeforeReconciliation( + hook: NormalizedPromptHookPayload +): PromptBranchRow | null { + try { + const rows = promptsForSession(hook.source, hook.sessionId); + if (hook.promptId) { + return rows.find((row) => row.promptId === hook.promptId) ?? null; + } + return ( + [...rows] + .reverse() + .find((row) => row.reconciledAt === null) ?? null + ); + } catch { + return null; + } +} + function recordPayload(payload: unknown): Record { if (typeof payload !== 'object' || payload === null || Array.isArray(payload)) { throw new PromptHookPayloadError('expected a JSON object'); @@ -168,13 +190,19 @@ export function recordPromptHookEvent( : null, cwd: hook.cwd, }); - captureHookEvent('prompt recorded', { - source: hook.source, - has_previous_agent_output: row.previousAgentOutput !== null, + captureHookEvent('rudder prompt captured', () => { + const sessionRows = promptsForSession(hook.source, hook.sessionId); + return { + source: hook.source, + is_session_start: sessionRows.length === 1, + captured_prompt_count_for_session: sessionRows.length, + has_previous_agent_output: row.previousAgentOutput !== null, + }; }); return row; } + const previousRow = targetBeforeReconciliation(hook); const branchInput = { source: hook.source, sessionId: hook.sessionId, @@ -183,7 +211,13 @@ export function recordPromptHookEvent( }; const row = reconcilePromptBranch(branchInput); if (row) { - captureHookEvent('prompt reconciled', { source: hook.source }); + captureHookEvent('rudder prompt reconciled', () => { + return { + source: hook.source, + branch_changed: + previousRow === null ? null : previousRow.branch !== row.branch, + }; + }); } return row; } diff --git a/src/rudder-telemetry.ts b/src/rudder-telemetry.ts new file mode 100644 index 0000000..e051837 --- /dev/null +++ b/src/rudder-telemetry.ts @@ -0,0 +1,249 @@ +import { + capture, + type TelemetryCaptureContext, + type TelemetryProperties, +} from './telemetry.ts'; + +export const rudderUsageEvents = [ + 'run-started', + 'context-refreshed', + 'test-backup-created', + 'question-asked', + 'run-finished', +] as const; + +export type RudderUsageEvent = (typeof rudderUsageEvents)[number]; + +type JsonObject = Record; + +function object(value: unknown): JsonObject { + if (typeof value !== 'object' || value === null || Array.isArray(value)) { + throw new TypeError('Rudder telemetry payload must be an object'); + } + return value as JsonObject; +} + +function string(value: unknown, field: string): string { + if (typeof value !== 'string' || !value.trim()) { + throw new TypeError(`${field} must be a non-empty string`); + } + return value.trim(); +} + +function count(value: unknown, field: string): number { + if (!Number.isSafeInteger(value) || (value as number) < 0) { + throw new TypeError(`${field} must be a non-negative integer`); + } + return value as number; +} + +function positiveCount(value: unknown, field: string): number { + const result = count(value, field); + if (result === 0) { + throw new TypeError(`${field} must be a positive integer`); + } + return result; +} + +function choice( + value: unknown, + field: string, + choices: readonly T[] +): T { + if (typeof value === 'string' && choices.includes(value as T)) { + return value as T; + } + throw new TypeError(`${field} must be one of: ${choices.join(', ')}`); +} + +function promptSourceCounts(value: unknown): Record { + const sources = object(value); + const result: Record = {}; + for (const source of ['claude-code', 'codex', 'cursor', 'other']) { + if (sources[source] !== undefined) { + result[source] = count(sources[source], `promptSourceCounts.${source}`); + } + } + return result; +} + +function host(value: unknown): string { + return choice(value, 'host', ['claude-code', 'codex', 'unknown'] as const); +} + +function repositoryProperties( + context: TelemetryCaptureContext, + payload: JsonObject +): TelemetryProperties { + const repository = string(payload.repository, 'repository'); + const branch = string(payload.branch, 'branch'); + return { + repository_pseudonym: context.pseudonymize('repository', repository), + branch_pseudonym: context.pseudonymize( + 'branch', + `${repository}\0${branch}` + ), + repository_is_local: repository.startsWith('local:'), + }; +} + +function runProperties( + context: TelemetryCaptureContext, + payload: JsonObject +): TelemetryProperties { + const repository = string(payload.repository, 'repository'); + const branch = string(payload.branch, 'branch'); + const runId = string(payload.runId, 'runId'); + return { + ...repositoryProperties(context, payload), + run_pseudonym: context.pseudonymize( + 'run', + `${repository}\0${branch}\0${runId}` + ), + }; +} + +function contextProperties( + context: TelemetryCaptureContext, + payload: JsonObject +): TelemetryProperties { + const sources = promptSourceCounts(payload.promptSourceCounts); + const capturedPromptCount = count( + payload.capturedPromptCount, + 'capturedPromptCount' + ); + const reconciledPromptCount = count( + payload.reconciledPromptCount, + 'reconciledPromptCount' + ); + return { + ...runProperties(context, payload), + host: host(payload.host), + captured_prompt_count: capturedPromptCount, + captured_session_count: count( + payload.capturedSessionCount, + 'capturedSessionCount' + ), + reconciled_prompt_count: reconciledPromptCount, + unreconciled_prompt_count: Math.max( + 0, + capturedPromptCount - reconciledPromptCount + ), + captured_prompt_sources: Object.keys(sources), + captured_prompt_source_counts: sources, + has_captured_intent: capturedPromptCount > 0, + changed_path_count: count(payload.changedPathCount, 'changedPathCount'), + changed_test_path_count: count( + payload.changedTestPathCount, + 'changedTestPathCount' + ), + changed_production_candidate_path_count: count( + payload.changedProductionPathCount, + 'changedProductionPathCount' + ), + untracked_path_count: count( + payload.untrackedPathCount, + 'untrackedPathCount' + ), + test_lines_added_from_base: count( + payload.testLineAdditionCount, + 'testLineAdditionCount' + ), + test_lines_deleted_from_base: count( + payload.testLineDeletionCount, + 'testLineDeletionCount' + ), + }; +} + +export function rudderUsageEventProperties( + event: RudderUsageEvent, + input: unknown, + context: TelemetryCaptureContext +): TelemetryProperties { + const payload = object(input); + switch (event) { + case 'run-started': + case 'context-refreshed': + return contextProperties(context, payload); + case 'test-backup-created': + return { + ...runProperties(context, payload), + host: host(payload.host), + approved_test_path_count: count( + payload.approvedTestPathCount, + 'approvedTestPathCount' + ), + copied_untracked_test_path_count: count( + payload.copiedUntrackedTestPathCount, + 'copiedUntrackedTestPathCount' + ), + }; + case 'question-asked': + return { + ...runProperties(context, payload), + host: host(payload.host), + question_number: positiveCount( + payload.questionNumber, + 'questionNumber' + ), + }; + case 'run-finished': + return { + ...runProperties(context, payload), + host: host(payload.host), + status: choice(payload.status, 'status', [ + 'completed', + 'stopped', + 'blocked', + ] as const), + tests_passed: choice(payload.testsPassed, 'testsPassed', [ + 'yes', + 'no', + 'unknown', + ] as const), + coverage_target_met: choice( + payload.coverageTargetMet, + 'coverageTargetMet', + ['yes', 'no', 'unknown'] as const + ), + final_changed_path_count: count( + payload.changedPathCount, + 'changedPathCount' + ), + final_changed_test_path_count: count( + payload.changedTestPathCount, + 'changedTestPathCount' + ), + final_changed_production_candidate_path_count: count( + payload.changedProductionPathCount, + 'changedProductionPathCount' + ), + prompt_backed_test_count: count( + payload.promptBackedTestCount, + 'promptBackedTestCount' + ), + final_test_lines_added_from_base: count( + payload.testLineAdditionCount, + 'testLineAdditionCount' + ), + final_test_lines_deleted_from_base: count( + payload.testLineDeletionCount, + 'testLineDeletionCount' + ), + questions_asked_count: count( + payload.questionsAskedCount, + 'questionsAskedCount' + ), + }; + } +} + +export function captureRudderUsageEvent( + event: RudderUsageEvent, + input: unknown +): void { + capture(`rudder ${event.replaceAll('-', ' ')}`, (context) => + rudderUsageEventProperties(event, input, context) + ); +} diff --git a/src/telemetry.ts b/src/telemetry.ts index cb1f547..702e302 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -1,5 +1,11 @@ -import { randomUUID } from 'node:crypto'; -import { readFileSync, writeFileSync, existsSync, mkdirSync } from 'node:fs'; +import { createHmac, randomBytes, randomUUID } from 'node:crypto'; +import { + chmodSync, + existsSync, + mkdirSync, + readFileSync, + writeFileSync, +} from 'node:fs'; import { join } from 'node:path'; import { PostHog } from 'posthog-node'; import { rudderHome } from './db/index.ts'; @@ -13,34 +19,108 @@ const POSTHOG_PROJECT_TOKEN = process.env.POSTHOG_PROJECT_TOKEN || BUILT_IN_POSTHOG_PROJECT_TOKEN; const POSTHOG_HOST = process.env.POSTHOG_HOST || BUILT_IN_POSTHOG_HOST || DEFAULT_POSTHOG_HOST; +const TELEMETRY_SCHEMA_VERSION = 1; + +export interface TelemetryCaptureContext { + pseudonymize(namespace: string, value: string): string; +} + +export type TelemetryProperties = Record; +export type TelemetryPropertiesFactory = ( + context: TelemetryCaptureContext +) => TelemetryProperties; + +interface TelemetryIdentity { + id: string; + pseudonymizationKey: string; +} + +function restrictIdentityPath(path: string, mode: number): void { + try { + chmodSync(path, mode); + } catch { + // Some Windows and network filesystems do not expose POSIX mode bits. + } +} export function telemetryDisabled(env: NodeJS.ProcessEnv = process.env): boolean { return env.DO_NOT_TRACK === '1'; } -/** Read or generate a stable anonymous installation ID. */ -function loadDistinctId(): string { +function packageVersion(): string { + try { + const manifest = JSON.parse( + readFileSync(new URL('../package.json', import.meta.url), 'utf8') + ) as Record; + return typeof manifest.version === 'string' && manifest.version + ? manifest.version + : 'unknown'; + } catch { + return 'unknown'; + } +} + +export function runtimeTelemetryProperties(): TelemetryProperties { + return { + telemetry_schema_version: TELEMETRY_SCHEMA_VERSION, + rudder_version: packageVersion(), + }; +} + +function persistIdentity(idPath: string, identity: TelemetryIdentity): void { + mkdirSync(rudderHome(), { recursive: true, mode: 0o700 }); + restrictIdentityPath(rudderHome(), 0o700); + writeFileSync( + idPath, + JSON.stringify({ + id: identity.id, + pseudonymization_key: identity.pseudonymizationKey, + }), + { mode: 0o600 } + ); + restrictIdentityPath(idPath, 0o600); +} + +/** Read or generate a stable anonymous installation ID and local-only key. */ +function loadIdentity(): TelemetryIdentity { const idPath = join(rudderHome(), 'identity.json'); + let id: string | null = null; + let pseudonymizationKey: string | null = null; try { if (existsSync(idPath)) { const obj = JSON.parse(readFileSync(idPath, 'utf8')) as Record; - if (typeof obj.id === 'string' && obj.id) return obj.id; + if (typeof obj.id === 'string' && obj.id) id = obj.id; + if ( + typeof obj.pseudonymization_key === 'string' && + obj.pseudonymization_key.length >= 32 + ) { + pseudonymizationKey = obj.pseudonymization_key; + } } } catch { - // fall through to generate a new one + // Fall through to generate missing identity fields. } - const id = randomUUID(); + + const identity = { + id: id ?? randomUUID(), + pseudonymizationKey: + pseudonymizationKey ?? randomBytes(32).toString('hex'), + }; try { - mkdirSync(rudderHome(), { recursive: true }); - writeFileSync(idPath, JSON.stringify({ id })); + if (id === null || pseudonymizationKey === null) { + persistIdentity(idPath, identity); + } else { + restrictIdentityPath(rudderHome(), 0o700); + restrictIdentityPath(idPath, 0o600); + } } catch { - // best-effort; use an in-memory ID if we can't persist + // Best-effort; use an in-memory identity if it cannot be persisted. } - return id; + return identity; } let _client: PostHog | null = null; -let _distinctId: string | null = null; +let _identity: TelemetryIdentity | null = null; function client(): PostHog | null { if (!POSTHOG_PROJECT_TOKEN || telemetryDisabled()) return null; @@ -57,16 +137,64 @@ function client(): PostHog | null { } export function distinctId(): string { - if (!_distinctId) _distinctId = loadDistinctId(); - return _distinctId; + if (!_identity) _identity = loadIdentity(); + return _identity.id; +} + +function pseudonymizationKey(): string { + if (!_identity) _identity = loadIdentity(); + return _identity.pseudonymizationKey; } -export function capture(event: string, properties?: Record): void { - client()?.capture({ distinctId: distinctId(), event, properties }); +/** + * Derive a stable, installation-scoped pseudonym without exposing the source + * identifier or allowing it to be correlated across Rudder installations. + */ +export function pseudonymize( + secret: string, + namespace: string, + value: string +): string { + return createHmac('sha256', secret) + .update(namespace) + .update('\0') + .update(value) + .digest('hex'); +} + +export function capture( + event: string, + properties?: TelemetryProperties | TelemetryPropertiesFactory +): void { + const telemetryClient = client(); + if (!telemetryClient) return; + + const installationId = distinctId(); + const eventProperties = + typeof properties === 'function' + ? properties({ + pseudonymize: (namespace, value) => + pseudonymize(pseudonymizationKey(), namespace, value), + }) + : properties; + + telemetryClient.capture({ + distinctId: installationId, + event, + properties: { + ...runtimeTelemetryProperties(), + ...eventProperties, + }, + }); } -export function captureException(err: unknown, extra?: Record): void { - client()?.captureException(err, distinctId(), extra); +export function captureException(err: unknown, extra?: TelemetryProperties): void { + const telemetryClient = client(); + if (!telemetryClient) return; + telemetryClient.captureException(err, distinctId(), { + ...runtimeTelemetryProperties(), + ...extra, + }); } export async function shutdown(): Promise { diff --git a/test/plugin-package.test.ts b/test/plugin-package.test.ts index 6533d73..03b84bc 100644 --- a/test/plugin-package.test.ts +++ b/test/plugin-package.test.ts @@ -110,6 +110,7 @@ test('keeps the Rudder package version synchronized across the codebase', () => }); +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb386-4741-7860-89a9-97f3697fa4f1 test('ships a public marketplace catalog and its package resources', () => { const marketplace = JSON.parse( readFileSync( @@ -136,6 +137,7 @@ test('ships a public marketplace catalog and its package resources', () => { ['skills', 'rudder', 'scripts', 'backup-tests.mjs'], ['skills', 'rudder', 'scripts', 'context.mjs'], ['skills', 'rudder', 'scripts', 'manage-data.mjs'], + ['skills', 'rudder', 'scripts', 'telemetry.mjs'], ['docs', 'install.md'], ['docs', 'privacy.md'], ['docs', 'support.md'], diff --git a/test/prompt-hook.test.ts b/test/prompt-hook.test.ts index bd7d176..5bd18f0 100644 --- a/test/prompt-hook.test.ts +++ b/test/prompt-hook.test.ts @@ -295,7 +295,7 @@ test('the executable performs both phases without model-visible output', () => { assert.equal(storedPrompt?.previousAgentOutput, null); }); -// codex/019fb375-79ec-7b02-b9d8-19fc4bfcc939/019fb376-ca9b-7243-af54-c8affdbc0dc3 +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb386-4741-7860-89a9-97f3697fa4f1 test('the executable flushes metadata-only telemetry before exiting', async () => { closeDb(); const capturePath = join(root, 'telemetry-capture.jsonl'); @@ -343,14 +343,110 @@ test('the executable flushes metadata-only telemetry before exiting', async () = cwd: repo, }), }); + execFileSync( + process.execPath, + [hookExecutable, '--rudder-event', 'run-started'], + { + cwd: repo, + encoding: 'utf8', + env: { + ...process.env, + DO_NOT_TRACK: '', + POSTHOG_PROJECT_TOKEN: 'test-project-token', + POSTHOG_HOST: receiver.host, + RUDDER_HOME: process.env.RUDDER_HOME, + }, + input: JSON.stringify({ + host: 'codex', + repository: 'github.com/private/RAW-REPOSITORY', + branch: 'RAW-BRANCH', + runId: 'RAW-RUN-ID', + capturedPromptCount: 3, + capturedSessionCount: 2, + reconciledPromptCount: 2, + promptSourceCounts: { codex: 2, 'claude-code': 1 }, + changedPathCount: 4, + changedTestPathCount: 1, + changedProductionPathCount: 3, + untrackedPathCount: 1, + testLineAdditionCount: 8, + testLineDeletionCount: 2, + inputTokens: 10, + model: 'RAW-MODEL', + toolUsage: { Read: 1 }, + costUsd: 0.25, + }), + } + ); const requestBodies = readFileSync(capturePath, 'utf8'); - assert.match(requestBodies, /"event":"prompt recorded"/); - assert.match(requestBodies, /"event":"prompt reconciled"/); + assert.match(requestBodies, /"event":"rudder prompt captured"/); + assert.match(requestBodies, /"event":"rudder prompt reconciled"/); + assert.match(requestBodies, /"event":"rudder run started"/); assert.match(requestBodies, /"source":"codex"/); assert.match(requestBodies, /"has_previous_agent_output":false/); + assert.match(requestBodies, /"telemetry_schema_version":1/); + assert.match(requestBodies, /"repository_pseudonym":"[a-f0-9]{64}"/); + assert.match(requestBodies, /"run_pseudonym":"[a-f0-9]{64}"/); + assert.match(requestBodies, /"captured_prompt_count":3/); + assert.match(requestBodies, /"captured_session_count":2/); + assert.match(requestBodies, /"changed_test_path_count":1/); + assert.match(requestBodies, /"test_lines_added_from_base":8/); + assert.match(requestBodies, /"test_lines_deleted_from_base":2/); assert.doesNotMatch(requestBodies, /This prompt must stay local/); assert.doesNotMatch(requestBodies, /telemetry-session|telemetry-turn/); + assert.doesNotMatch( + requestBodies, + /RAW-|inputTokens|RAW-MODEL|toolUsage|costUsd/ + ); + assert.equal(requestBodies.includes(repo), false); + } finally { + await receiver.stop(); + } +}); + +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb3a8-3fc7-7410-aa52-473842eeab9a +test('DO_NOT_TRACK prevents Rudder event delivery and identity creation', async () => { + closeDb(); + const capturePath = join(root, 'do-not-track-capture.jsonl'); + const statePath = join(root, 'do-not-track-state'); + const receiver = await startTelemetryReceiver(capturePath); + + try { + execFileSync( + process.execPath, + [hookExecutable, '--rudder-event', 'run-started'], + { + cwd: repo, + encoding: 'utf8', + env: { + ...process.env, + DO_NOT_TRACK: '1', + POSTHOG_PROJECT_TOKEN: 'test-project-token', + POSTHOG_HOST: receiver.host, + RUDDER_HOME: statePath, + }, + input: JSON.stringify({ + host: 'codex', + repository: 'github.com/private/repository', + branch: 'private-branch', + runId: 'private-run', + capturedPromptCount: 1, + capturedSessionCount: 1, + reconciledPromptCount: 1, + promptSourceCounts: { codex: 1 }, + changedPathCount: 1, + changedTestPathCount: 1, + changedProductionPathCount: 0, + untrackedPathCount: 0, + testLineAdditionCount: 4, + testLineDeletionCount: 0, + }), + } + ); + + assert.equal(existsSync(capturePath), false); + assert.equal(existsSync(join(statePath, 'identity.json')), false); } finally { await receiver.stop(); } @@ -394,6 +490,7 @@ test('the legacy PostHog API key does not enable telemetry', async () => { } }); +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb386-4741-7860-89a9-97f3697fa4f1 test('a release build sends telemetry without user environment configuration', async () => { closeDb(); const capturePath = join(root, 'built-telemetry-capture.jsonl'); @@ -462,7 +559,7 @@ test('a release build sends telemetry without user environment configuration', a assert.equal(stdout, ''); const requestBodies = readFileSync(capturePath, 'utf8'); - assert.match(requestBodies, /"event":"prompt recorded"/); + assert.match(requestBodies, /"event":"rudder prompt captured"/); assert.doesNotMatch( requestBodies, /This built prompt must stay local|built-telemetry-session|built-telemetry-turn/ diff --git a/test/rudder-telemetry.test.ts b/test/rudder-telemetry.test.ts new file mode 100644 index 0000000..23f3007 --- /dev/null +++ b/test/rudder-telemetry.test.ts @@ -0,0 +1,248 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + rudderUsageEventProperties, + type RudderUsageEvent, +} from '../src/rudder-telemetry.ts'; +import { + pseudonymize, + runtimeTelemetryProperties, + telemetryDisabled, + type TelemetryCaptureContext, +} from '../src/telemetry.ts'; + +const context: TelemetryCaptureContext = { + pseudonymize: (namespace) => `${namespace}-pseudonym`, +}; + +function properties(event: RudderUsageEvent, payload: unknown) { + return rudderUsageEventProperties(event, payload, context); +} + +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb386-4741-7860-89a9-97f3697fa4f1 +test('run telemetry measures Rudder intent and scope without agent usage', () => { + const result = properties('run-started', { + host: 'codex', + repository: 'github.com/private/raw-repository', + branch: 'raw-branch', + runId: 'raw-run-id', + capturedPromptCount: 4, + capturedSessionCount: 2, + reconciledPromptCount: 3, + promptSourceCounts: { + codex: 3, + 'claude-code': 1, + }, + changedPathCount: 5, + changedTestPathCount: 2, + changedProductionPathCount: 3, + untrackedPathCount: 1, + testLineAdditionCount: 18, + testLineDeletionCount: 6, + inputTokens: 100, + model: 'private-model', + toolUsage: { Read: 2 }, + costUsd: 1.25, + }); + + assert.deepEqual(result, { + repository_pseudonym: 'repository-pseudonym', + branch_pseudonym: 'branch-pseudonym', + run_pseudonym: 'run-pseudonym', + repository_is_local: false, + host: 'codex', + captured_prompt_count: 4, + captured_session_count: 2, + reconciled_prompt_count: 3, + unreconciled_prompt_count: 1, + captured_prompt_sources: ['claude-code', 'codex'], + captured_prompt_source_counts: { + 'claude-code': 1, + codex: 3, + }, + has_captured_intent: true, + changed_path_count: 5, + changed_test_path_count: 2, + changed_production_candidate_path_count: 3, + untracked_path_count: 1, + test_lines_added_from_base: 18, + test_lines_deleted_from_base: 6, + }); + assert.doesNotMatch( + JSON.stringify(result), + /raw-repository|raw-branch|raw-run-id|inputTokens|private-model|toolUsage|costUsd/ + ); +}); + +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb39d-12e9-7673-b7d7-04d6c3f27243 +test('finished runs report test line snapshots and Rudder question counts', () => { + const result = properties('run-finished', { + host: 'claude-code', + repository: 'github.com/private/raw-repository', + branch: 'raw-branch', + runId: 'raw-run-id', + status: 'completed', + testsPassed: 'yes', + coverageTargetMet: 'no', + changedPathCount: 6, + changedTestPathCount: 4, + changedProductionPathCount: 2, + promptBackedTestCount: 5, + testLineAdditionCount: 42, + testLineDeletionCount: 9, + questionsAskedCount: 3, + }); + + assert.equal(result.status, 'completed'); + assert.equal(result.tests_passed, 'yes'); + assert.equal(result.coverage_target_met, 'no'); + assert.equal(result.prompt_backed_test_count, 5); + assert.equal(result.final_test_lines_added_from_base, 42); + assert.equal(result.final_test_lines_deleted_from_base, 9); + assert.equal(result.questions_asked_count, 3); + assert.doesNotMatch( + JSON.stringify(result), + /raw-repository|raw-branch|raw-run-id/ + ); +}); + +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb39d-12e9-7673-b7d7-04d6c3f27243 +test('question telemetry counts questions without question or answer text', () => { + const result = properties('question-asked', { + host: 'codex', + repository: 'github.com/private/raw-repository', + branch: 'raw-branch', + runId: 'raw-run-id', + questionNumber: 2, + questionText: 'RAW-QUESTION', + answerText: 'RAW-ANSWER', + }); + + assert.equal(result.run_pseudonym, 'run-pseudonym'); + assert.equal(result.question_number, 2); + assert.deepEqual(Object.keys(result).sort(), [ + 'branch_pseudonym', + 'host', + 'question_number', + 'repository_is_local', + 'repository_pseudonym', + 'run_pseudonym', + ]); + assert.doesNotMatch( + JSON.stringify(result), + /raw-repository|raw-branch|raw-run-id|RAW-QUESTION|RAW-ANSWER/ + ); +}); + +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb3c0-6a86-7e53-a68a-3721b8c4ed70 +test('backup telemetry reports copied paths and rejects impossible counts', () => { + const payload = { + host: 'codex', + repository: 'github.com/private/raw-repository', + branch: 'raw-branch', + runId: 'raw-run-id', + approvedTestPathCount: 3, + copiedUntrackedTestPathCount: 1, + }; + const result = properties('test-backup-created', payload); + + assert.equal(result.approved_test_path_count, 3); + assert.equal(result.copied_untracked_test_path_count, 1); + assert.equal(result.run_pseudonym, 'run-pseudonym'); + assert.doesNotMatch( + JSON.stringify(result), + /raw-repository|raw-branch|raw-run-id/ + ); + for (const field of [ + 'approvedTestPathCount', + 'copiedUntrackedTestPathCount', + ]) { + assert.throws( + () => + properties('test-backup-created', { + ...payload, + [field]: -1, + }), + new RegExp(`${field} must be a non-negative integer`) + ); + } +}); + +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb386-4741-7860-89a9-97f3697fa4f1 +test('Rudder telemetry validates product outcomes and question ordinals', () => { + assert.throws( + () => + properties('run-finished', { + host: 'codex', + repository: 'repository', + branch: 'branch', + runId: 'run-id', + status: 'successful-ish', + testsPassed: 'yes', + coverageTargetMet: 'yes', + changedPathCount: 0, + changedTestPathCount: 0, + changedProductionPathCount: 0, + promptBackedTestCount: 0, + testLineAdditionCount: 0, + testLineDeletionCount: 0, + questionsAskedCount: 0, + }), + /status must be one of/ + ); + assert.throws( + () => + properties('question-asked', { + host: 'codex', + repository: 'repository', + branch: 'branch', + runId: 'run-id', + questionNumber: 0, + }), + /questionNumber must be a positive integer/ + ); +}); + +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb386-4741-7860-89a9-97f3697fa4f1 +test('repository pseudonyms are stable and local-secret scoped', () => { + const first = pseudonymize( + 'local-secret-a', + 'repository', + 'github.com/private/repository' + ); + assert.equal( + first, + pseudonymize( + 'local-secret-a', + 'repository', + 'github.com/private/repository' + ) + ); + assert.notEqual( + first, + pseudonymize( + 'local-secret-b', + 'repository', + 'github.com/private/repository' + ) + ); + assert.doesNotMatch(first, /private|repository/); +}); + +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb386-4741-7860-89a9-97f3697fa4f1 +test('common telemetry is limited to Rudder schema and version', () => { + const result = runtimeTelemetryProperties(); + assert.equal(result.telemetry_schema_version, 1); + assert.equal(typeof result.rudder_version, 'string'); + assert.deepEqual(Object.keys(result).sort(), [ + 'rudder_version', + 'telemetry_schema_version', + ]); +}); + +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb3a8-3fc7-7410-aa52-473842eeab9a +test('the canonical DO_NOT_TRACK value disables Rudder telemetry', () => { + assert.equal(telemetryDisabled({ DO_NOT_TRACK: '1' }), true); + assert.equal(telemetryDisabled({ DO_NOT_TRACK: '0' }), false); + assert.equal(telemetryDisabled({}), false); +}); diff --git a/test/skill-runtime.test.ts b/test/skill-runtime.test.ts index ff3c317..a669ce4 100644 --- a/test/skill-runtime.test.ts +++ b/test/skill-runtime.test.ts @@ -9,12 +9,15 @@ import { rmSync, writeFileSync, } from 'node:fs'; +import { createServer } from 'node:http'; +import type { Socket } from 'node:net'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { after, before, test } from 'node:test'; import { fileURLToPath } from 'node:url'; import { closeDb } from '../src/db/client.ts'; import { recordPromptHookEvent } from '../src/prompt-hook.ts'; +import { captureRudderTelemetry } from '../skills/rudder/scripts/telemetry.mjs'; import { promptsForBranch, promptsForSession, @@ -42,6 +45,13 @@ const dataScript = join( 'scripts', 'manage-data.mjs' ); +const telemetryScript = join( + pluginRoot, + 'skills', + 'rudder', + 'scripts', + 'telemetry.mjs' +); const updateScriptUrl = new URL( '../skills/rudder/scripts/update.mjs', import.meta.url @@ -72,6 +82,7 @@ interface UpdateModule { let root: string; let repo: string; let stateRoot: string; +let rudderRunId: string; let originalRudderHome: string | undefined; function git(...args: string[]): string { @@ -93,6 +104,48 @@ async function loadUpdateModule(): Promise { return import(updateScriptUrl.href) as Promise; } +async function startHangingTelemetryReceiver(): Promise<{ + host: string; + received: Promise; + stop: () => Promise; +}> { + let resolveReceipt: () => void; + const received = new Promise((resolve) => { + resolveReceipt = resolve; + }); + const sockets = new Set(); + const server = createServer((request) => { + request.resume(); + resolveReceipt(); + }); + server.on('connection', (socket) => { + sockets.add(socket); + socket.once('close', () => sockets.delete(socket)); + }); + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(0, '127.0.0.1', () => { + server.off('error', reject); + resolve(); + }); + }); + const address = server.address(); + if (!address || typeof address === 'string') { + throw new Error('telemetry receiver did not bind to a TCP port'); + } + + return { + host: `http://127.0.0.1:${address.port}`, + received, + stop: async () => { + for (const socket of sockets) socket.destroy(); + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + }, + }; +} + before(() => { root = mkdtempSync(join(tmpdir(), 'rudder-skill-runtime-')); repo = join(root, 'repo'); @@ -126,7 +179,8 @@ test('data controls do not permit disabling prompt capture', () => { assert.match(disabled.stderr, /status\|delete/); }); -test('the skill helper returns branch changes and locally captured intent', () => { +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb39d-12e9-7673-b7d7-04d6c3f27243 +test('the skill helper returns intent, run identity, and initial test lines', () => { const originalCaptureDisabled = process.env.RUDDER_DISABLE_PROMPT_CAPTURE; mkdirSync(stateRoot, { recursive: true }); writeFileSync( @@ -161,7 +215,15 @@ test('the skill helper returns branch changes and locally captured intent', () = const context = JSON.parse( execFileSync( process.execPath, - [contextScript, '--cwd', repo, '--base', 'HEAD'], + [ + contextScript, + '--cwd', + repo, + '--base', + 'HEAD', + '--phase', + 'start', + ], { encoding: 'utf8', env: { ...process.env, RUDDER_HOME: stateRoot }, @@ -169,6 +231,9 @@ test('the skill helper returns branch changes and locally captured intent', () = ) ) as { branch: string; + rudderRunId: string; + testLineAdditionCount: number; + testLineDeletionCount: number; otherPaths: string[]; testPaths: string[]; prompts: Array<{ @@ -180,6 +245,10 @@ test('the skill helper returns branch changes and locally captured intent', () = }; assert.equal(context.branch, 'main'); + assert.match(context.rudderRunId, /^[a-f0-9-]{36}$/); + assert.equal(context.testLineAdditionCount, 1); + assert.equal(context.testLineDeletionCount, 0); + rudderRunId = context.rudderRunId; assert.deepEqual(context.testPaths, ['test/feature.test.ts']); assert.ok(context.otherPaths.includes('src/feature.ts')); assert.deepEqual( @@ -412,6 +481,7 @@ test('retries a failed update twice without blocking the flow', async () => { } }); +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb386-4741-7860-89a9-97f3697fa4f1 test('the skill helper backs up only explicit test paths', () => { const backup = JSON.parse( execFileSync( @@ -422,6 +492,8 @@ test('the skill helper backs up only explicit test paths', () => { repo, '--base', 'HEAD', + '--run-id', + rudderRunId, '--path', 'test/feature.test.ts', ], @@ -454,6 +526,176 @@ test('the skill helper backs up only explicit test paths', () => { ); }); +// codex/019fb3c6-46eb-7bc1-8367-9f8b11fbd7c2/019fb3e2-de62-7150-86a1-5e2421c5ceb6 +test('the skill telemetry dispatcher does not wait for an unresponsive receiver', async () => { + const receiver = await startHangingTelemetryReceiver(); + try { + const startedAt = Date.now(); + const result = spawnSync( + process.execPath, + [ + telemetryScript, + 'question-asked', + '--cwd', + repo, + '--run-id', + rudderRunId, + '--question-number', + '2', + ], + { + encoding: 'utf8', + env: { + ...process.env, + DO_NOT_TRACK: '', + POSTHOG_PROJECT_TOKEN: 'test-project-token', + POSTHOG_HOST: receiver.host, + RUDDER_HOME: stateRoot, + }, + } + ); + + assert.equal(result.status, 0, result.stderr); + assert.ok( + Date.now() - startedAt < 1_000, + 'telemetry dispatch must not wait for the receiver response' + ); + await new Promise((resolve, reject) => { + const timeout = setTimeout( + () => reject(new Error('telemetry event was not dispatched')), + 1_000 + ); + receiver.received.then( + () => { + clearTimeout(timeout); + resolve(); + }, + reject + ); + }); + } finally { + await receiver.stop(); + } +}); + +// codex/019fb3c6-46eb-7bc1-8367-9f8b11fbd7c2/019fb3e2-de62-7150-86a1-5e2421c5ceb6 +test('the skill telemetry dispatcher ignores invalid payloads and launch errors', async () => { + assert.equal( + captureRudderTelemetry('question-asked', { unsupported: BigInt(1) }), + false + ); + + const originalExecutable = process.execPath; + process.execPath = join(root, 'missing-node-executable'); + try { + assert.equal( + captureRudderTelemetry('question-asked', { questionNumber: 3 }), + true + ); + await new Promise((resolve) => setTimeout(resolve, 20)); + } finally { + process.execPath = originalExecutable; + } +}); + +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb39d-12e9-7673-b7d7-04d6c3f27243 +test('the skill records each behavioral question in its Rudder run', () => { + const question = JSON.parse( + execFileSync( + process.execPath, + [ + telemetryScript, + 'question-asked', + '--cwd', + repo, + '--run-id', + rudderRunId, + '--question-number', + '1', + ], + { + encoding: 'utf8', + env: { ...process.env, RUDDER_HOME: stateRoot }, + } + ) + ) as { + questionNumber: number; + telemetryDispatched: boolean; + }; + + assert.deepEqual(question, { + schemaVersion: 1, + telemetryDispatched: true, + questionNumber: 1, + }); +}); + +// codex/019fb36f-4dfe-7c91-8674-5caaf68fcced/019fb39d-12e9-7673-b7d7-04d6c3f27243 +test('the skill reports final test lines and total Rudder questions', () => { + writeFileSync( + join(repo, 'test', 'feature.test.ts'), + [ + '// codex/skill-context/skill-turn', + "test('returns cached data', () => {});", + '', + ].join('\n') + ); + const outcome = JSON.parse( + execFileSync( + process.execPath, + [ + telemetryScript, + 'complete', + '--cwd', + repo, + '--base', + 'HEAD', + '--run-id', + rudderRunId, + '--status', + 'completed', + '--tests-passed', + 'yes', + '--coverage-target-met', + 'no', + '--questions-asked', + '1', + ], + { + encoding: 'utf8', + env: { ...process.env, RUDDER_HOME: stateRoot }, + } + ) + ) as { + telemetryDispatched: boolean; + status: string; + testsPassed: string; + coverageTargetMet: string; + changedPathCount: number; + changedTestPathCount: number; + changedProductionPathCount: number; + promptBackedTestCount: number; + finalTestLineAdditionCount: number; + finalTestLineDeletionCount: number; + questionsAskedCount: number; + }; + + assert.deepEqual(outcome, { + schemaVersion: 1, + telemetryDispatched: true, + status: 'completed', + testsPassed: 'yes', + coverageTargetMet: 'no', + changedPathCount: 2, + changedTestPathCount: 1, + changedProductionPathCount: 1, + promptBackedTestCount: 1, + finalTestLineAdditionCount: 2, + finalTestLineDeletionCount: 0, + questionsAskedCount: 1, + }); +}); + test('data controls require confirmation and delete only prompt records', () => { assert.equal(runData('status').promptCount, 1);