diff --git a/.github/workflows/release-downloads.yml b/.github/workflows/release-downloads.yml new file mode 100644 index 0000000000..39e40c1101 --- /dev/null +++ b/.github/workflows/release-downloads.yml @@ -0,0 +1,33 @@ +name: Release downloads + +# Reports release download counts to PostHog once a day. The counts come from +# public GitHub release data — the API publishes a cumulative download_count +# per asset — so this needs no code in the binary. One event per binary asset +# (codeaf--[.exe]); sidecars like checksums.txt are not reported. +on: + schedule: + # 06:00 UTC every day. + - cron: '0 6 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: release-downloads + cancel-in-progress: false + +jobs: + report: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Report release download counts + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + CODEAF_POSTHOG_PROJECT_KEY: ${{ secrets.CODEAF_POSTHOG_PROJECT_KEY }} + # Repository variable; unset falls back to https://us.i.posthog.com. + POSTHOG_HOST: ${{ vars.POSTHOG_HOST }} + run: | + python3 scripts/release_downloads.py diff --git a/README.md b/README.md index d2c44aec56..acd788a98c 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,29 @@ pinning are on the [releases page](https://github.com/Agent-Field/codeaf/release On first start it asks for a key: OpenRouter, DeepSeek, GLM, Kimi, MiniMax or Qwen. Ollama needs none. +## Telemetry + +```text +codeaf sends anonymous usage counts to AgentField. + Sent: version, OS, mode (chat or task), how many sessions, how many errors. + Never: anything about you or your work. No prompts, code, file names, + paths, repo names, keys, email, IP, or machine name. + See exactly what leaves: codeaf telemetry show + Turn off: CODEAF_TELEMETRY=off +``` + +The events are counts and buckets only — no prompts, code, or file names ever +leave the machine. `docs/TELEMETRY.md` lists every field that is sent and every +way to turn the counts off. The installer records how and when this copy was +installed (`telemetry/install.json` under the state root) so the counts can +bucket by channel; it sends nothing. + +`codeaf telemetry show` prints exactly what is waiting to leave the machine; +`status`, `on` and `off` are the rest. Turn it off with `CODEAF_TELEMETRY=off` +(also `0` or `false`), `DO_NOT_TRACK=1`, `codeaf telemetry off`, `telemetry = +off` in a project's `.codeaf/config.json`, or an empty +`CODEAF_TELEMETRY_ENDPOINT`. + ## One window for every project An agent that lives in one folder means a terminal per repository, and a tmux diff --git a/cmd/codeaf/do.go b/cmd/codeaf/do.go index 9ce388f972..787c8cd0be 100644 --- a/cmd/codeaf/do.go +++ b/cmd/codeaf/do.go @@ -589,6 +589,21 @@ func errandRun(request doRequest, seats config.Seats, started time.Time) (outcom // nothing deletes the store either. signalled, stopSignals := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stopSignals() + // The person's own ending of the run is the one fact the usage counts + // keep about how it ended: the interrupt word, not a failure word. + // signalled cannot answer it — stopSignals cancels that context on the + // way out too, so its Err reads the same for a clean end and a ctrl-c. + // A channel only a signal fills can. + interrupts := make(chan os.Signal, 1) + signal.Notify(interrupts, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(interrupts) + defer func() { + select { + case <-interrupts: + telemetryInterrupted = true + default: + } + }() // A PERSON TYPED THIS, so its calls are made for somebody who is reading // them (exec.go's [typedDoorContext]): the talk pin rides them and a // refused pin is said to the one who is waiting. diff --git a/cmd/codeaf/fault.go b/cmd/codeaf/fault.go index 7a1139e349..0cb4e0361a 100644 --- a/cmd/codeaf/fault.go +++ b/cmd/codeaf/fault.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "io" "os" @@ -9,6 +10,7 @@ import ( "time" "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/telemetry" ) // faultMessage is what the user reads when codeaf could not keep going. It is @@ -17,7 +19,10 @@ import ( const faultMessage = "codeaf hit an internal fault and had to stop. Nothing is lost — the graph is durable, and restarting resumes where it left off. Details: %s\n" // reportFault writes the stack where it is useful and the sentence where it is -// read, and answers with the process exit code. +// read, and answers with the process exit code. The anonymous fault count is +// the last thing it does, once the sentence is out: a flush may spend a second +// this process no longer has, and the person should not wait on it to read +// what happened. func reportFault(stderr io.Writer, detail string, stack []byte) int { // A fault has no settings to read — it is what is left when the launch did // not get that far — but the profile is an environment pin and is still @@ -29,9 +34,39 @@ func reportFault(stderr io.Writer, detail string, stack []byte) int { path := chatLogPath(config.ProfileDir()) writeFaultLog(path, detail, stack) fmt.Fprintf(stderr, faultMessage, displayPath(path)) + telemetryFault(stack) return 1 } +// telemetryFault spools the contract's fault event for a dying process and +// sends what the spool holds. It is given the stack and nothing else. The +// library hashes the fingerprint out of the stack's own function names, and +// `detail` — the panic's text — never crosses this line: a panic value is the +// person's words, a path on their machine or a fragment of the file they were +// working in, and the fault event has no property for any of it. +// +// SpoolSync and then Flush under a one-second context, the same budget +// telemetryEnd gives a run's last event, because this is the last thing the +// process does: there is no later flush to carry the line, and an append that +// has not landed when the process exits is an event that never happened. The +// gate is the opt-out ladder's own answer, so a person who turned the counts +// off gets the same crash report they always got and nothing on the wire. +func telemetryFault(stack []byte) { + if !telemetry.Enabled() { + return + } + session := currentTelemetrySession + event := telemetry.FaultEvent(telemetry.Fault{ + Mode: string(session.mode), + Scope: telemetry.ScopeMain, + Stack: stack, + }, session.sessionID, time.Now()) + _ = telemetry.SpoolSync(event) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = telemetry.Flush(ctx) +} + // chatLogPath is the ONE name of the file this binary parks the standard logger // in: EVERY door that opens the v3 surface does it for the surface's whole // lifetime so a log line cannot tear through the frame (chatv3_surface.go's diff --git a/cmd/codeaf/main.go b/cmd/codeaf/main.go index 9e03dd5229..5c59f6512c 100644 --- a/cmd/codeaf/main.go +++ b/cmd/codeaf/main.go @@ -31,6 +31,7 @@ import ( lanes "github.com/Agent-Field/codeaf/internal/lane" "github.com/Agent-Field/codeaf/internal/plan" "github.com/Agent-Field/codeaf/internal/router" + "github.com/Agent-Field/codeaf/internal/telemetry" "github.com/Agent-Field/codeaf/internal/trace" ) @@ -71,6 +72,21 @@ func execute() (code int) { code = reportFault(os.Stderr, fmt.Sprint(recovered), debug.Stack()) } }() + // The anonymous usage counts get their one configured answer before any + // command is dispatched, and their one session event at the exit every + // command shares. Both are here, in execute, because nothing else in this + // file is reached by all of chat, do, exec, run and `plan run` — and a + // counter that missed a door would miscount the runs it was built to count. + // + // The CONFIG read must never become a failure of its own: a machine with + // no profile yet, or a broken project file, is a machine `codeaf version` + // still owes an answer to. Any error reads as "on" — the default — and + // the run carries on. + telemetry.Configure(telemetryConfiguredOff()) + telemetrySession := telemetryBegin() + defer func() { + telemetryEnd(telemetrySession, code) + }() // The model-call log's file descriptor goes back at the one exit every // command shares (internal/calllog). Nothing depends on it — every record is // written and flushed as it happens — but a process that closes what it @@ -225,6 +241,10 @@ func run() error { return runRebuild(os.Args[2:]) case "why": return runWhy(os.Args[2:]) + case "telemetry": + // The person's door onto the anonymous-usage pipe: what it is doing, + // exactly what would leave, and the switch. It emits nothing itself. + return runTelemetry(os.Args[2:]) case "manual": // Everything codeaf knows about itself, read straight (manual.go). It // is the same corpus the chat's manual tool reads, printed as it is @@ -301,11 +321,10 @@ const ( // and where taking it out of the group's lines had silently removed it. One // source of truth, two places it is read. var handWorkFooter = ` the three differ by how much thinking happens first: do plans and may split - the job, exec does not plan, run follows a plan somebody saved. None of them - takes --yolo: they run with nobody watching and nothing in them stops to ask. - What do and run can still refuse is a plan whose price crosses your limit, - and --yes-spend answers that in advance. All three end the same way, and - why is in --json's stop field: + the job, exec does not plan, run follows a plan somebody saved. None takes + --yolo, and What do and run can still refuse is a plan whose price crosses + your limit — --yes-spend answers that in advance. All three end the same + way, and why is in --json's stop field: ` + foldedExitLadder(2, helpWidth) + ` CODEAF_EXIT_CODES=legacy restores exec's old 2/3/4/5/6 for one release` @@ -345,11 +364,13 @@ Look at what happened — read-only, no key, nothing spent show today's self-spend receipts codeaf why [--db path] what one piece of work did — its turns, tools, arguments, how it ended + codeaf telemetry + the anonymous usage counts: status, show, off, on codeaf logs [--tail 40] [--follow] [--path] [--json] [--run id] [--call id] [--tag t] [--model m] [--node n] [--body id] every model call codeaf made — what was asked, which lane answered, what - came back. The filters are exact and combine; --json prints the rows as - they are on disk, --body one call's bodies. CODEAF_CALL_LOG=off is off + came back; --json prints the rows as on disk, --body one call's bodies. + The filters are exact and combine; CODEAF_CALL_LOG=off is off codeaf models [--refresh] the models this machine will use, and what each has been measured at codeaf doctor [--db path] @@ -374,8 +395,7 @@ Housekeeping — changes state on disk or on the network codeaf devices list the devices paired with this machine codeaf devices revoke [--all] - stop one device opening a conversation here; --all, every device of that - name + stop one device opening a conversation here, --all every device of it codeaf notebook [--db path] what it has learned, and what it has been corrected on codeaf notebook retract|restore [--db path] @@ -399,8 +419,8 @@ Plan work by hand — a plan you can read, edit and diff [--out plan.json] [--model slug] [--plan-model slug] codeaf plan run [--dir dir] [--parallel 8] [--out done.json] [--yes-spend] [--model slug] [--plan-model slug] - a plan written to a file, then executed exactly as written. It is not - what most people want: nothing learnt mid-flight moves a frozen plan + a plan written to a file, then executed exactly as written. It is not what + most people want: nothing learnt mid-flight moves a frozen plan Examples: codeaf open the conversation you were having @@ -502,6 +522,15 @@ than fighting your shell. also record each call's whole request and response — your prompts included. Off by default, and for one run at a time. + CODEAF_TELEMETRY on (default). off turns the anonymous usage counts off; + ` + "`codeaf telemetry`" + ` says what they are and what would + leave, DO_NOT_TRACK=1 does the same + CODEAF_TELEMETRY_ENDPOINT + where the usage counts go + (default https://agentfield.ai/api/oss/codeaf/telemetry); + set to empty to turn them off entirely + DO_NOT_TRACK=1 the ecosystem's own opt-out word, honoured as if it were + CODEAF_TELEMETRY=off The user-facing knobs above — budgets, rhythm, the document reader, the vision and media slots — are also the ` + "`/settings`" + ` sheet in the chat, which persists diff --git a/cmd/codeaf/telemetry.go b/cmd/codeaf/telemetry.go new file mode 100644 index 0000000000..792ee28e72 --- /dev/null +++ b/cmd/codeaf/telemetry.go @@ -0,0 +1,166 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/telemetry" +) + +// The `telemetry` command: the person's door onto the anonymous-usage pipe. +// Four verbs, one per question a person arrives with — what is it doing, what +// exactly would leave, and the two ways of turning it off or back on. It +// EMITS NOTHING ITSELF: it is a command about telemetry, not a session, and +// the wiring in main.go's execute() looks at os.Args to make sure of it. +func runTelemetry(args []string) error { + if len(args) == 0 { + return runTelemetryStatus(nil) + } + switch args[0] { + case "status": + return runTelemetryStatus(args[1:]) + case "show": + return runTelemetryShow(args[1:]) + case "on", "off": + return runTelemetrySet(args[0], args[1:]) + default: + return fmt.Errorf("telemetry takes one of: status, show, on, off") + } +} + +// telemetryFlags holds the one flag every verb accepts so a --help reader and +// the tests share one parser. +func telemetryFlags(name string) *flag.FlagSet { + flags := flag.NewFlagSet(name, flag.ContinueOnError) + flags.SetOutput(os.Stderr) + return flags +} + +// runTelemetryStatus prints the pipe's whole answer: on or off, why it is off, +// where events would go, whether the notice has been shown, how many events +// are waiting, and the first 12 characters of the install id hash — enough to +// recognise, far too little to be a person. +func runTelemetryStatus(args []string) error { + flags := telemetryFlags("status") + if err := flags.Parse(args); err != nil { + return err + } + // The config answer goes through the package's single door so the row + // this reads and the events the binary spools cannot disagree about + // whether telemetry is off. + profileDir := config.ProfileDir() + configuredOff := false + if cwd, err := os.Getwd(); err == nil { + configuredOff, _ = config.TelemetryOffReason(cwd, profileDir) + } + telemetry.Configure(configuredOff) + + reason := telemetry.OffReason() + state := "on" + if reason != "" { + state = "off" + } + notice := "not shown" + if telemetry.NoticeShown() { + notice = "shown" + } + fmt.Fprintf(usageOut, "telemetry %s\n", state) + if reason != "" { + fmt.Fprintf(usageOut, " reason: %s\n", reason) + } + fmt.Fprintf(usageOut, " endpoint: %s\n", telemetry.Endpoint()) + fmt.Fprintf(usageOut, " notice: %s\n", notice) + fmt.Fprintf(usageOut, " spooled events: %d\n", len(telemetry.SpoolContents())) + fmt.Fprintf(usageOut, " install: %s…\n", telemetryInstallPrefix()) + return nil +} + +// telemetryInstallPrefix is the first 12 characters of the install id hash — +// the identity the wire sees, truncated to something a person can compare +// between machines and nothing more. +func telemetryInstallPrefix() string { + hash := telemetry.InstallIDHash() + if len(hash) > 12 { + return hash[:12] + } + return hash +} + +// runTelemetryShow prints exactly what is waiting to leave the machine, the +// package's own rendering so the command and `codeaf telemetry show` a person +// reads in the notice are one and the same thing. +func runTelemetryShow(args []string) error { + flags := telemetryFlags("show") + if err := flags.Parse(args); err != nil { + return err + } + fmt.Fprintln(usageOut, telemetry.Show()) + return nil +} + +// runTelemetrySet writes the settings row from internal/config: `telemetry off` +// turns the pipe off for this machine, `telemetry on` turns it back on. It +// writes the PROFILE row — the person's own answer — and says one confirming +// line, because a command that changed a setting silently would be a change +// nobody could audit. +func runTelemetrySet(word string, args []string) error { + flags := telemetryFlags(word) + if err := flags.Parse(args); err != nil { + return err + } + profileDir := config.ProfileDir() + var value bool + switch word { + case "on": + value = true + case "off": + value = false + } + if err := config.WriteTelemetry(profileDir, value); err != nil { + return err + } + if value { + fmt.Fprintln(usageOut, "telemetry on — anonymous usage counts are sent (see `codeaf telemetry show`)") + return nil + } + fmt.Fprintln(usageOut, "telemetry off — nothing is sent; the session counters still count") + return nil +} + +// telemetryMode decides, from the command line, whether this invocation is a +// session worth counting and which of the two modes it ran in. Every command +// that is not named here emits nothing and sends nothing. +func telemetryMode(args []string) (mode telemetry.Mode, resumed bool, session bool) { + // `plan run` is two args: the verb is the second word. Matching `plan` + // alone would also count `plan new`, which writes a file and runs nothing. + if len(args) >= 2 && args[0] == "plan" && args[1] == "run" { + return telemetry.ModeTask, false, true + } + if len(args) < 1 { + return telemetry.ModeChat, false, true + } + switch args[0] { + case "chat": + return telemetry.ModeChat, false, true + case "resume": + return telemetry.ModeChat, true, true + case "do", "exec", "run": + return telemetry.ModeTask, false, true + } + return "", false, false +} + +// telemetryHasJSON reports whether this invocation carries --json, which +// stdout must stay machine-clean for: the notice would be one more line in +// somebody's jq pipeline. +func telemetryHasJSON(args []string) bool { + for _, arg := range args { + if arg == "--json" || strings.HasPrefix(arg, "--json=") { + return true + } + } + return false +} diff --git a/cmd/codeaf/telemetry_fault_test.go b/cmd/codeaf/telemetry_fault_test.go new file mode 100644 index 0000000000..ae95a0ef17 --- /dev/null +++ b/cmd/codeaf/telemetry_fault_test.go @@ -0,0 +1,149 @@ +package main + +// The crash site's one telemetry obligation, held down: reportFault leaves a +// fault event in the spool carrying the contract's three properties, and the +// panic's own text reaches neither a key nor a value of it. +// +// The event is read out of the spool rather than off the wire because that is +// where a dying process leaves it. The flush reportFault makes is a no-op until +// the notice has been shown, and a test's home has not shown it, so the line is +// still on disk here rather than on its way to the endpoint. + +import ( + "bytes" + "regexp" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/telemetry" +) + +// faultDetail is the panic value reportFault is handed: the runtime's words, a +// path on the person's machine, and a token shaped like a credential. A panic +// text carries whatever the failing line was holding, which is exactly why the +// contract gives the fault event no property for it. +const faultDetail = "runtime error: index out of range [3:] reading " + + "/Users/somebody/private/notes.md (OPENROUTER_API_KEY=sk-or-v1-abcdef)" + +// faultStack is a fake dump: two codeaf frames for the fingerprint to hash, +// each with a file half carrying a machine path of its own. +const faultStack = `goroutine 1 [running]: +main.runDo(0x0?) + /Users/somebody/work/codeaf/cmd/codeaf/do.go:42 +0x1c8 +github.com/Agent-Field/codeaf/internal/session.(*Agent).Run(0x0?, {0x0?, 0x0?}) + /Users/somebody/work/codeaf/internal/session/agent.go:100 +0x60 +` + +// faultLeaks are the strings that must not appear anywhere in a spooled row: +// the panic text whole and in its sensitive pieces, and the stack's paths, +// which are the same leak through another door. +var faultLeaks = []string{ + faultDetail, + "index out of range", + "/Users/somebody/private/notes.md", + "OPENROUTER_API_KEY", + "sk-or-v1-abcdef", + "/Users/somebody/work/codeaf", +} + +var reFaultFingerprint = regexp.MustCompile(`^[0-9a-f]{16}$`) + +// TestReportFaultSpoolsAFaultEventAndNotThePanicText runs one crash through +// reportFault and reads what it left behind: a single fault event, scoped to +// the main goroutine, carrying the session's mode and a fingerprint hashed from +// the stack — and nothing that says what panicked. The exit code and the +// sentence on stderr are asserted too, because the count is allowed to cost +// this function neither of them. +func TestReportFaultSpoolsAFaultEventAndNotThePanicText(t *testing.T) { + telemetryLifecycleHome(t) + // The fault log is appended under the profile, and a profile pinned by the + // environment would put it outside this test's home. + t.Setenv(config.ProfileDirEnv, "") + + previous := currentTelemetrySession + currentTelemetrySession = telemetrySession{mode: telemetry.ModeChat, sessionID: "fault-session"} + t.Cleanup(func() { currentTelemetrySession = previous }) + + stderr := &bytes.Buffer{} + if code := reportFault(stderr, faultDetail, []byte(faultStack)); code != 1 { + t.Fatalf("exit code = %d, want 1", code) + } + if shown := stderr.String(); !strings.Contains(shown, "codeaf hit an internal fault and had to stop") { + t.Fatalf("the calm sentence did not reach stderr: %q", shown) + } + + rows := telemetrySpoolRows(t) + faults := 0 + var fault map[string]any + for _, row := range rows { + // Every row is walked, not just the fault's: an asynchronous append + // from an earlier test in this binary can land in this home, so the + // leak check reads all of them and the count below reads only the ones + // named fault. + assertNoFaultLeak(t, row) + if row["event_name"] == "fault" { + faults++ + fault = row + } + } + if faults != 1 { + t.Fatalf("%d fault events for one crash, want 1 (spool holds %v)", + faults, telemetryEventNames(t, rows)) + } + + props, ok := fault["props"].(map[string]any) + if !ok { + t.Fatalf("the fault row carries no props object: %v", fault) + } + if props["scope"] != telemetry.ScopeMain { + t.Fatalf("scope = %v, want main", props["scope"]) + } + if props["mode"] != string(telemetry.ModeChat) { + t.Fatalf("mode = %v, want the session's chat: the event is built off currentTelemetrySession", + props["mode"]) + } + if fingerprint, _ := props["fingerprint"].(string); !reFaultFingerprint.MatchString(fingerprint) { + t.Fatalf("fingerprint = %q, want 16 lowercase hex characters", fingerprint) + } +} + +// assertNoFaultLeak fails on the first key or value of a parsed spool row that +// carries any of the panic's text. Walking the row is the assertion the +// contract actually makes: it is not enough that the fault event has no +// property named for the panic message, because a key or a value anywhere in +// the row could hold one. +func assertNoFaultLeak(t *testing.T, row map[string]any) { + t.Helper() + for _, text := range spoolRowText(row) { + for _, leak := range faultLeaks { + if strings.Contains(text, leak) { + t.Fatalf("the panic text reached the spool: %q holds %q", text, leak) + } + } + } +} + +// spoolRowText is every key and every string value in a parsed row, at any +// depth. Anything that is neither — a number, a bool — cannot carry text and is +// left out. +func spoolRowText(value any) []string { + switch typed := value.(type) { + case map[string]any: + var out []string + for key, child := range typed { + out = append(out, key) + out = append(out, spoolRowText(child)...) + } + return out + case []any: + var out []string + for _, child := range typed { + out = append(out, spoolRowText(child)...) + } + return out + case string: + return []string{typed} + } + return nil +} diff --git a/cmd/codeaf/telemetry_lifecycle.go b/cmd/codeaf/telemetry_lifecycle.go new file mode 100644 index 0000000000..e230d8ffd5 --- /dev/null +++ b/cmd/codeaf/telemetry_lifecycle.go @@ -0,0 +1,243 @@ +package main + +// The execute() lifecycle for the anonymous usage counts: which invocation is +// a session, which mode it ran in, when the notice is printed, and how a run +// ends in the contract's stop-reason vocabulary. Everything here is driven +// from execute() (main.go), the one exit every command leaves through, so no +// door can be missed by a counter that would then miscount the runs it was +// built to count. + +import ( + "context" + "crypto/rand" + "encoding/hex" + "os" + "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/guard" + "github.com/Agent-Field/codeaf/internal/telemetry" + "github.com/Agent-Field/codeaf/internal/trace" +) + +// telemetryConfiguredOff is the config's one answer to the ladder, read the +// same way the `telemetry status` verb reads it. An unreadable config is the +// ladder's default — on — never a failure the run inherits: `codeaf version` +// still owes its answer on a machine with no profile, and a config error has +// never been a reason to refuse one. +func telemetryConfiguredOff() bool { + cwd, err := os.Getwd() + if err != nil { + return false + } + off, _ := config.TelemetryOffReason(cwd, config.ProfileDir()) + return off +} + +// telemetrySession is what execute() needs to close one run: the mode it +// decided, whether it was a resume, and the id the run's events are hashed +// from. A nil mode means this invocation is not a session at all (version, +// help, doctor, the rest) and nothing is spooled for it. +type telemetrySession struct { + mode telemetry.Mode + resumed bool + sessionID string +} + +// currentTelemetrySession is the session this invocation decided, kept where +// the two crash sites can reach it. execute() holds the same value in a local, +// but neither reporter runs where that local is in scope: reportFault is the +// deferred recover at the top of main.go, and guard's hook runs on a goroutine +// that never saw execute's frame at all. Both read it here. +// +// The zero value is a command that is not a session — version, help, doctor — +// and its fault still counts, carrying the mode the contract spells "other". +var currentTelemetrySession telemetrySession + +// telemetryBegin decides, from the command line alone, whether this +// invocation is a session worth counting, and spools its opening events. +// +// Only session commands emit: chat and resume are "chat", do, exec, run and +// `plan run` are "task", everything else emits NOTHING and sends NOTHING. +// The mode is read from os.Args here rather than handed down from each door +// because a door that had to remember it would be a door that could forget +// it — and the doors differ only in the word they call themselves. +// +// THE NOTICE: printed once per install, to stderr, before the first session's +// events are ever sent — but never into a --json stdout, and never into a +// pipe: a person who cannot see stderr is a person who cannot be asked, so +// the events wait in the spool until a session that can show the notice runs. +// A task command is always shown it, because a task runs unattended and its +// person may never open a chat at all. +func telemetryBegin() telemetrySession { + // Wired first: a fault can arrive from any goroutine the run spawns from + // here on, and the hook reads the session assigned below when it fires. + telemetryFaultHook() + args := os.Args[1:] + mode, resumed, session := telemetryMode(args) + if !session { + currentTelemetrySession = telemetrySession{} + return currentTelemetrySession + } + id := telemetrySessionID() + currentTelemetrySession = telemetrySession{mode: mode, resumed: resumed, sessionID: id} + // A run the ladder has turned off prints no notice either: the notice is + // the sentence that asks permission to send, and a pipe that will never + // send has nobody to ask — and marking it shown would create the very + // directory this run promised not to write. + if telemetry.Enabled() && !telemetry.NoticeShown() && !telemetryHasJSON(args) && + (mode == telemetry.ModeTask || stderrIsTerminal()) { + telemetry.PrintNotice() + telemetry.MarkNoticeShown() + } + // Both opening events go through SpoolSync, not the fire-and-forget Spool: + // first_run must be on disk before session_started even exists, and a run's + // session_started must be spooled before the process can reach the exit and + // flush — an append on its own goroutine can land after the flush has + // already renamed the spool, or not at all when a short run exits first. + // + // Each constructor is reached only when the ladder is on: the event's own + // identity mints and writes the install id, so building one under a run + // that will not send would touch the state root this run promised to leave + // alone — the same reason telemetryEnd gates its constructor. + if telemetry.Enabled() && telemetry.FirstRunPending() { + _ = telemetry.SpoolSync(telemetry.FirstRun(time.Now())) + telemetry.MarkFirstRunSent() + } + if telemetry.Enabled() { + _ = telemetry.SpoolSync(telemetry.SessionStarted(mode, resumed, id, time.Now())) + } + return currentTelemetrySession +} + +// telemetryFaultHook hands guard the reporter it cannot import. guard sits +// below internal/telemetry — the spool's own append runs on guard.Go — so the +// package that absorbs a goroutine's panic must not depend on the package that +// would count it. It calls a func value instead, and this is the one place the +// binary wires the two together. +// +// Spool, not SpoolSync: a guarded goroutine's fault does not end the process, +// so the event waits for the flush this run already makes at its exit and the +// faulting goroutine is not held for a write. The hook is set only when the +// ladder is on, which is the same answer the crash site in fault.go asks. +// +// The scope guard passes is its own label for the log — narrator, +// telemetry.spool — while the contract's scope is one of three words, and the +// word for every fault recovered here is "goroutine". +func telemetryFaultHook() { + if !telemetry.Enabled() { + return + } + guard.SetOnFault(func(scope string, stack []byte) { + session := currentTelemetrySession + telemetry.Spool(telemetry.FaultEvent(telemetry.Fault{ + Mode: string(session.mode), + Scope: telemetry.ScopeGoroutine, + Stack: stack, + }, session.sessionID, time.Now())) + }) +} + +// telemetryEnd closes one run at the exit, from the process's own tally +// (telemetry.Snapshot), the time it took, the exit code the run produced, and +// the stop reason the exit code speaks in the contract's vocabulary. +// +// The SessionEnded event goes through SpoolSync and then Flush with a context +// deadline of exactly one second — the whole of what a telemetry failure may +// ever add to a run. Both are no-ops when the ladder is off, which is what +// keeps this defer free for every non-session command: `codeaf version` runs +// it and it costs a mode read and nothing else. +func telemetryEnd(session telemetrySession, code int) { + // Nothing is built when the ladder is off, and that is load-bearing: the + // event's identity alone reads the install id, which mints and writes one + // on a machine that has never sent anything — the very file this run + // promised not to write. SpoolSync and Flush are no-ops under the same + // rung, so this gate is what keeps a disabled run from touching the state + // root at all. + if session.mode == "" || !telemetry.Enabled() { + return + } + // The run's own tally comes from Snapshot and only the three facts the + // counters cannot know — how long it took, why it ended, what it left + // with — are set on top of it. + stats := telemetry.Snapshot() + stats.Duration = time.Since(telemetryStarted) + stats.StopReason = telemetryStopReason(code) + stats.ExitCode = code + event := telemetry.SessionEnded(session.mode, stats, session.sessionID, time.Now()) + _ = telemetry.SpoolSync(event) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = telemetry.Flush(ctx) +} + +// telemetryInterrupted is whether the person ended the run themselves — +// ctrl-c on the surface, the signal a headless run caught. It is set where +// the signal is caught (do.go's errand), and read once, here, to give the +// session-ended event the contract's "interrupted" word instead of a failure +// word it was not. +var telemetryInterrupted bool + +// telemetryStarted is when this process began its session, the honest measure +// of a run's duration: the wall time from the first event to the exit, not a +// guess at when the model first answered. +var telemetryStarted = time.Now() + +// telemetrySessionID is the run id the events are hashed from, in preference +// order: the id trace minted at the door (chat, do, exec and `plan run` all +// call openDebugRecord, which calls trace.Begin), the process's own trace run +// where a door did not mint one, and a fresh random id where this process +// never minted a run at all. The package hashes whatever it is handed. +func telemetrySessionID() string { + if id := trace.RunFrom(context.Background()); id != "" { + return id + } + var raw [8]byte + if _, err := rand.Read(raw[:]); err != nil { + return "" + } + return hex.EncodeToString(raw[:]) +} + +// telemetryStopReasons maps the exit ladder's rungs (envelope.go) onto the +// contract's stop_reason vocabulary. It is a straight table keyed by the rung +// itself: the five codes the ladder names, each with the one word the exit +// cannot carry. exitLimit is the ladder's single rung for four limits — wall, +// budget, turn cap, price — and the contract gives the rung a single word, +// "budget", which is the one it names for a limit the person set. +var telemetryStopReasons = map[exitStatus]string{ + exitDone: telemetry.StopDone, + exitCannotRun: telemetry.StopError, + exitIncomplete: telemetry.StopIncomplete, + exitLimit: telemetry.StopBudget, + exitUnanswered: telemetry.StopQuestion, +} + +// telemetryStopReason maps the exit code onto the contract's stop_reason +// vocabulary. A code the ladder does not name is "unknown" — the honest +// answer, because the fact that the run ended survives even when the +// vocabulary does not. +// +// A user interrupt is not on the ladder: interrupting a chat is not a failure +// of the conversation, it is the person ending it, and the contract gives it +// its own word. The interrupt path is the one place this build asks a signal +// directly rather than the exit code, because the person's keypress and the +// process's exit are two events and only one of them names why. +func telemetryStopReason(code int) string { + if telemetryInterrupted { + return telemetry.StopInterrupted + } + if reason, ok := telemetryStopReasons[exitStatus(code)]; ok { + return reason + } + return telemetry.StopUnknown +} + +// stderrIsTerminal is whether a person can see the notice: stderr attached to +// a terminal. A redirected or piped stderr is the CI case, and CI is a rung +// of the ladder's own — but the notice's rule is narrower than that, because +// `codeaf do 2>/dev/null` in a person's own script is not CI and still must +// not spend the one line the person will never read. +func stderrIsTerminal() bool { + return stdinIsTerminal(os.Stderr) +} diff --git a/cmd/codeaf/telemetry_lifecycle_test.go b/cmd/codeaf/telemetry_lifecycle_test.go new file mode 100644 index 0000000000..d42f1682d1 --- /dev/null +++ b/cmd/codeaf/telemetry_lifecycle_test.go @@ -0,0 +1,323 @@ +package main + +// The execute() lifecycle, exercised the way the binary runs it: os.Args +// pointed at a real command, CODEAF_HOME in a temporary directory, and the +// endpoint at a dead loopback so a flush never reaches the real relay and the +// spool keeps what the run wrote for the assertions to read. +// +// The package is off under `go test` by design — the go-test rung is a +// production rule — and these tests do not weaken it. They use the one door +// past it, telemetry.EnableForTest, a func variable no environment variable +// or config value can reach. + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/telemetry" +) + +// telemetryLifecycleHome points the process at a temporary home and a dead +// endpoint, and turns the ladder on for this test only. The endpoint refuses +// every connection, so the flush a lifecycle call ends with fails locally +// instead of draining the spool — an endpoint that answered would send the +// events and remove them, and the assertions below would have nothing to read. +func telemetryLifecycleHome(t *testing.T) { + t.Helper() + t.Setenv(home.EnvVar, t.TempDir()) + // Nothing listens here, and loopback dialing is refused immediately. + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "http://127.0.0.1:1/telemetry") + telemetry.EnableForTest(t, true) +} + +// telemetrySpoolRows is what the spool holds, parsed. An absent spool is an +// empty slice, which is the assertion `codeaf version` makes. +func telemetrySpoolRows(t *testing.T) []map[string]any { + t.Helper() + body, err := os.ReadFile(telemetrySpoolPath(t)) + if err != nil { + return nil + } + var rows []map[string]any + for _, line := range strings.Split(strings.TrimSpace(string(body)), "\n") { + if line == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(line), &row); err != nil { + t.Fatalf("spool line is not JSON: %v", err) + } + rows = append(rows, row) + } + return rows +} + +// telemetrySpoolPath is the spool file inside the test's home. +func telemetrySpoolPath(t *testing.T) string { + t.Helper() + return filepath.Join(os.Getenv(home.EnvVar), "telemetry", "spool.jsonl") +} + +// telemetryEventNames is the event names the spool holds, in order. +func telemetryEventNames(t *testing.T, rows []map[string]any) []string { + t.Helper() + var names []string + for _, row := range rows { + names = append(names, row["event_name"].(string)) + } + return names +} + +// telemetryRowNamed is the one spooled row with an event name, failed on when +// there is none. A fresh home spools first_run before the session events, so +// the tests below find the row they are about by name rather than by index. +func telemetryRowNamed(t *testing.T, rows []map[string]any, name string) map[string]any { + t.Helper() + for _, row := range rows { + if row["event_name"] == name { + return row + } + } + t.Fatalf("no %s row in %v", name, telemetryEventNames(t, rows)) + return nil +} + +// telemetryProps is a spooled row's props object, failed on when it carries +// none. The wire row nests every contract property under "props"; the +// identity fields sit beside it. +func telemetryProps(t *testing.T, row map[string]any) map[string]any { + t.Helper() + props, ok := row["props"].(map[string]any) + if !ok { + t.Fatalf("%v row carries no props", row["event_name"]) + } + return props +} + +// telemetryArgs points os.Args at the given words for the duration of one +// lifecycle call, because telemetryBegin reads the command line itself. +func telemetryArgs(args ...string) func() { + previous := os.Args + os.Args = append([]string{"codeaf"}, args...) + return func() { os.Args = previous } +} + +// TestTelemetryTaskSessionSpoolsStartedAndEnded runs one session through the +// lifecycle's own begin and end and reads what it left in the spool: a fresh +// install spools first_run first, then the task session's session_started and +// session_ended, both with mode "task", and the ended event carries the exit +// code's stop reason. +func TestTelemetryTaskSessionSpoolsStartedAndEnded(t *testing.T) { + telemetryLifecycleHome(t) + restore := telemetryArgs("plan", "run", "p.json") + defer restore() + + session := telemetryBegin() + if session.mode != telemetry.ModeTask || session.resumed { + t.Fatalf("plan run: mode=%q resumed=%v, want task, false", session.mode, session.resumed) + } + if session.sessionID == "" { + t.Fatalf("plan run minted no session id") + } + telemetryEnd(session, 0) + + rows := telemetrySpoolRows(t) + names := telemetryEventNames(t, rows) + if len(names) != 3 || names[0] != "first_run" || names[1] != "session_started" || names[2] != "session_ended" { + t.Fatalf("spool holds %v, want [first_run session_started session_ended]", names) + } + startedRow := telemetryRowNamed(t, rows, "session_started") + endedRow := telemetryRowNamed(t, rows, "session_ended") + started := telemetryProps(t, startedRow) + ended := telemetryProps(t, endedRow) + if started["mode"] != "task" || ended["mode"] != "task" { + t.Fatalf("modes were %v and %v, want task", started["mode"], ended["mode"]) + } + if ended["stop_reason"] != "done" { + t.Fatalf("stop_reason was %v, want done for exit 0", ended["stop_reason"]) + } + if startedRow["session_id_hash"] != endedRow["session_id_hash"] { + t.Fatalf("started and ended hashed different session ids") + } +} + +// TestTelemetrySessionEndedCarriesTheSessionCounters: the end event is built +// from the process's own tally (telemetry.Snapshot), so a turn, a failed model +// call with its cost, and a tool call counted during the run reach the spooled +// session_ended line as the contract's bands. The counters are process-wide +// and other tests in this package drive model and tool calls through the +// doors that count, so the tally is zeroed first and the count is exactly the +// three calls made here. +func TestTelemetrySessionEndedCarriesTheSessionCounters(t *testing.T) { + telemetryLifecycleHome(t) + telemetry.ResetCountersForTest(t) + restore := telemetryArgs("do", "fix the bug") + defer restore() + + telemetry.CountTurn() + telemetry.CountModelCall(false, 0.02) + telemetry.CountToolCall(true) + + session := telemetryBegin() + telemetryEnd(session, 0) + + ended := telemetryProps(t, telemetryRowNamed(t, telemetrySpoolRows(t), "session_ended")) + want := map[string]string{ + "turns": "1", + "model_calls": "1", + "model_calls_failed": "1", + "tool_calls": "1", + "tool_calls_failed": "0", + "cost_usd": "0.01-0.1", + } + for key, value := range want { + if ended[key] != value { + t.Errorf("session_ended %s = %v, want %s", key, ended[key], value) + } + } +} + +// TestTelemetryModeIsReadFromTheCommandLine is the mode table: only session +// commands emit, chat and resume are chat, the task verbs are task, and +// nothing else is a session at all. +func TestTelemetryModeIsReadFromTheCommandLine(t *testing.T) { + cases := []struct { + args []string + mode telemetry.Mode + resumed bool + session bool + }{ + {[]string{"chat"}, telemetry.ModeChat, false, true}, + {[]string{"resume", "abc"}, telemetry.ModeChat, true, true}, + {[]string{"do", "fix the bug"}, telemetry.ModeTask, false, true}, + {[]string{"exec", "ls"}, telemetry.ModeTask, false, true}, + {[]string{"run", "p.json"}, telemetry.ModeTask, false, true}, + {[]string{"plan", "run", "p.json"}, telemetry.ModeTask, false, true}, + {[]string{"version"}, "", false, false}, + {[]string{"help"}, "", false, false}, + {[]string{"doctor"}, "", false, false}, + {[]string{"telemetry", "status"}, "", false, false}, + {[]string{"plan", "new"}, "", false, false}, + {[]string{"logs"}, "", false, false}, + } + for _, want := range cases { + mode, resumed, session := telemetryMode(want.args) + if mode != want.mode || resumed != want.resumed || session != want.session { + t.Errorf("%v: mode=%q resumed=%v session=%v, want %q %v %v", + want.args, mode, resumed, session, want.mode, want.resumed, want.session) + } + } +} + +// TestTelemetryVersionSpoolsNothingAndCreatesNoDirectory: a non-session +// command emits nothing and sends nothing, not even the directory the spool +// would live in. The lifecycle is driven directly rather than through +// execute(): the real command starts writers under the state root that +// outlive the test, and the tally under test is the telemetry's alone. +func TestTelemetryVersionSpoolsNothingAndCreatesNoDirectory(t *testing.T) { + telemetryLifecycleHome(t) + restore := telemetryArgs("version") + defer restore() + + session := telemetryBegin() + if session.mode != "" { + t.Fatalf("version was taken for a %q session", session.mode) + } + telemetryEnd(session, 0) + if rows := telemetrySpoolRows(t); len(rows) != 0 { + t.Fatalf("version spooled %v", telemetryEventNames(t, rows)) + } + if _, err := os.Stat(filepath.Join(os.Getenv(home.EnvVar), "telemetry")); !os.IsNotExist(err) { + t.Fatalf("version created a telemetry directory") + } +} + +// TestTelemetryOffWritesNothingAtAll: CODEAF_TELEMETRY=off is the ladder's +// first rung and nothing may be written for a session command either. The +// ladder is read honestly here — EnableForTest(false) — so the off rung is +// what stops the writes, not the test override. +func TestTelemetryOffWritesNothingAtAll(t *testing.T) { + telemetryLifecycleHome(t) + telemetry.EnableForTest(t, false) + t.Setenv("CODEAF_TELEMETRY", "off") + restore := telemetryArgs("do", "fix the bug") + defer restore() + + session := telemetryBegin() + telemetryEnd(session, 0) + if rows := telemetrySpoolRows(t); len(rows) != 0 { + t.Fatalf("%d events written with telemetry off: %v", len(rows), telemetryEventNames(t, rows)) + } + if _, err := os.Stat(filepath.Join(os.Getenv(home.EnvVar), "telemetry")); !os.IsNotExist(err) { + t.Fatalf("a telemetry directory was created with telemetry off") + } +} + +// TestTelemetryNoticePrintsOnceAcrossTwoInvocations: the notice is shown once +// per install. The first task session marks it shown and spools first_run +// once; the second session of the same install marks nothing and spools no +// second first_run. +func TestTelemetryNoticePrintsOnceAcrossTwoInvocations(t *testing.T) { + telemetryLifecycleHome(t) + restore := telemetryArgs("do", "fix the bug") + defer restore() + + if telemetry.NoticeShown() { + t.Fatalf("a fresh home read as notice-already-shown") + } + // A task command shows the notice even with piped stderr, because a task + // runs unattended and its person may never open a chat. + first := telemetryBegin() + if !telemetry.NoticeShown() { + t.Fatalf("the notice was not marked shown for a task session") + } + telemetryEnd(first, 0) + + rows := telemetrySpoolRows(t) + firstRuns := 0 + for _, row := range rows { + if row["event_name"] == "first_run" { + firstRuns++ + } + } + if firstRuns != 1 { + t.Fatalf("%d first_run events for one install, want 1", firstRuns) + } + + second := telemetryBegin() + telemetryEnd(second, 0) + // The first session's first_run is still in the spool — the flush could not + // send and kept every line — so the check is that the second invocation + // added none: one first_run for the install across both invocations. + firstRuns = 0 + for _, row := range telemetrySpoolRows(t) { + if row["event_name"] == "first_run" { + firstRuns++ + } + } + if firstRuns != 1 { + t.Fatalf("two invocations spooled %d first_run events, want 1", firstRuns) + } +} + +// TestTelemetryNoticeNeverPrintsUnderJSON: a --json stdout must stay +// machine-clean, so a --json session never shows the notice; the events wait +// in the spool until a session that can show it runs. +func TestTelemetryNoticeNeverPrintsUnderJSON(t *testing.T) { + telemetryLifecycleHome(t) + restore := telemetryArgs("do", "--json", "fix the bug") + defer restore() + + if !telemetryHasJSON(os.Args[1:]) { + t.Fatalf("telemetryHasJSON missed --json") + } + session := telemetryBegin() + telemetryEnd(session, 0) + if telemetry.NoticeShown() { + t.Fatalf("the notice was marked shown under --json") + } +} diff --git a/cmd/codeaf/telemetry_test.go b/cmd/codeaf/telemetry_test.go new file mode 100644 index 0000000000..7563439dd9 --- /dev/null +++ b/cmd/codeaf/telemetry_test.go @@ -0,0 +1,167 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/home" +) + +// telemetryHome is the clean room every telemetry-verb test runs in: a +// throwaway CODEAF_HOME and a local httptest sink pinned as the endpoint, so +// nothing a test spools can ever leave the machine (the package's own spool +// tests POST to the production relay when the endpoint is unset). +func telemetryHome(t *testing.T) string { + t.Helper() + root := t.TempDir() + t.Setenv(home.EnvVar, root) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "") + t.Setenv("CODEAF_TELEMETRY", "") + t.Setenv("DO_NOT_TRACK", "") + return root +} + +// telemetrySink pins the endpoint at a local server that answers 2xx and +// swallows everything. +func telemetrySink(t *testing.T) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", server.URL) + return server +} + +func TestTelemetryStatusReportsOn(t *testing.T) { + telemetryHome(t) + telemetrySink(t) + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry([]string{"status"}); err != nil { + t.Fatal(err) + } + got := usageOut.(*strings.Builder).String() + if !strings.HasPrefix(got, "telemetry ") { + t.Fatalf("status should open with the state, got:\n%s", got) + } + if !strings.Contains(got, "endpoint") { + t.Fatalf("status should name the endpoint, got:\n%s", got) + } + if !strings.Contains(got, "install") { + t.Fatalf("status should name the install prefix, got:\n%s", got) + } +} + +func TestTelemetryStatusNamesTheReasonWhenOff(t *testing.T) { + telemetryHome(t) + telemetrySink(t) + t.Setenv("CODEAF_TELEMETRY", "off") + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry([]string{"status"}); err != nil { + t.Fatal(err) + } + got := usageOut.(*strings.Builder).String() + if !strings.Contains(got, "telemetry off") { + t.Fatalf("status should report off, got:\n%s", got) + } + if !strings.Contains(got, "CODEAF_TELEMETRY") { + t.Fatalf("off should say why, got:\n%s", got) + } +} + +func TestTelemetryShowPrintsTheSpool(t *testing.T) { + telemetryHome(t) + telemetrySink(t) + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry([]string{"show"}); err != nil { + t.Fatal(err) + } + got := usageOut.(*strings.Builder).String() + if strings.TrimSpace(got) == "" { + t.Fatal("show printed nothing") + } +} + +func TestTelemetryOffWritesTheSetting(t *testing.T) { + root := telemetryHome(t) + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry([]string{"off"}); err != nil { + t.Fatal(err) + } + if !strings.Contains(usageOut.(*strings.Builder).String(), "off") { + t.Fatal("off should confirm with one line") + } + data, err := os.ReadFile(filepath.Join(root, "config.json")) + if err != nil { + t.Fatalf("the profile config should exist: %v", err) + } + if !strings.Contains(string(data), "telemetry") { + t.Fatalf("the config should carry the telemetry row, got:\n%s", data) + } +} + +func TestTelemetryOnUndoesOff(t *testing.T) { + root := telemetryHome(t) + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry([]string{"off"}); err != nil { + t.Fatal(err) + } + if err := runTelemetry([]string{"on"}); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(root, "config.json")) + if err != nil { + t.Fatalf("the profile config should exist: %v", err) + } + if !strings.Contains(string(data), `"telemetry": true`) { + t.Fatalf("on should write the row back as true, got:\n%s", data) + } +} + +func TestTelemetryBareDefaultsToStatus(t *testing.T) { + telemetryHome(t) + telemetrySink(t) + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry(nil); err != nil { + t.Fatal(err) + } + got := usageOut.(*strings.Builder).String() + if !strings.HasPrefix(got, "telemetry ") { + t.Fatal("a bare `telemetry` should answer with the status") + } +} + +func TestTelemetryUnknownVerbIsRefused(t *testing.T) { + telemetryHome(t) + if err := runTelemetry([]string{"nonsense"}); err == nil { + t.Fatal("an unknown verb should be an error") + } +} + +func TestTelemetryIsInUsageAndEnvironmentText(t *testing.T) { + telemetryHome(t) + for name, text := range map[string]string{ + "usage": usageText, + "environment": environmentText, + } { + if !strings.Contains(text, "telemetry") { + t.Errorf("%s text should name the telemetry command", name) + } + } + if !strings.Contains(environmentText, "CODEAF_TELEMETRY_ENDPOINT") { + t.Error("the environment table should name CODEAF_TELEMETRY_ENDPOINT") + } + if !strings.Contains(environmentText, "DO_NOT_TRACK") { + t.Error("the environment table should name DO_NOT_TRACK") + } +} diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md new file mode 100644 index 0000000000..c0d7faf69d --- /dev/null +++ b/docs/TELEMETRY.md @@ -0,0 +1,124 @@ +# Telemetry + +codeaf counts how it is used — how often, in which modes, on which platforms — +so the parts people rely on get the work. The counts are anonymous: nothing +about you or your work ever leaves this machine. + +## The notice + +Before the first session's events are sent, codeaf prints this to stderr once; +the installer prints it too: + +``` +codeaf sends anonymous usage counts to AgentField. + Sent: version, OS, mode (chat or task), how many sessions, how many errors. + Never: anything about you or your work. No prompts, code, file names, + paths, repo names, keys, email, IP, or machine name. + See exactly what leaves: codeaf telemetry show + Turn off: CODEAF_TELEMETRY=off +``` + +## What is sent + +Exactly four events. Each carries the every-event properties; three of them +add more. Every value is a count, a band, or a word from a fixed list. The +table is generated from the same allowlist the code is held to, and a test +fails the build if the two ever drift apart. + +| Event | Property | What it is | +| --- | --- | --- | +| every event | codeaf_version | the release tag this binary was built from, at most 64 characters | +| every event | channel | stable, rc, staging, dev, or unknown | +| every event | os | darwin, linux, windows, or other | +| every event | arch | amd64, arm64, or other | +| every event | usage_context | local (a person's machine), ci, or container | +| every event | install_method | script, source, or unknown | +| session_started | mode | chat or task | +| session_started | resumed | whether the session continued an earlier one | +| session_ended | mode | chat or task | +| session_ended | duration | a band: under 1m, 1-5m, 5-30m, 30m-2h, 2h or more | +| session_ended | turns | a count band | +| session_ended | model_calls | a count band | +| session_ended | model_calls_failed | a count band | +| session_ended | tool_calls | a count band | +| session_ended | tool_calls_failed | a count band | +| session_ended | cost_usd | a dollar band | +| session_ended | stop_reason | done, error, incomplete, budget, turn-cap, deadline, price, question, interrupted, or unknown | +| session_ended | exit_code | 0 to 5 | +| fault | mode | chat, task, or other | +| fault | scope | main, goroutine, or surface | +| fault | fingerprint | 16 hex characters hashed from codeaf function names in the stack | + +Count bands are 0, 1, 2-5, 6-20, 21-100 and 100+. Dollar bands are 0, under +0.01, 0.01-0.1, 0.1-1, 1-10 and 10+. + +first_run carries only the every-event properties and is sent once per +install. + +Each event also carries its identity as hashes: a random per-event id, sha256 +of the install id, and — except on first_run — sha256 of the run id. The raw +ids never leave this machine. + +## What is never sent + +Prompts, model replies, code, file names, paths, repo or directory names, git +remotes, hostnames, usernames, IP addresses, environment values, API keys, +email addresses, model names, error text, panic messages. A test builds every +event from inputs stuffed with exactly these and fails if any of them reach +the marshalled output. + +## Where events wait + +Events wait in ~/.codeaf/telemetry/spool.jsonl until they are sent: at most 50 +per request, nothing older than 7 days, at most 1000 lines kept, and nothing +sent before the notice has been shown. `codeaf telemetry show` prints exactly +what has not left yet. + +## Turning it off + +Any one of these turns the counts off. They are checked in this order: + +1. `CODEAF_TELEMETRY=off` — also `0` or `false`. +2. `DO_NOT_TRACK=1` — also `true`, the ecosystem's own word for it. +3. `telemetry = off` in the project's settings file, `.codeaf/config.json`. A + project may only turn the counts off, never on. +4. `codeaf telemetry off`, which writes the profile setting; `codeaf telemetry + on` is the way back. +5. an empty `CODEAF_TELEMETRY_ENDPOINT`. + +A build that cannot name its own source — dirty or unstamped — never reports, +and neither does a test binary. + +## The command + +`codeaf telemetry` reads the counts and never sends anything of its own. + +- `codeaf telemetry status` says whether the counts are on, and why not when + they are off. +- `codeaf telemetry show` prints exactly what is waiting to leave the machine. +- `codeaf telemetry off` and `codeaf telemetry on` write the profile setting. + +``` +telemetry on + endpoint: https://agentfield.ai/api/oss/codeaf/telemetry + notice: shown + spooled events: 3 + install: 4f2a9c1b7e08… +``` + +When the counts are off, a `reason:` line follows `telemetry off` and names the +switch that turned them off. + +## Download counts + +The download numbers this repository reports come from public GitHub release +data: the GitHub API publishes a cumulative download count for every asset on +a release, and a daily scheduled workflow reads those counts and sends one +event per binary asset to PostHog (`codeaf:release_downloads`). No code in +the binary is involved, nothing is collected from the person downloading, and +the only facts in the event are the release tag, the asset's platform, and +the count GitHub already shows on the release page. Sidecar files such as +`checksums.txt` are not counted. The workflow lives in +`.github/workflows/release-downloads.yml`; the script behind it is +`scripts/release_downloads.py`, and running it with `--dry-run` prints the +batch it would send. diff --git a/docs/changes/unreleased/1095-anonymous-usage-counts.md b/docs/changes/unreleased/1095-anonymous-usage-counts.md new file mode 100644 index 0000000000..d60cc2034c --- /dev/null +++ b/docs/changes/unreleased/1095-anonymous-usage-counts.md @@ -0,0 +1,16 @@ +--- +kind: added +title: Anonymous usage counts, on by default with a notice printed once +pr: 1095 +surface: [docs] +invalidates: [] +--- + +codeaf now sends anonymous usage counts — version, OS, mode (chat or task), and +banded session and error counts — so the parts people rely on get the work. It +never sends prompts, code, paths, names, or anything that identifies you. The +counts are on by default, and the first session prints a notice naming exactly +what leaves; `codeaf telemetry show` prints what is waiting to leave right now. +Turn it off with `CODEAF_TELEMETRY=off`, `DO_NOT_TRACK=1`, `codeaf telemetry +off`, `telemetry = off` in a project's `.codeaf/config.json`, or an empty +`CODEAF_TELEMETRY_ENDPOINT`. The whole contract is `docs/TELEMETRY.md`. diff --git a/internal/buildinfo/buildinfo.go b/internal/buildinfo/buildinfo.go index 0b7be62272..914f77d7ba 100644 --- a/internal/buildinfo/buildinfo.go +++ b/internal/buildinfo/buildinfo.go @@ -180,3 +180,9 @@ func newRunningDetector() *Detector { func StaleNotice() string { return running.Notice() } + +// Dirty reports whether the running binary was built from a modified tree, +// read from the same build settings Identity derives from. Callers that need +// the build's honesty — telemetry's opt-out ladder — ask here rather than +// parsing a display string. +func Dirty() bool { return current.Dirty } diff --git a/internal/buildinfo/buildinfo_test.go b/internal/buildinfo/buildinfo_test.go index 796db871fd..cf08e7889a 100644 --- a/internal/buildinfo/buildinfo_test.go +++ b/internal/buildinfo/buildinfo_test.go @@ -140,3 +140,19 @@ func TestBuildIdentityDistinguishesUnstampedRebuilds(t *testing.T) { t.Fatal("an unstamped rebuild retained the prior engine identity") } } + +// Dirty answers from the build settings, not from parsing Identity's display +// string: a dirty build and a clean one can spell the same Identity prefix, +// so the flag must be read where it is stored. +func TestDirtyReadsTheStoredFlag(t *testing.T) { + previous := current + t.Cleanup(func() { current = previous }) + current = Info{Revision: "dirty-revision", Dirty: true, BuiltAt: time.Date(2026, 9, 8, 20, 0, 1, 0, time.UTC)} + if !Dirty() { + t.Fatal("a dirty build answered Dirty() false") + } + current = Info{Revision: "clean-revision", Dirty: false, BuiltAt: time.Date(2026, 9, 8, 20, 0, 1, 0, time.UTC)} + if Dirty() { + t.Fatal("a clean build answered Dirty() true") + } +} diff --git a/internal/config/derivation_test.go b/internal/config/derivation_test.go index 8f1e3c9268..de53521e8a 100644 --- a/internal/config/derivation_test.go +++ b/internal/config/derivation_test.go @@ -147,6 +147,12 @@ var settingReaders = map[string]string{ // is the one thing this table exists to prevent. KeyHistoryEnabled: "KeyHistoryEnabled", KeyDraftPersist: "KeyDraftPersist", + // The telemetry row is answered by the package the events come from: + // internal/telemetry reads the CODEAF_TELEMETRY pin itself (its off + // state), so the row names the pin the far side touches — the same + // spelling [KeyTenureAfter] uses for its pin. cmd/codeaf's door will + // tighten this to its own accessor when it wires the package in. + KeyTelemetry: "CODEAF_TELEMETRY", // The v3 session's rows name what READS the value on the far side, which // for these six is not a function in this package: the two approval rows // become the policy hung off session.Config.ApprovalPolicy, the ceiling is diff --git a/internal/config/projectconfig.go b/internal/config/projectconfig.go index 684c12e4fa..6904543a46 100644 --- a/internal/config/projectconfig.go +++ b/internal/config/projectconfig.go @@ -94,6 +94,7 @@ var ProjectKeys = []string{ KeySpendRail, KeyHistoryEnabled, KeyDraftPersist, + KeyTelemetry, } // ProjectKeyAllowed reports whether a row may live in a project file. @@ -313,6 +314,8 @@ func (p ProjectConfig) ResolveBool(profileDir, key string) (bool, error) { name = "CODEAF_HISTORY" case KeyDraftPersist: name = "CODEAF_DRAFT_PERSIST" + case KeyTelemetry: + name = "CODEAF_TELEMETRY" default: if !ProjectKeyAllowed(key) { return false, fmt.Errorf("%q is not a project-local settings row", key) @@ -332,6 +335,9 @@ func (p ProjectConfig) ResolveBool(profileDir, key string) (bool, error) { if key == KeyHistoryEnabled { return HistoryEnabledAt(profileDir), nil } + if key == KeyTelemetry { + return TelemetryAt(profileDir), nil + } return DraftPersistAt(profileDir), nil } diff --git a/internal/config/settings.go b/internal/config/settings.go index 48255ce915..3ea16e3322 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -124,6 +124,13 @@ const ( KeyHistoryEnabled = "history.enabled" KeyDraftPersist = "draft.persist" + // KeyTelemetry is the anonymous-usage switch: one row, default on, off + // turns the whole pipe (see internal/telemetry). It sits beside the + // history row because they answer the same question at different + // depths — "may codeaf record what happened on this machine" — and + // the session counters count either way: only the sending asks. + KeyTelemetry = "telemetry" + // The v3 session's own keys. They are DOTTED where the older ones are // snake_case because they name a path into a settings tree the file writer // will eventually hold — tools.approval is a map, models.roles is a map — @@ -1073,6 +1080,14 @@ var OperatorEnvPins = []string{ // somebody should have to make on purpose, in a shell, for one run. "CODEAF_CALL_LOG", "CODEAF_CALL_LOG_BODIES", + // CODEAF_TELEMETRY_ENDPOINT points the anonymous usage events at some + // other collector (a sink, an "empty" box, a CI stub). It is plumbing + // for the reason CODEAF_BASE_URL is — it is an address — and the + // reason the bodies pin is: a sheet row that could be written from a + // project file would be a repository able to say where a machine's + // usage counts are sent, which is not a preference any team should + // hold over a person's own telemetry off switch. + "CODEAF_TELEMETRY_ENDPOINT", // THE DEBUG RECORD'S SWITCH AND ITS TWO CEILINGS (internal/trace). // CODEAF_DEBUG turns on the full record of a run — the bodies of its model // calls, its tool calls and the choices it made, in one folder per run under @@ -1267,6 +1282,13 @@ const ( // internal/history caps it and never sends it anywhere. DefaultHistoryEnabled = true + // DefaultTelemetry is on, because the events are coarse counts the + // contract allows (docs/TELEMETRY.md) and the notice names them before + // the first byte leaves the machine. One row turns the pipe off, and + // DO_NOT_TRACK answers the same question for anyone who arrives with + // the ecosystem's own word for it. + DefaultTelemetry = true + // DefaultDraftPersist keeps the unsent sentence across a restart, for the // reason a text field in any other application does: the draft is the // PERSON's, not the session's, and losing it to a crash or a closed window @@ -2461,6 +2483,16 @@ func (s *Settings) build() []Setting { read: func() string { return formatBool(HistoryEnabledAt(dir)) }, write: func(raw string) error { return writeBool(dir, KeyHistoryEnabled, raw) }, }, + Setting{ + Key: KeyTelemetry, Category: CategoryInterface, Kind: SettingBool, + Label: "telemetry", Env: "CODEAF_TELEMETRY", + Hint: "sends the anonymous usage counts described in docs/TELEMETRY.md — session " + + "starts and ends, tool and model call counts, coarse cost — after a notice " + + "has been printed once. Off sends nothing. The session's own counters still " + + "count, because counting is free; a change lands the next time codeaf starts.", + read: func() string { return formatBool(TelemetryAt(dir)) }, + write: func(raw string) error { return writeBool(dir, KeyTelemetry, raw) }, + }, Setting{ Key: KeyDraftPersist, Category: CategoryInterface, Kind: SettingBool, Label: "keep drafts", Env: "CODEAF_DRAFT_PERSIST", @@ -4309,6 +4341,76 @@ func writeText(profileDir, key, raw string) error { return writeProfileValue(profileDir, key, strings.TrimSpace(raw)) } +// TelemetryAt resolves whether the anonymous-usage pipe is on, default on. +// The environment pin wins over the project file over the profile config over +// the built-in default — the same ladder every other on/off row walks, with +// the working directory's own file between the pin and the profile because a +// repository may answer for itself what a machine answers for everybody +// (see ProjectKeys). A pin or file that will not parse reads as the default +// rather than refusing a launch over a count nobody can see. +func TelemetryAt(profileDir string) bool { + return TelemetryAtIn("", profileDir) +} + +// TelemetryAtIn is [TelemetryAt] with a working directory whose project file +// may hold the answer. An empty cwd has no project layer, and a project file +// that cannot be loaded is skipped for the same reason a malformed one is — +// the row reads as if nobody had written it. +func TelemetryAtIn(cwd, profileDir string) bool { + if raw := strings.TrimSpace(env.Get("CODEAF_TELEMETRY")); raw != "" { + if value, err := parseBool(raw); err == nil { + return value + } + return DefaultTelemetry + } + // THE PIN IS THE OPERATOR SPEAKING, so it lands before the project file. + // A pin that says nothing stays out of the project layer's way, which is + // the same order [ProjectConfig.ResolveBool] walks for the history row. + if cwd != "" { + // ProjectBoolAt resolves through the project file first, then the + // profile reader — which is TelemetryAt, on the rung below the pin + // it was about to read anyway. One ladder, not two. + if value, err := ProjectBoolAt(cwd, profileDir, KeyTelemetry); err == nil && !value { + return false + } + } + if value, ok := persistedBool(profileDir, KeyTelemetry); ok { + return value + } + return DefaultTelemetry +} + +// WriteTelemetry persists the person's own answer to the telemetry row — +// the writer `codeaf telemetry on|off` goes through, so the command and the +// settings sheet write the same file the same way and cannot drift. +func WriteTelemetry(profileDir string, on bool) error { + return writeProfileValue(profileDir, KeyTelemetry, on) +} + +// TelemetryOffReason answers what turned the pipe off, in the words the +// telemetry command prints beside the off reading: the environment pin, the +// project file, the profile config, or nothing at all when the answer is on. +// It is the accessor the binary calls to learn “did config turn telemetry +// off”, and the reason is returned with it because a switch that went quiet +// without saying why is a row nobody can audit. +func TelemetryOffReason(cwd, profileDir string) (off bool, reason string) { + if raw := strings.TrimSpace(env.Get("CODEAF_TELEMETRY")); raw != "" { + if value, err := parseBool(raw); err == nil && !value { + return true, "CODEAF_TELEMETRY is set to " + raw + } + return DefaultTelemetry == false, "" + } + if cwd != "" { + if value, err := ProjectBoolAt(cwd, profileDir, KeyTelemetry); err == nil && !value { + return true, "the project config turns telemetry off" + } + } + if value, ok := persistedBool(profileDir, KeyTelemetry); ok && !value { + return true, "the profile config turns telemetry off" + } + return false, "" +} + // ContextFillAt resolves the fill law: environment pin, then the persisted // row, then the package default. A malformed pin reads as the default. func ContextFillAt(profileDir string) int { diff --git a/internal/config/telemetry_test.go b/internal/config/telemetry_test.go new file mode 100644 index 0000000000..adc1406018 --- /dev/null +++ b/internal/config/telemetry_test.go @@ -0,0 +1,137 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +// The telemetry row is registered the way the call-log and history rows are: +// a key, a default, an environment pin, a place in the sheet, and a reader +// the binary can call. These tests hold that shape still, because the +// completeness gate in settings_test.go only fails when a pin is MISSING from +// the registry — it says nothing about a row that reads the wrong value. + +func writeTelemetryProfile(t *testing.T, dir string, on bool) { + t.Helper() + if err := writeBool(dir, KeyTelemetry, formatBool(on)); err != nil { + t.Fatalf("writeBool: %v", err) + } +} + +func TestTelemetryRowIsRegisteredLikeItsNeighbours(t *testing.T) { + dir := t.TempDir() + t.Setenv("CODEAF_TELEMETRY", "") + row, found := registry(t, dir).Row(KeyTelemetry) + if !found { + t.Fatal("the telemetry row is not registered") + } + if row.Key != KeyTelemetry || row.Label != "telemetry" || row.Env != "CODEAF_TELEMETRY" { + t.Errorf("telemetry row = %+v", row) + } + if got := row.Value(); got != "on" { + t.Errorf("an unset row reads %q, want on (the default)", got) + } + if row.Accepts() != "on or off" { + t.Errorf("the row accepts %q, want on or off", row.Accepts()) + } + + // The row writes, and the write lands where the reader looks. + if err := row.Apply("off"); err != nil { + t.Fatalf("Apply(off): %v", err) + } + if got := TelemetryAt(dir); got { + t.Fatal("after Apply(off) the row still reads on") + } + + // A pinned row refuses calmly, the way every other pinned row does. + t.Setenv("CODEAF_TELEMETRY", "off") + if err := row.Apply("on"); err == nil { + t.Fatal("a pinned row accepted a write") + } + if _, reason := TelemetryOffReason("", dir); reason == "" { + t.Fatal("the off reason does not name the pin") + } +} + +func TestTelemetryAtWalksTheLadder(t *testing.T) { + dir := t.TempDir() + t.Setenv("CODEAF_TELEMETRY", "") + + if !TelemetryAt(dir) { + t.Fatal("default is on") + } + + writeTelemetryProfile(t, dir, false) + if TelemetryAt(dir) { + t.Fatal("the profile config did not turn telemetry off") + } + if off, reason := TelemetryOffReason("", dir); !off || reason == "" { + t.Fatalf("off = %v, reason = %q, want the profile config named", off, reason) + } + + // The environment wins over the profile. + t.Setenv("CODEAF_TELEMETRY", "on") + if !TelemetryAt(dir) { + t.Fatal("the pin did not win over the profile config") + } + if off, _ := TelemetryOffReason("", dir); off { + t.Fatal("an on pin reports itself off") + } +} + +func TestTelemetryAtReadsTheProjectConfig(t *testing.T) { + dir := t.TempDir() + work := t.TempDir() + t.Setenv("CODEAF_TELEMETRY", "") + + if err := os.MkdirAll(work, 0o755); err != nil { + t.Fatal(err) + } + path := ProjectConfigPath(work) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(`{"telemetry": false}`), 0o644); err != nil { + t.Fatal(err) + } + + if TelemetryAtIn(work, dir) { + t.Fatal("the project config did not turn telemetry off") + } + if off, reason := TelemetryOffReason(work, dir); !off || reason == "" { + t.Fatalf("off = %v, reason = %q, want the project file named", off, reason) + } + + // The pin outranks the project file, and a profile off behind an + // unpinned project off is still off. + t.Setenv("CODEAF_TELEMETRY", "on") + if !TelemetryAtIn(work, dir) { + t.Fatal("the pin did not outrank the project file") + } + t.Setenv("CODEAF_TELEMETRY", "") + writeTelemetryProfile(t, dir, true) + if TelemetryAtIn(work, dir) { + t.Fatal("a profile on did not restore telemetry under a project off") + } + + // A malformed value is nobody's answer, and the row reads as if the + // line were not there. + if err := os.WriteFile(path, []byte(`{"telemetry": "maybe"}`), 0o644); err != nil { + t.Fatal(err) + } + if !TelemetryAtIn(work, dir) { + t.Fatal("a malformed project value turned telemetry off") + } +} + +// The counters-whether-or-not rule's config half: the accessor answers +// whether the pipe is off without ever claiming the counters stopped. +func TestTelemetryOffReasonAnswersNothingWhenOn(t *testing.T) { + dir := t.TempDir() + t.Setenv("CODEAF_TELEMETRY", "") + writeTelemetryProfile(t, dir, true) + if off, reason := TelemetryOffReason("", dir); off || reason != "" { + t.Fatalf("off = %v, reason = %q, want both empty", off, reason) + } +} diff --git a/internal/guard/guard.go b/internal/guard/guard.go index 43c8b8852c..38cc456865 100644 --- a/internal/guard/guard.go +++ b/internal/guard/guard.go @@ -13,6 +13,7 @@ import ( "log" "runtime/debug" "strings" + "sync/atomic" ) // Fault is what a recovered panic becomes: an ordinary error the caller can @@ -53,12 +54,45 @@ func note(scope string, recovered any, stack []byte) error { return fault } +// The fault hook is called from Recover once the fault has been recorded, with the +// scope Recover was given and the stack it captured for the log. It is nil by +// default, and it is a variable rather than a call because of where this +// package sits: internal/telemetry spools its own writes through guard.Go, so +// an import from here back to anything that would report a fault is a cycle. +// The reporting arrives as a func value instead, set once by the top of the +// binary and left nil by every package that has nothing to report to. +// +// It runs on the recovered goroutine, inside the deferred Recover, so it must +// neither block nor panic: the goroutine ends the moment it returns, and a +// fault in the reporter would be the one fault guard could not absorb. +type faultHook func(scope string, stack []byte) + +// onFault holds the hook behind an atomic pointer: the binary sets it once at +// start-up and every recovered goroutine reads it, and a plain variable read +// on one goroutine while another writes it is a race the detector rightly +// reports. +var onFault atomic.Pointer[faultHook] + +// SetOnFault installs the hook, or removes it with nil. +func SetOnFault(fn func(scope string, stack []byte)) { + if fn == nil { + onFault.Store(nil) + return + } + hook := faultHook(fn) + onFault.Store(&hook) +} + // Recover absorbs a panic in the deferring goroutine and lets it end quietly: // `defer guard.Recover("narrator")`. Use it where there is no result to carry // the fault — a fire-and-forget spawn whose only obligation is not to crash. func Recover(scope string) { if recovered := recover(); recovered != nil { - _ = note(scope, recovered, debug.Stack()) + stack := debug.Stack() + _ = note(scope, recovered, stack) + if hook := onFault.Load(); hook != nil { + (*hook)(scope, stack) + } } } diff --git a/internal/guard/guard_test.go b/internal/guard/guard_test.go index eeecb2bd88..5d0e36e0c4 100644 --- a/internal/guard/guard_test.go +++ b/internal/guard/guard_test.go @@ -128,3 +128,46 @@ func TestFaultWithoutScopeStillReads(t *testing.T) { t.Fatalf("unexpected text: %s", err.Error()) } } + +// TestGoCallsTheFaultHookOnceWithTheGoroutineStack holds the hook's whole +// contract: a guarded goroutine that panics calls the fault hook exactly once, with +// the scope Recover was given and the stack of the goroutine that faulted — +// the same bytes the log line carries, because a reporter above this package +// has nothing else to group the fault by. +func TestGoCallsTheFaultHookOnceWithTheGoroutineStack(t *testing.T) { + captureLog(t) + t.Cleanup(func() { SetOnFault(nil) }) + + type hookCall struct { + scope string + stack []byte + } + // Buffered for two so a second call is a readable failure rather than a + // goroutine blocked on a send nobody drains. + calls := make(chan hookCall, 2) + SetOnFault(func(scope string, stack []byte) { calls <- hookCall{scope: scope, stack: stack} }) + + Go("narrator", func() { panic("narration blew up") }) + + var got hookCall + select { + case got = <-calls: + case <-time.After(5 * time.Second): + t.Fatal("the fault hook was never called") + } + if got.scope != "narrator" { + t.Fatalf("hook scope = %q, want narrator", got.scope) + } + if len(got.stack) == 0 { + t.Fatal("the hook was called with an empty stack") + } + if !strings.Contains(string(got.stack), "guard_test.go") { + t.Fatalf("the stack is not the recovered goroutine's: %q", got.stack) + } + + select { + case second := <-calls: + t.Fatalf("the hook was called twice, second scope %q", second.scope) + case <-time.After(50 * time.Millisecond): + } +} diff --git a/internal/manual/chat/running-from-the-terminal.md b/internal/manual/chat/running-from-the-terminal.md index cb6bc70c71..6cda78fde4 100644 --- a/internal/manual/chat/running-from-the-terminal.md +++ b/internal/manual/chat/running-from-the-terminal.md @@ -613,7 +613,7 @@ as a command that broke. ## Which of these cost money, and which need no API key -**These read, need no key and spend nothing**: `why`, `notebook`, `competence`, `services` +**These read, need no key and spend nothing**: `why`, `telemetry`, `notebook`, `competence`, `services` (listing), `doctor`, `logs`, `cache`, `show`, `manual`, `version` and `--help`. They are safe in a shell prompt, a CI step or a bug report. @@ -642,6 +642,16 @@ first — `cache clean` wants the word `now` typed out, the same word `/cache cl wants in the chat, and `rebuild` wants `y` — and `--yes` skips the question on both. The other three act at once, and all three can be undone: a retracted belief restores, a stopped service starts again, a revoked device pairs again. +## What does it count about a run — the anonymous usage counts, and `codeaf telemetry` + +`codeaf telemetry` is the door onto the anonymous usage counts: `status` says whether +they are on and why not when they are off, `show` prints exactly what is waiting to +leave the machine, and `off` and `on` write the answer to your profile. It reads and +sends nothing of its own — it is a command about the counts, not a session. The +notice the first session prints names the bargain before the first byte leaves, and +`CODEAF_TELEMETRY=off` or `DO_NOT_TRACK=1` turns the counts off entirely. See +docs/TELEMETRY.md for the whole contract. + ## Reading a plan by hand — codeaf plan new, show, revise and run `codeaf plan` writes a plan to a file you can read, edit and diff, then runs it exactly as diff --git a/internal/provider/calllog.go b/internal/provider/calllog.go index 393998aeb6..7598841d83 100644 --- a/internal/provider/calllog.go +++ b/internal/provider/calllog.go @@ -10,6 +10,7 @@ import ( "github.com/Agent-Field/agentfield/sdk/go/ai" "github.com/Agent-Field/codeaf/internal/calllog" + "github.com/Agent-Field/codeaf/internal/telemetry" "github.com/Agent-Field/codeaf/internal/trace" ) @@ -510,6 +511,15 @@ func (c *Client) record(facts recordFacts) { record.Ended = facts.ended facts.knobs.trace.track(facts, record.ID) calllog.Append(record) + // THE TALLY SITS AT THE SAME DOOR AS THE LOG, for the reason the log does: + // every outbound call in the process passes through here, so a call cannot + // be missed by a caller who forgot to count one. Only the row that ENDS an + // attempt is an answered call: the start row says it is in flight, and + // counting that too would count every call twice. The record's own error + // sentence is the failure verdict and its cost the spend, 0 when absent. + if record.Phase != calllog.PhaseStart { + telemetry.CountModelCall(record.Error == "", record.Cost) + } c.recordBodies(facts, record, model) } diff --git a/internal/session/loop.go b/internal/session/loop.go index 3a1690bee0..27a53fd9ef 100644 --- a/internal/session/loop.go +++ b/internal/session/loop.go @@ -92,6 +92,7 @@ import ( "github.com/Agent-Field/codeaf/internal/roles" "github.com/Agent-Field/codeaf/internal/store" "github.com/Agent-Field/codeaf/internal/taxonomy" + "github.com/Agent-Field/codeaf/internal/telemetry" ) // ── retry constants (pi spec §4, verbatim from internal/exec/bare) ────────── @@ -1621,6 +1622,9 @@ func (a *Agent) keepSteeredPartial(partial *partialBuffer, reasoning *reasoningB // (see [sessionFile.writeLine]) — and a turn that spent nothing writes no line // at all (see [sessionFile.appendUsage]). func (a *Agent) sealTurn(turn Usage, started time.Time, model string) Usage { + // The tally sits at the seal because the seal is the shape of a turn: every + // turn that seals counts one, whether or not telemetry is sent later. + telemetry.CountTurn() turn.Duration = time.Since(started) a.mu.Lock() a.usage.Duration += turn.Duration @@ -3219,6 +3223,13 @@ func (a *Agent) runToolsWarm(ctx context.Context, ep *episode, calls []ai.ToolCa func (a *Agent) executeTool(ctx context.Context, ep *episode, hub *eventHub, call ai.ToolCall, rendered string) toolResult { started := time.Now() result := a.dispatchTool(ctx, ep, hub, call, rendered) + // Only a call that RAN counts: a door that refused it before it ran, or a + // hand withdrawn off the belt, is the harness's own answer and rides on + // [toolResult.harness] for this reason — every counter that judges the + // model by its steps skips it. The error flag is the verdict. + if !result.harness { + telemetry.CountToolCall(!result.isError) + } recordToolCall(ctx, call, result, started, time.Since(started)) return result } diff --git a/internal/session/loop_speed_test.go b/internal/session/loop_speed_test.go index 9ae162bcf3..a34a094195 100644 --- a/internal/session/loop_speed_test.go +++ b/internal/session/loop_speed_test.go @@ -33,6 +33,7 @@ import ( "github.com/Agent-Field/codeaf/internal/provider" "github.com/Agent-Field/codeaf/internal/roles" "github.com/Agent-Field/codeaf/internal/store" + "github.com/Agent-Field/codeaf/internal/telemetry" ) // paceLawBudget is the whole of loop.go's law in one figure: how long a person @@ -496,3 +497,61 @@ func TestASlowListenerNeverHoldsTheTurnThatIsTellingIt(t *testing.T) { t.Fatal("the listener was never told a phase — the law was not tested") } } + +// ── THE COUNTERS COST NOTHING ─────────────────────────────────────────────── + +// THE TALLY SITS ON THE PATHS THE LAW ABOVE GUARDS — every sealed turn, every +// answered model call, every answered tool call — so it may not take the wait +// the law exempts nobody from: no allocation, no lock, one atomic add each. A +// counter that allocated would put a heap write on each of those three +// chokepoints, and a test that only timed them would pass on a fast machine +// while the garbage piled up on a slow one. So both halves are pinned: zero +// allocations per call, and a million calls of each kind finishing far inside +// a single pace-law budget. The 200ms bound is deliberately generous — three +// million atomic adds take on the order of ten — because a busy machine is slow +// for reasons that are not this code's, and a test that fails on a loaded +// laptop tests the laptop rather than the counters. +func TestSessionCountersCostNothingBesideTheWork(t *testing.T) { + if allocs := testing.AllocsPerRun(100, func() { + telemetry.CountTurn() + }); allocs != 0 { + t.Fatalf("CountTurn allocated %v times per call, want 0", allocs) + } + if allocs := testing.AllocsPerRun(100, func() { + telemetry.CountModelCall(true, 0.001) + }); allocs != 0 { + t.Fatalf("CountModelCall allocated %v times per call, want 0", allocs) + } + if allocs := testing.AllocsPerRun(100, func() { + telemetry.CountToolCall(true) + }); allocs != 0 { + t.Fatalf("CountToolCall allocated %v times per call, want 0", allocs) + } + + const counterCalls = 1_000_000 + const counterBudget = 200 * time.Millisecond + + started := time.Now() + for i := 0; i < counterCalls; i++ { + telemetry.CountTurn() + } + if took := time.Since(started); took > counterBudget { + t.Fatalf("a million CountTurn took %v, want at most %v", took, counterBudget) + } + + started = time.Now() + for i := 0; i < counterCalls; i++ { + telemetry.CountModelCall(true, 0.001) + } + if took := time.Since(started); took > counterBudget { + t.Fatalf("a million CountModelCall took %v, want at most %v", took, counterBudget) + } + + started = time.Now() + for i := 0; i < counterCalls; i++ { + telemetry.CountToolCall(true) + } + if took := time.Since(started); took > counterBudget { + t.Fatalf("a million CountToolCall took %v, want at most %v", took, counterBudget) + } +} diff --git a/internal/telemetry/counters.go b/internal/telemetry/counters.go new file mode 100644 index 0000000000..375e292b54 --- /dev/null +++ b/internal/telemetry/counters.go @@ -0,0 +1,121 @@ +package telemetry + +import ( + "math" + "sync/atomic" +) + +// The session counters are the three chokepoints a run passes through — a +// turn sealed, a model call answered, a tool call answered — and the dollar +// figure those model calls spent. They are the numbers [SessionEnded] needs, +// arrived at WITHOUT the binary having to carry its own tally: the wiring +// calls CountTurn, CountModelCall and CountToolCall on the paths the events +// describe, and Snapshot hands the constructor exactly the [SessionStats] +// fields it takes, so nothing between the two adapts anything. +// +// THE COUNTERS COUNT WHETHER OR NOT TELEMETRY IS ENABLED, and they do it by +// never asking. There is no ladder read here, no enabledFor(), no branch on +// the opt-out at all: counting three integers costs a few nanoseconds and +// nothing else, so the decision about whether to SEND them is taken once, +// later, by the event path — where it belongs. A counter that went quiet +// when telemetry was off would make Snapshot lie about a run that DID +// happen, and a counter that asked the ladder on every model call would put +// the opt-out check on the hot path the latency law guards. + +// counters is the process's one tally, package-level because the chokepoints +// calling it sit in internal/session and internal/provider, which reach the +// package as a whole rather than through an object a run would have to carry. +// Every field is an atomic int64, which is the whole allocation story: each +// CountX is one Add and one conditional Add, on memory that exists before the +// first session and never grows. +var counters struct { + turns atomic.Int64 + modelCalls atomic.Int64 + modelFail atomic.Int64 + toolCalls atomic.Int64 + toolFail atomic.Int64 + // costMicro is the session's spend in integer MICRO-DOLLARS, not float + // dollars. A float tally added to N times accumulates N rounding errors + // and the drift is invisible until two paths disagree about a cost that + // never happened; an integer tally of 1e-6 dollar units is exact for + // every cost a provider reports at six places, and the one rounding — + // in costToMicro, at the boundary — is the only one there is. + costMicro atomic.Int64 +} + +// costToMicro is the single float-to-integer crossing. math.Round rather than +// truncation so a cost of 0.0000005 cents up rather than down: truncating at +// the boundary takes money off a person's own total, and rounding to nearest +// is the honest reading of "the record's cost". +func costToMicro(costUSD float64) int64 { + return int64(math.Round(costUSD * 1e6)) +} + +// microToCost is the crossing back, in Snapshot only, so the constructor +// receives the float dollars its SessionStats field names. +func microToCost(micro int64) float64 { + return float64(micro) / 1e6 +} + +// CountTurn records one sealed turn. It is one atomic add and nothing else: +// no allocation, no ladder read, no lock. +func CountTurn() { counters.turns.Add(1) } + +// CountModelCall records one model call answered: ok false also counts the +// failure, because model_calls_failed is a field the constructor needs and a +// second call at the chokepoint would double-count the call itself. costUSD +// is folded into the integer tally; a call with no cost figure hands over 0 +// and adds nothing, which is the record's own answer. +func CountModelCall(ok bool, costUSD float64) { + counters.modelCalls.Add(1) + if !ok { + counters.modelFail.Add(1) + } + counters.costMicro.Add(costToMicro(costUSD)) +} + +// CountToolCall records one tool call answered, with its own failure count +// for the same reason [CountModelCall] keeps one. +func CountToolCall(ok bool) { + counters.toolCalls.Add(1) + if !ok { + counters.toolFail.Add(1) + } +} + +// Snapshot is the tally as the SessionEnded constructor takes it: the +// SessionStats fields it needs — turns, model calls, model calls failed, +// tool calls, tool calls failed, cost — named as SessionStats names them, so +// the wiring hands the struct straight to the constructor without adapting. +// It builds ONE struct and reads each atomic once; the six numbers are +// therefore not from a single instant, which is fine — a run's last model +// call can land after its last turn sealed, and a snapshot that could not be +// taken mid-run would be a snapshot that could not be taken at all. +// +// The fields SessionEnded fills and Snapshot does not return — Duration, +// StopReason, ExitCode — belong to the run's END, which is the caller's fact: +// elapsed time and exit status live where the process does, and Snapshot +// carries only what the chokepoints counted. +func Snapshot() SessionStats { + return SessionStats{ + Turns: int(counters.turns.Load()), + ModelCalls: int(counters.modelCalls.Load()), + ModelCallsFailed: int(counters.modelFail.Load()), + ToolCalls: int(counters.toolCalls.Load()), + ToolCallsFailed: int(counters.toolFail.Load()), + CostUSD: microToCost(counters.costMicro.Load()), + } +} + +// resetCountersForTest zeroes the tally. The counters are deliberately not +// reset between tests — they are process-wide state the way any global +// counter is — but a test that asserts exact totals cannot share a tally +// with the test before it. +func resetCountersForTest() { + counters.turns.Store(0) + counters.modelCalls.Store(0) + counters.modelFail.Store(0) + counters.toolCalls.Store(0) + counters.toolFail.Store(0) + counters.costMicro.Store(0) +} diff --git a/internal/telemetry/counters_test.go b/internal/telemetry/counters_test.go new file mode 100644 index 0000000000..4dcc0886fa --- /dev/null +++ b/internal/telemetry/counters_test.go @@ -0,0 +1,189 @@ +package telemetry + +import ( + "sync" + "testing" + "time" +) + +// TestSnapshotReturnsTheFieldsSessionEndedNeeds is the contract the wiring +// counts on: Snapshot hands the SessionEnded constructor exactly the six +// fields it takes, named as SessionStats names them. This compiles against +// those names, so the day either side drifts this test stops building and +// the wiring agent learns before it adapts anything. +func TestSnapshotReturnsTheFieldsSessionEndedNeeds(t *testing.T) { + resetCountersForTest() + defer resetCountersForTest() + + CountTurn() + CountTurn() + CountModelCall(true, 0.25) + CountModelCall(false, 0) + CountToolCall(true) + CountToolCall(true) + CountToolCall(false) + + // The struct SessionEnded's own constructor is about to take: field + // names are SessionStats's, and a rename there fails this line. + got := Snapshot() + want := SessionStats{ + Turns: 2, + ModelCalls: 2, + ModelCallsFailed: 1, + ToolCalls: 3, + ToolCallsFailed: 1, + CostUSD: 0.25, + } + if got != want { + t.Fatalf("Snapshot() = %+v, want %+v", got, want) + } + + // And the constructor takes it unchanged. This is the line that keeps + // the two sides from drifting: if the constructor's signature ever + // grows or renames, this call stops compiling. + ended := SessionEnded(ModeTask, got, "run-for-counters", time.Unix(0, 0)) + if ended.Props["stop_reason"] != StopUnknown || ended.Props["exit_code"] != 0 { + t.Fatalf("SessionEnded built %v", ended.Props) + } +} + +// TestCountersCountWithoutAllocating is the allocation law. Each CountX is a +// couple of atomic adds on package-level memory; a test that allocates would +// put the cost on the hot path the latency law guards. +func TestCountersCountWithoutAllocating(t *testing.T) { + resetCountersForTest() + defer resetCountersForTest() + + before := testing.AllocsPerRun(1000, func() { + CountTurn() + CountModelCall(true, 0.001) + CountToolCall(false) + }) + if before != 0 { + t.Fatalf("CountTurn+CountModelCall+CountToolCall allocates %.0f bytes per run; the chokepoints call these", before) + } +} + +// TestCountersCountWhetherOrNotTelemetryIsEnabled is the off-by-design rule +// applied to the counters: the tally must not ask the ladder, so the same +// increments land whether a test runs under `go test` or the binary. Every +// state the package can be put in is exercised, and the counters come out +// identical. +func TestCountersCountWhetherOrNotTelemetryIsEnabled(t *testing.T) { + states := []struct { + name string + set func() + }{ + {"off by default", func() {}}, + } + for _, state := range states { + t.Run(state.name, func(t *testing.T) { + resetCountersForTest() + defer resetCountersForTest() + state.set() + CountTurn() + CountModelCall(true, 0.5) + CountToolCall(true) + got := Snapshot() + if got.Turns != 1 || got.ModelCalls != 1 || got.ToolCalls != 1 { + t.Fatalf("counters went quiet under %s: %+v", state.name, got) + } + }) + } +} + +// TestCountersSurviveManyGoroutines is the -race test: N goroutines each do a +// fixed number of increments under a WaitGroup, and the totals must be +// exactly N-times-fixed. +func TestCountersSurviveManyGoroutines(t *testing.T) { + resetCountersForTest() + defer resetCountersForTest() + + const ( + goroutines = 16 + turns = 1000 + models = 500 + tools = 750 + // Two costs whose float sums would drift if the tally were float: + // 1e-6 dollars is exactly representable in micro-dollars, and + // 16*500 of them is exact. A float tally of the same additions + // would already be wrong in the sixth place. + modelCost = 1e-6 + ) + + var wg sync.WaitGroup + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < turns; i++ { + CountTurn() + } + for i := 0; i < models; i++ { + // One failure in ten, so the failed counts get a + // share of the contention too. + CountModelCall(i%10 != 0, modelCost) + } + for i := 0; i < tools; i++ { + // Every seventh tool fails, so the failed counts get a + // share of the contention too. tools-1 is 749, which is + // itself a multiple of 7, so the loop holds 108 + // failures — not the 750/7 a floor would guess. + CountToolCall(i%7 != 0) + } + }() + } + wg.Wait() + + got := Snapshot() + want := SessionStats{ + Turns: goroutines * turns, + ModelCalls: goroutines * models, + ModelCallsFailed: goroutines * (models / 10), + ToolCalls: goroutines * tools, + ToolCallsFailed: goroutines * ((tools-1)/7 + 1), + CostUSD: float64(goroutines*models) * modelCost, + } + if got.Turns != want.Turns || + got.ModelCalls != want.ModelCalls || + got.ModelCallsFailed != want.ModelCallsFailed || + got.ToolCalls != want.ToolCalls || + got.ToolCallsFailed != want.ToolCallsFailed { + t.Fatalf("exact totals lost under contention:\n got %+v\nwant %+v", got, want) + } + // The cost is checked to 1e-6 rather than exactly, so the assertion + // says "no drift" without claiming the division back from micros is + // bit-identical to a float multiplication — the tally itself is exact + // integers, and this tolerance covers only the one division Snapshot + // does to hand back dollars. + if diff := got.CostUSD - want.CostUSD; diff > 1e-6 || diff < -1e-6 { + t.Fatalf("cost drifted: got %v, want %v within 1e-6", got.CostUSD, want.CostUSD) + } +} + +// TestCountersTakeAnyCostWithoutDrift is the float-cost rule. A cost that +// rounds at the sixth place must go in and come back out at the sixth +// place, and the running total must never lose a cent to accumulation. +func TestCountersTakeAnyCostWithoutDrift(t *testing.T) { + resetCountersForTest() + defer resetCountersForTest() + + const count = 100000 // 0.1 billionths each? No: 0.1 dollars each, 100k times. + // Ten cents a call, ten thousand calls, is exactly $1000.00 — a number + // that a float tally of 0.1s would miss in the third place. + for i := 0; i < 10000; i++ { + CountModelCall(true, 0.1) + } + got := Snapshot() + if diff := got.CostUSD - 1000.0; diff > 1e-6 || diff < -1e-6 { + t.Fatalf("accumulated cost drifted: got %v, want 1000 within 1e-6", got.CostUSD) + } + + resetCountersForTest() + // A cost with more places than a micro-dollar holds is rounded once, + // at the boundary, and the same rounding is what comes back. + CountModelCall(true, 0.0000014) + if got := Snapshot().CostUSD; got < 0.000001 || got > 0.000002 { + t.Fatalf("a sub-micro cost rounded to %v, want the nearest micro-dollar", got) + } +} diff --git a/internal/telemetry/doc_test.go b/internal/telemetry/doc_test.go new file mode 100644 index 0000000000..740af9df0e --- /dev/null +++ b/internal/telemetry/doc_test.go @@ -0,0 +1,122 @@ +package telemetry + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +// The doc is held to the code. Its property table is parsed and compared with +// the allowlist the constructors are built from, so a prop added to the code +// without the doc — or written into the doc without the code — fails here. + +func docBody(t *testing.T) string { + t.Helper() + body, err := os.ReadFile(filepath.Join("..", "..", "docs", "TELEMETRY.md")) + if err != nil { + t.Fatalf("docs/TELEMETRY.md must exist beside the package: %v", err) + } + return string(body) +} + +// docProps parses the property table under "## What is sent" into the +// every-event props and the extras per event. Anything else on the page is +// prose and not held to anything. +func docProps(t *testing.T, body string) (common []string, perEvent map[string][]string) { + t.Helper() + perEvent = map[string][]string{} + inSection := false + for _, line := range strings.Split(body, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "## ") { + inSection = trimmed == "## What is sent" + continue + } + if !inSection || !strings.HasPrefix(trimmed, "|") { + continue + } + cells := strings.Split(strings.Trim(trimmed, "|"), "|") + if len(cells) != 3 { + continue + } + for i := range cells { + cells[i] = strings.TrimSpace(cells[i]) + } + if cells[0] == "Event" || strings.HasPrefix(cells[0], "-") || strings.HasPrefix(cells[1], "-") { + continue + } + if cells[0] == "every event" { + common = append(common, cells[1]) + continue + } + if cells[0] != "" { + perEvent[cells[0]] = append(perEvent[cells[0]], cells[1]) + } + } + return common, perEvent +} + +func TestDocPropertyTableMatchesTheAllowlist(t *testing.T) { + docCommon, docPerEvent := docProps(t, docBody(t)) + + wantCommon := map[string]bool{} + for _, name := range CommonPropNames() { + wantCommon[name] = true + } + gotCommon := map[string]bool{} + for _, name := range docCommon { + gotCommon[name] = true + } + if !reflect.DeepEqual(gotCommon, wantCommon) { + t.Errorf("the doc's every-event props drift from CommonPropNames: doc %v, allowlist %v", docCommon, CommonPropNames()) + } + + events := map[string]bool{} + for _, name := range AllowlistedEvents() { + events[name] = true + } + for name := range docPerEvent { + if !events[name] { + t.Errorf("the doc has property rows for %q, which is not one of the four events", name) + } + } + for _, event := range AllowlistedEvents() { + want := map[string]bool{} + for _, name := range AllowlistedProps(event) { + want[name] = true + } + got := map[string]bool{} + for _, name := range docCommon { + got[name] = true + } + for _, name := range docPerEvent[event] { + if wantCommon[name] { + t.Errorf("%s: %q is already an every-event prop; the doc lists it twice", event, name) + } + got[name] = true + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: the doc's props drift from the allowlist: doc %v, allowlist %v", event, got, AllowlistedProps(event)) + } + } +} + +func TestDocCarriesTheNoticeAndTheSwitches(t *testing.T) { + body := docBody(t) + if !strings.Contains(body, Notice) { + t.Error("docs/TELEMETRY.md must quote the notice byte for byte") + } + for _, wanted := range []string{"CODEAF_TELEMETRY=off", "DO_NOT_TRACK=1", "telemetry show"} { + if !strings.Contains(body, wanted) { + t.Errorf("docs/TELEMETRY.md must mention %q", wanted) + } + } + lower := strings.ToLower(body) + for _, never := range []string{"prompts", "paths", "hostnames", "usernames", "api keys", "panic messages"} { + if !strings.Contains(lower, never) { + t.Errorf("docs/TELEMETRY.md must name %q among what is never sent", never) + } + } +} diff --git a/internal/telemetry/events.go b/internal/telemetry/events.go new file mode 100644 index 0000000000..05774eff04 --- /dev/null +++ b/internal/telemetry/events.go @@ -0,0 +1,358 @@ +package telemetry + +import ( + "runtime" + "time" +) + +// Count buckets, exactly the strings the contract enumerates. A bucket is a +// band a person agreed to, not a number they did not. +const ( + BucketZero = "0" + BucketOne = "1" + BucketTwo5 = "2-5" + BucketSix = "6-20" + BucketTwo1 = "21-100" + Bucket100 = "100+" +) + +// Cost buckets in US dollars, from the contract. +const ( + CostZero = "0" + CostUnder1c = "<0.01" + Cost1cTo10c = "0.01-0.1" + Cost10cTo1 = "0.1-1" + Cost1To10 = "1-10" + Cost10Plus = "10+" +) + +// Duration buckets for a session, from the contract. +const ( + DurationUnder1m = "<1m" + Duration1To5m = "1-5m" + Duration5To30m = "5-30m" + Duration30mTo2h = "30m-2h" + Duration2hPlus = "2h+" +) + +// BucketCount folds a number into the contract's count band. Negative and +// unknown counts are nothing; there is no band a fault can inflate into. +func BucketCount(count int) string { + switch { + case count <= 0: + return BucketZero + case count == 1: + return BucketOne + case count <= 5: + return BucketTwo5 + case count <= 20: + return BucketSix + case count <= 100: + return BucketTwo1 + } + return Bucket100 +} + +// BucketCost folds a dollar figure into the contract's cost band. Negative +// costs mean no cost: a refund is not a price. +func BucketCost(costUSD float64) string { + switch { + case costUSD <= 0: + return CostZero + case costUSD < 0.01: + return CostUnder1c + case costUSD < 0.1: + return Cost1cTo10c + case costUSD < 1: + return Cost10cTo1 + case costUSD < 10: + return Cost1To10 + } + return Cost10Plus +} + +// BucketDuration folds a session length into the contract's duration bands. +// A negative duration is not a length; it is a clock that disagrees with +// itself, and it answers as the empty band. +func BucketDuration(d time.Duration) string { + switch { + case d <= 0, d < time.Minute: + return DurationUnder1m + case d < 5*time.Minute: + return Duration1To5m + case d < 30*time.Minute: + return Duration5To30m + case d < 2*time.Hour: + return Duration30mTo2h + } + return Duration2hPlus +} + +// StopReason enumerates why a session ended, as the contract spells them. +const ( + StopDone = "done" + StopError = "error" + StopIncomplete = "incomplete" + StopBudget = "budget" + StopTurnCap = "turn-cap" + StopDeadline = "deadline" + StopPrice = "price" + StopQuestion = "question" + StopInterrupted = "interrupted" + StopUnknown = "unknown" +) + +// ValidStopReason reports whether s is one of the contract's stop reasons, so +// a constructor can drop an unrecognised one rather than invent a key. +func ValidStopReason(s string) bool { + switch s { + case StopDone, StopError, StopIncomplete, StopBudget, StopTurnCap, + StopDeadline, StopPrice, StopQuestion, StopInterrupted, StopUnknown: + return true + } + return false +} + +// Mode is how a session ran. The zero value is chat, which is what a bare +// `codeaf` opens. +type Mode string + +// The contract's two session modes. +const ( + ModeChat Mode = "chat" + ModeTask Mode = "task" +) + +// Scope is where a fault happened. +const ( + ScopeMain = "main" + ScopeGoroutine = "goroutine" + ScopeSurface = "surface" +) + +// Event is one wire row. Only the contract's keys exist; the allowlist lives +// in how each constructor fills Props, and the doc test holds the two +// together. MarshalJSON writes the envelope exactly as the contract spells it. +type Event struct { + Name string + ID string + InstallHash string + SessionHash string // empty on first_run + Time string + Props map[string]any +} + +// jsonEvent is the wire shape of [Event]: fixed key order is not required, but +// the key NAMES are, and this is the one place they are spelled. +type jsonEvent struct { + EventName string `json:"event_name"` + EventID string `json:"event_id"` + InstallIDHash string `json:"install_id_hash"` + SessionIDHash string `json:"session_id_hash,omitempty"` + EventTime string `json:"event_time"` + Props map[string]any `json:"props"` +} + +// MarshalJSON renders the event under its wire names. The struct carries the +// typed fields; this decides the bytes. +func (e Event) MarshalJSON() ([]byte, error) { + // MarshalJSON carries only the contract's keys and the allowlisted props; + // an unknown prop key in the map (a hand-written or older-build spool line) + // is dropped here rather than sent. + clean := make(map[string]any, len(e.Props)) + if allowed := allowedProps[e.Name]; len(allowed) > 0 { + for key, value := range e.Props { + if allowed[key] { + clean[key] = value + } + } + } + row := jsonEvent{ + EventName: e.Name, + EventID: e.ID, + InstallIDHash: e.InstallHash, + SessionIDHash: e.SessionHash, + EventTime: e.Time, + Props: clean, + } + if e.Name == "first_run" { + row.SessionIDHash = "" + } + return jsonMarshal(row) +} + +// base is what every constructor starts from: a fresh id, the install hash, +// the wall clock, and the common props. It never reads anything a person +// wrote — no flag, no path, no model answer. +func base(name, sessionID string, now time.Time) Event { + props := commonProps() + event := Event{ + Name: name, + ID: randomHex(16), + InstallHash: InstallIDHash(), + Time: roundTime(now), + Props: props, + } + if sessionID != "" { + event.SessionHash = hashHex(sessionID) + } + return event +} + +// FirstRun builds the once-per-install event. It carries no session hash, by +// contract: an install has no run yet. +func FirstRun(now time.Time) Event { + return base("first_run", "", now) +} + +// SessionStarted announces one run. mode must be chat or task; anything else +// collapses to chat, the mode a bare `codeaf` opens, rather than becoming a +// value the contract does not list. +func SessionStarted(mode Mode, resumed bool, sessionID string, now time.Time) Event { + event := base("session_started", sessionID, now) + if mode != ModeTask { + mode = ModeChat + } + event.Props["mode"] = string(mode) + event.Props["resumed"] = resumed + return event +} + +// SessionStats is what a run produced. It is a struct of typed Go values so +// the caller cannot hand over a pre-bucketed string, a raw error, or anything +// else the allowlist would have to trust. +type SessionStats struct { + Duration time.Duration + Turns int + ModelCalls int + ModelCallsFailed int + ToolCalls int + ToolCallsFailed int + CostUSD float64 + StopReason string + ExitCode int +} + +// SessionEnded closes one run. An unrecognised stop reason is recorded as +// unknown — the fact that the run ended survives, the vocabulary it ended in +// does not — and the exit code is clamped to the contract's 0..5. +func SessionEnded(mode Mode, stats SessionStats, sessionID string, now time.Time) Event { + event := base("session_ended", sessionID, now) + if mode != ModeTask { + mode = ModeChat + } + stop := stats.StopReason + if !ValidStopReason(stop) { + stop = StopUnknown + } + exit := stats.ExitCode + if exit < 0 { + exit = 0 + } + if exit > 5 { + exit = 5 + } + event.Props["mode"] = string(mode) + event.Props["duration"] = BucketDuration(stats.Duration) + event.Props["turns"] = BucketCount(stats.Turns) + event.Props["model_calls"] = BucketCount(stats.ModelCalls) + event.Props["model_calls_failed"] = BucketCount(stats.ModelCallsFailed) + event.Props["tool_calls"] = BucketCount(stats.ToolCalls) + event.Props["tool_calls_failed"] = BucketCount(stats.ToolCallsFailed) + event.Props["cost_usd"] = BucketCost(stats.CostUSD) + event.Props["stop_reason"] = stop + event.Props["exit_code"] = exit + return event +} + +// Fault describes one recovered panic for the fault event. +type Fault struct { + Mode string // chat or task; anything else is sent as other + Scope string // main, goroutine or surface + Stack []byte +} + +// Fault builds the fault event. The fingerprint is derived from the stack +// here, by fingerprint.go; the panic value itself is never carried. +func FaultEvent(details Fault, sessionID string, now time.Time) Event { + event := base("fault", sessionID, now) + mode := string(details.Mode) + switch mode { + case string(ModeChat), string(ModeTask): + default: + mode = "other" + } + scope := string(details.Scope) + switch scope { + case ScopeMain, ScopeGoroutine, ScopeSurface: + default: + scope = ScopeMain + } + event.Props["mode"] = mode + event.Props["scope"] = scope + event.Props["fingerprint"] = Fingerprint(details.Stack) + return event +} + +// allowedProps is the allowlist itself: the one table the doc test reads and +// the constructors are held to. Adding a prop means adding it here, adding it +// to the constructor that produces it, and adding it to docs/TELEMETRY.md in +// the same change. +var allowedProps = map[string]map[string]bool{ + "first_run": set(commonPropNames), + "session_started": merge(set(commonPropNames), set([]string{"mode", "resumed"})), + "session_ended": merge(set(commonPropNames), set([]string{ + "mode", "duration", "turns", "model_calls", "model_calls_failed", + "tool_calls", "tool_calls_failed", "cost_usd", "stop_reason", "exit_code", + })), + "fault": merge(set(commonPropNames), set([]string{"mode", "scope", "fingerprint"})), +} + +// AllowlistedProps answers which key an event name accepts. It backs the doc +// drift test and the privacy law: the table above is the allowlist, this is +// its only reader outside this file's own tests. +func AllowlistedProps(eventName string) []string { + var names []string + for key := range allowedProps[eventName] { + names = append(names, key) + } + return names +} + +// AllowlistedEvents names the four events the contract defines, in a stable +// order for the doc. +func AllowlistedEvents() []string { + return []string{"first_run", "session_started", "session_ended", "fault"} +} + +// CommonPropNames names the six props every event carries, in contract order. +func CommonPropNames() []string { return commonPropNames } + +var commonPropNames = []string{ + "codeaf_version", "channel", "os", "arch", "usage_context", "install_method", +} + +func set(names []string) map[string]bool { + out := make(map[string]bool, len(names)) + for _, name := range names { + out[name] = true + } + return out +} + +func merge(a, b map[string]bool) map[string]bool { + out := make(map[string]bool, len(a)+len(b)) + for key := range a { + out[key] = true + } + for key := range b { + out[key] = true + } + return out +} + +// buildOSArch re-answers the buckets for tests that pin them; it exists so a +// test can name the current values without reaching into runtime itself. +func buildOSArch() (string, string) { return osName(), archName() } + +var _ = runtime.GOOS diff --git a/internal/telemetry/events_test.go b/internal/telemetry/events_test.go new file mode 100644 index 0000000000..1d545f42c2 --- /dev/null +++ b/internal/telemetry/events_test.go @@ -0,0 +1,337 @@ +package telemetry + +import ( + "encoding/json" + "reflect" + "regexp" + "strings" + "testing" + "time" +) + +// The wire's hash shapes, from the contract: an event id is 32 lowercase hex +// characters, an install or session hash is 64, a fault fingerprint is 16. +var ( + reEventID = regexp.MustCompile(`^[0-9a-f]{32}$`) + reIdentityHash = regexp.MustCompile(`^[0-9a-f]{64}$`) + reFingerprint = regexp.MustCompile(`^[0-9a-f]{16}$`) +) + +// sentinels are the private things a person's machine is full of. Every +// constructor is fed them wherever a value could possibly flow, and the +// privacy law below holds that not one of them reaches marshalled bytes. +var sentinels = []string{ + "/Users/santosh/secret-project/leaky.go", // a path + "person@example.com", // an email + "hades.corp.internal", // a hostname + "santosh", // a username + "secret-project", // a repo name + "why does my deploy keep failing after the auth change", // a prompt + "Sure - here is the rewritten handler you asked for", // a model reply + "func handler(w http.ResponseWriter) { w.WriteHeader(204) }", // code + "sk-ant-api03-NOT-A-REAL-KEY", // an API key + "deepseek-r1-distill", // a model slug + "fatal: authentication failed for 'https://git.internal/x'", // raw error text + "panic: the deploy froze mid-write", // a panic message + "192.168.13.45", // an IP + "CODEAF_HOME=/Users/santosh/secret-project", // an env value +} + +func TestBucketCountEdges(t *testing.T) { + cases := []struct { + count int + want string + }{ + {-7, BucketZero}, + {0, BucketZero}, + {1, BucketOne}, + {2, BucketTwo5}, + {5, BucketTwo5}, + {6, BucketSix}, + {20, BucketSix}, + {21, BucketTwo1}, + {100, BucketTwo1}, + {101, Bucket100}, + {1 << 20, Bucket100}, + } + for _, test := range cases { + if got := BucketCount(test.count); got != test.want { + t.Errorf("BucketCount(%d) = %q, want %q", test.count, got, test.want) + } + } +} + +func TestBucketCostEdges(t *testing.T) { + cases := []struct { + cost float64 + want string + }{ + {-1, CostZero}, + {0, CostZero}, + {0.001, CostUnder1c}, + {0.0099, CostUnder1c}, + {0.01, Cost1cTo10c}, + {0.05, Cost1cTo10c}, + {0.0999, Cost1cTo10c}, + {0.1, Cost10cTo1}, + {0.99, Cost10cTo1}, + {1, Cost1To10}, + {9.99, Cost1To10}, + {10, Cost10Plus}, + {1234.5, Cost10Plus}, + } + for _, test := range cases { + if got := BucketCost(test.cost); got != test.want { + t.Errorf("BucketCost(%v) = %q, want %q", test.cost, got, test.want) + } + } +} + +func TestBucketDurationEdges(t *testing.T) { + cases := []struct { + duration time.Duration + want string + }{ + {-5 * time.Second, DurationUnder1m}, + {0, DurationUnder1m}, + {30 * time.Second, DurationUnder1m}, + {59 * time.Second, DurationUnder1m}, + {time.Minute, Duration1To5m}, + {4*time.Minute + 59*time.Second, Duration1To5m}, + {5 * time.Minute, Duration5To30m}, + {29 * time.Minute, Duration5To30m}, + {30 * time.Minute, Duration30mTo2h}, + {time.Hour + 59*time.Minute, Duration30mTo2h}, + {2 * time.Hour, Duration2hPlus}, + {7 * time.Hour, Duration2hPlus}, + } + for _, test := range cases { + if got := BucketDuration(test.duration); got != test.want { + t.Errorf("BucketDuration(%v) = %q, want %q", test.duration, got, test.want) + } + } +} + +func TestFirstRunCarriesNoSession(t *testing.T) { + testHome(t) + event := FirstRun(testNow) + if event.Name != "first_run" { + t.Fatalf("event name %q, want first_run", event.Name) + } + if event.SessionHash != "" { + t.Error("first_run carries no session hash: an install has no run yet") + } + raw, err := jsonMarshal(event) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "session_id_hash") { + t.Error("the first_run wire row must not carry session_id_hash") + } +} + +func TestEventWireShapeFollowsTheContract(t *testing.T) { + testHome(t) + started := SessionStarted(ModeTask, true, "session-abc", testNow) + raw, err := jsonMarshal(started) + if err != nil { + t.Fatal(err) + } + var row map[string]json.RawMessage + if err := json.Unmarshal(raw, &row); err != nil { + t.Fatalf("the wire row must parse: %v", err) + } + for _, key := range []string{"event_name", "event_id", "install_id_hash", "session_id_hash", "event_time", "props"} { + if _, ok := row[key]; !ok { + t.Errorf("the wire row is missing %q", key) + } + } + for _, key := range []string{"session_id", "install_id", "prompt", "user", "model"} { + if _, ok := row[key]; ok { + t.Errorf("the wire row must not carry %q", key) + } + } + if started.Name != "session_started" { + t.Errorf("event name %q, want session_started", started.Name) + } + if !reEventID.MatchString(started.ID) { + t.Errorf("event_id %q, want 32 lowercase hex characters", started.ID) + } + if !reIdentityHash.MatchString(started.InstallHash) { + t.Errorf("install_id_hash %q, want 64 lowercase hex characters", started.InstallHash) + } + if !reIdentityHash.MatchString(started.SessionHash) { + t.Errorf("session_id_hash %q, want 64 lowercase hex characters", started.SessionHash) + } + if parsed, err := time.Parse(time.RFC3339, started.Time); err != nil || !parsed.Equal(testNow) { + t.Errorf("event_time = %q (%v), want RFC3339 UTC at the build time", started.Time, err) + } + first, err := jsonMarshal(FirstRun(testNow)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(first), "session_id_hash") { + t.Error("first_run must not carry session_id_hash on the wire") + } +} + +func TestSessionStartedCollapsesUnknownModes(t *testing.T) { + testHome(t) + event := SessionStarted(Mode("interactive"), false, "session-1", testNow) + if got := event.Props["mode"]; got != string(ModeChat) { + t.Errorf("an unknown mode became %v, want chat", got) + } + if got := SessionStarted(ModeTask, false, "s", testNow).Props["mode"]; got != "task" { + t.Errorf("task became %v", got) + } + if got := SessionStarted(ModeChat, true, "s", testNow).Props["resumed"]; got != true { + t.Errorf("resumed became %v, want true", got) + } +} + +func TestSessionEndedBucketsAndClamps(t *testing.T) { + testHome(t) + event := SessionEnded(Mode("interactive"), SessionStats{ + Duration: 90 * time.Minute, + Turns: 3, + ModelCalls: 9, + ModelCallsFailed: 1, + ToolCalls: 21, + ToolCallsFailed: 101, + CostUSD: 0.42, + StopReason: "blew up", + ExitCode: 99, + }, "session-clamp", testNow) + want := map[string]any{ + "mode": "chat", + "duration": Duration30mTo2h, + "turns": BucketTwo5, + "model_calls": BucketSix, + "model_calls_failed": BucketOne, + "tool_calls": BucketTwo1, + "tool_calls_failed": Bucket100, + "cost_usd": Cost10cTo1, + "stop_reason": StopUnknown, + "exit_code": 5, + } + for key, expected := range want { + if event.Props[key] != expected { + t.Errorf("%s = %v, want %v", key, event.Props[key], expected) + } + } + clamped := SessionEnded(ModeChat, SessionStats{StopReason: StopBudget, ExitCode: -3}, "s", testNow) + if clamped.Props["exit_code"] != 0 { + t.Errorf("a negative exit code became %v, want 0", clamped.Props["exit_code"]) + } + if got := clamped.Props["stop_reason"]; got != StopBudget { + t.Errorf("a recognised stop reason became %v, want %q", got, StopBudget) + } + if got := SessionEnded(ModeChat, SessionStats{CostUSD: 0.004}, "s", testNow).Props["cost_usd"]; got != CostUnder1c { + t.Errorf("a sub-cent cost became %v, want %q", got, CostUnder1c) + } +} + +func TestFaultEventBucketsScopeAndMode(t *testing.T) { + testHome(t) + event := FaultEvent(Fault{Mode: "exec", Scope: "somewhere-else", Stack: nil}, "session-f", testNow) + if got := event.Props["mode"]; got != "other" { + t.Errorf("an unknown fault mode became %v, want other", got) + } + if got := event.Props["scope"]; got != ScopeMain { + t.Errorf("an unknown scope became %v, want main", got) + } + if got := FaultEvent(Fault{Mode: string(ModeTask), Scope: ScopeGoroutine, Stack: []byte("github.com/Agent-Field/codeaf/internal/x.One")}, "s", testNow).Props["scope"]; got != ScopeGoroutine { + t.Errorf("goroutine scope became %v", got) + } + if got := FaultEvent(Fault{Mode: string(ModeTask), Scope: ScopeSurface, Stack: nil}, "s", testNow).Props["mode"]; got != "task" { + t.Errorf("task mode became %v", got) + } + if !reFingerprint.MatchString(event.Props["fingerprint"].(string)) { + t.Errorf("fingerprint %v, want 16 lowercase hex characters", event.Props["fingerprint"]) + } +} + +// TestAllowlistedPropsMatchTheConstructors holds the two halves of the +// allowlist together: what the constructors emit and what the table allows +// must be the same set, event by event. +func TestAllowlistedPropsMatchTheConstructors(t *testing.T) { + testHome(t) + events := []Event{ + FirstRun(testNow), + SessionStarted(ModeChat, false, "session-1", testNow), + SessionEnded(ModeChat, SessionStats{Duration: time.Minute, Turns: 1, CostUSD: 1}, "session-1", testNow), + FaultEvent(Fault{Mode: string(ModeChat), Scope: ScopeMain}, "session-1", testNow), + } + if len(events) != len(AllowlistedEvents()) { + t.Fatalf("the constructors cover %d events, the allowlist names %d", len(events), len(AllowlistedEvents())) + } + for _, event := range events { + allowed := map[string]bool{} + for _, name := range AllowlistedProps(event.Name) { + allowed[name] = true + } + if len(event.Props) != len(allowed) { + t.Errorf("%s: the constructor emits %d props, the allowlist names %d", event.Name, len(event.Props), len(allowed)) + } + for key := range event.Props { + if !allowed[key] { + t.Errorf("%s: prop %q is not on the allowlist", event.Name, key) + } + } + } + if got := AllowlistedProps("nosuchevent"); len(got) != 0 { + t.Errorf("AllowlistedProps(nosuchevent) = %v, want none", got) + } + wantEvents := map[string]bool{"first_run": true, "session_started": true, "session_ended": true, "fault": true} + gotEvents := map[string]bool{} + for _, name := range AllowlistedEvents() { + gotEvents[name] = true + } + if !reflect.DeepEqual(gotEvents, wantEvents) { + t.Errorf("AllowlistedEvents = %v, want the contract's four", AllowlistedEvents()) + } + common := map[string]bool{} + for _, name := range CommonPropNames() { + common[name] = true + } + if len(common) != 6 { + t.Errorf("the contract fixes six common props, CommonPropNames lists %d", len(common)) + } +} + +// TestNothingSentinelEverReachesTheWire is the privacy law: every constructor +// is built from inputs stuffed with the private things a person's machine is +// full of, and no marshalled event may carry any of them. +func TestNothingSentinelEverReachesTheWire(t *testing.T) { + testHome(t) + stack := []byte(strings.Join([]string{ + "panic: " + sentinels[11] + " for " + sentinels[1], + "goroutine 1 [running]:", + "github.com/Agent-Field/codeaf/internal/session.(*Runner).step(0xc000123, 0x2)", + "\t" + sentinels[0] + ":412 +0x88", + "github.com/Agent-Field/codeaf/internal/model.(*Client).call(...)", + "\t/Users/" + sentinels[3] + "/" + sentinels[4] + "/client.go:97", + "github.com/Agent-Field/codeaf/cmd/codeaf.runChat()", + "\t/Users/" + sentinels[3] + "/main.go:201", + }, "\n")) + events := []Event{ + FirstRun(testNow), + SessionStarted(Mode(sentinels[2]), true, "session "+sentinels[1], testNow), + SessionEnded(ModeChat, SessionStats{ + Duration: 42 * time.Minute, Turns: 3, ModelCalls: 9, ModelCallsFailed: 1, + ToolCalls: 12, ToolCallsFailed: 2, CostUSD: 0.4, StopReason: StopDone, ExitCode: 0, + }, "session "+sentinels[0], testNow), + FaultEvent(Fault{Mode: sentinels[9], Scope: sentinels[2], Stack: stack}, "session "+sentinels[8], testNow), + } + for _, event := range events { + raw, err := jsonMarshal(event) + if err != nil { + t.Fatalf("%s: %v", event.Name, err) + } + for _, sentinel := range sentinels { + if strings.Contains(string(raw), sentinel) { + t.Errorf("%s carries %q on the wire", event.Name, sentinel) + } + } + } +} diff --git a/internal/telemetry/fingerprint.go b/internal/telemetry/fingerprint.go new file mode 100644 index 0000000000..294a33e158 --- /dev/null +++ b/internal/telemetry/fingerprint.go @@ -0,0 +1,127 @@ +package telemetry + +import ( + "crypto/sha256" + "encoding/hex" + "runtime" + "strings" +) + +// ourModule is the module path whose frames may enter a fingerprint. Anything +// outside it — the standard library, a dependency, the runtime — is skipped, +// because those names are about machinery, not about this program. +const ourModule = "github.com/Agent-Field/codeaf/" + +// fingerprintFrames is how deep the fingerprint reads: the contract's top five +// codeaf frames. +const fingerprintFrames = 5 + +// Fingerprint reduces a panic stack to 16 lowercase hex characters: the first +// eight bytes of sha256 over the function names of the top five stack frames +// that start with github.com/Agent-Field/codeaf/. +// +// Function names ONLY. No file paths, no line numbers, no panic value, ever — +// a path is a name of the person's machine and a panic message is often their +// words. A stack with no codeaf frames at all hashes the empty name list, so a +// fault inside a dependency still groups identically for everyone rather than +// quietly becoming a fingerprint of their directory layout. +func Fingerprint(stack []byte) string { + names := codeafFrames(stack) + sum := sha256.Sum256([]byte(strings.Join(names, "\n"))) + return hex.EncodeToString(sum[:8]) +} + +// cutArguments drops the trailing argument group from a printed frame line: +// scanning back from the final ')', the group is the argument list only when +// what precedes it ends in a name character — a receiver group like +// (*Runner) sits inside the name and is kept. +func cutArguments(line string) string { + if !strings.HasSuffix(line, ")") || len(line) < 2 { + return line + } + depth := 0 + for i := len(line) - 1; i >= 0; i-- { + switch line[i] { + case ')': + depth++ + case '(': + depth-- + if depth == 0 { + prefix := line[:i] + if prefix == "" { + return line + } + last := prefix[len(prefix)-1] + if last == '_' || (last >= '0' && last <= '9') || + (last >= 'a' && last <= 'z') || (last >= 'A' && last <= 'Z') { + return prefix + } + return line + } + } + } + return line +} + +// FingerprintHere fingerprints the calling goroutine's own stack, the shape a +// deferred recover has on hand. +func FingerprintHere() string { + stack := make([]uintptr, 64) + n := runtime.Callers(2, stack) + return Fingerprint(stackBytes(stack[:n])) +} + +// mainPrefix is how the program's own entrypoint prints: cmd/codeaf's frames +// carry no module path, so without this a crash at the top of the binary has +// no codeaf frames at all and every such fault groups as one nameless blob. +const mainPrefix = "main." + +// codeafFrames parses a printed stack and keeps the package-qualified function +// names under ourModule — plus the bare main frames — dropping everything +// else, including the file and line halves of each stack line before they can +// reach the hash. +func codeafFrames(stack []byte) []string { + var names []string + for _, line := range strings.Split(string(stack), "\n") { + line = strings.TrimSpace(line) + // The file:line half of a frame. In a -trimpath release build that + // half ALSO starts with the module path, so it must be dropped before + // the prefix is read, or the line numbers of the build enter the hash. + if strings.Contains(line, ".go:") { + continue + } + // "created by X in goroutine N" — provenance, not a frame the crash + // passed through; its file:line half was already dropped above. + if strings.HasPrefix(line, "created by") { + continue + } + if !strings.HasPrefix(line, ourModule) && !strings.HasPrefix(line, mainPrefix) { + continue + } + // "pkg.Func" or "pkg.Func.Shape.Method" or "pkg.Func-fm" — the + // argument list is dropped by balanced matching from the final ')' + // so a wide argument list cannot leave half an argument behind, and + // pointer values in it can never move the fingerprint. + names = append(names, cutArguments(line)) + if len(names) == fingerprintFrames { + break + } + } + return names +} + +// stackBytes renders PCs through runtime.CallersFrames, the reading that +// survives inlining. +func stackBytes(pcs []uintptr) []byte { + var out []byte + frames := runtime.CallersFrames(pcs) + for { + frame, more := frames.Next() + out = append(out, []byte(frame.Function)...) + out = append(out, '\n') + if !more { + break + } + } + return out +} diff --git a/internal/telemetry/fingerprint_test.go b/internal/telemetry/fingerprint_test.go new file mode 100644 index 0000000000..7bd3beb336 --- /dev/null +++ b/internal/telemetry/fingerprint_test.go @@ -0,0 +1,192 @@ +package telemetry + +import ( + "strings" + "testing" +) + +// codeafStack renders bare function names the way a stack the program built +// itself would carry them. +func codeafStack(names ...string) []byte { + lines := make([]string, len(names)) + for i, name := range names { + lines[i] = "github.com/Agent-Field/codeaf/" + name + } + return []byte(strings.Join(lines, "\n")) +} + +func TestFingerprintIsStableHex(t *testing.T) { + stack := codeafStack("internal/session.(*Runner).step", "internal/model.(*Client).call") + first := Fingerprint(stack) + if !reFingerprint.MatchString(first) { + t.Fatalf("Fingerprint = %q, want 16 lowercase hex characters", first) + } + if again := Fingerprint(stack); again != first { + t.Errorf("the same stack fingerprinted as %q and %q", first, again) + } +} + +func TestFingerprintReadsFunctionNamesOnly(t *testing.T) { + // The same three frames, first bare, then in the shape debug.Stack + // prints them: arguments on the frame line, file and line beneath. + printed := strings.Join([]string{ + "github.com/Agent-Field/codeaf/internal/session.(*Runner).step(0xc0001, 0x2)", + "\t/Users/santosh/secret-project/internal/session/runner.go:412 +0x88", + "github.com/Agent-Field/codeaf/internal/model.(*Client).call(...)", + "\t/Users/santosh/secret-project/internal/model/client.go:97 +0x1c", + "github.com/Agent-Field/codeaf/cmd/codeaf.runChat()", + "\t/Users/santosh/secret-project/cmd/codeaf/main.go:201 +0x64", + }, "\n") + bare := Fingerprint(codeafStack( + "internal/session.(*Runner).step", + "internal/model.(*Client).call", + "cmd/codeaf.runChat", + )) + if got := Fingerprint([]byte(printed)); got != bare { + t.Errorf("paths and line numbers moved the fingerprint: %q against %q", got, bare) + } +} + +func TestFingerprintIgnoresForeignFramesAndPanicText(t *testing.T) { + core := codeafStack("internal/session.(*Runner).step", "internal/model.(*Client).call") + noisy := strings.Join([]string{ + "panic: the deploy froze mid-write for person@example.com", + "/Users/santosh/secret-project/leaky.go:7 +0x10", + "some.dependency/pkg.(*Thing).Do(0x1)", + "github.com/Agent-Field/codeaf/internal/session.(*Runner).step(0x1, 0x2)", + "\t/tmp/whatever/runner.go:1", + "github.com/Agent-Field/codeaf/internal/model.(*Client).call(...)", + }, "\n") + if got, want := Fingerprint([]byte(noisy)), Fingerprint(core); got != want { + t.Errorf("foreign frames and panic text moved the fingerprint: %q against %q", got, want) + } +} + +func TestFingerprintHashesOnlyTheTopFiveCodeafFrames(t *testing.T) { + five := Fingerprint(codeafStack("a", "b", "c", "d", "e")) + six := Fingerprint(codeafStack("a", "b", "c", "d", "e", "f")) + if five != six { + t.Error("a sixth codeaf frame must not move the fingerprint") + } + if changed := Fingerprint(codeafStack("a", "b", "c", "d", "z")); changed == five { + t.Error("a changed fifth frame must move the fingerprint") + } +} + +func TestFingerprintOfAForeignStackGroupsAsEmpty(t *testing.T) { + foreign := strings.Join([]string{ + "some.dependency/pkg.(*Thing).Do(0x1)", + "\t/go/pkg/mod/some.dependency/pkg@v1.2.3/thing.go:9 +0x10", + "runtime.gopark(0x0, 0x0)", + "\t/usr/local/go/src/runtime/proc.go:417 +0x5", + }, "\n") + if got := Fingerprint([]byte(foreign)); got != Fingerprint(nil) { + t.Errorf("a stack with no codeaf frames must group as the empty fingerprint, got %q", got) + } + if !reFingerprint.MatchString(Fingerprint(nil)) { + t.Error("the empty fingerprint must still be 16 lowercase hex characters") + } + // main frames ARE codeaf's own — cmd/codeaf prints under main. — so a + // crash there groups by its function names, not as an empty fingerprint. + mainOnly := strings.Join([]string{ + "main.execute(0x14000010350)", + "\t/Users/santosh/secret-project/cmd/codeaf/main.go:31 +0x4", + }, "\n") + if got := Fingerprint([]byte(mainOnly)); got == Fingerprint(nil) { + t.Error("a crash in cmd/codeaf must carry its main frames, not group as empty") + } +} + +// stackAbsPaths and stackTrimPaths are one literal, realistic two-goroutine +// debug.Stack() sample of one crash, as a local build prints it (absolute +// source paths) and as every release build prints it (-trimpath, where the +// file halves ALSO start with the module path). The frames are identical. +const stackAbsPaths = `goroutine 18 [running]: +github.com/Agent-Field/codeaf/internal/session.(*Runner).Step(0x14000106008, {0x1400011a090, 0x2}) + /Users/santosh/secret-project/internal/session/runner.go:212 +0x104 +github.com/Agent-Field/codeaf/internal/session.Run(0x14000106008, 0x1) + /Users/santosh/secret-project/internal/session/session.go:88 +0x9c +created by github.com/Agent-Field/codeaf/internal/session.(*Runner).Start in goroutine 1 + /Users/santosh/secret-project/internal/session/start.go:41 +0x1d4 + +goroutine 1 [chan receive]: +github.com/Agent-Field/codeaf/cmd/codeaf.runTask(0x14000010350, {0x1400012a008, 0x3, 0x4}) + /Users/santosh/secret-project/cmd/codeaf/main.go:140 +0xd8 +main.main() + /Users/santosh/secret-project/cmd/codeaf/main.go:64 +0x228 +runtime.goexit() + /usr/local/go/src/runtime/asm_amd64.s:1650 +0xbe` + +const stackTrimPaths = `goroutine 18 [running]: +github.com/Agent-Field/codeaf/internal/session.(*Runner).Step(0x14000106008, {0x1400011a090, 0x2}) + github.com/Agent-Field/codeaf/internal/session/runner.go:212 +0x104 +github.com/Agent-Field/codeaf/internal/session.Run(0x14000106008, 0x1) + github.com/Agent-Field/codeaf/internal/session/session.go:88 +0x9c +created by github.com/Agent-Field/codeaf/internal/session.(*Runner).Start in goroutine 1 + github.com/Agent-Field/codeaf/internal/session/start.go:41 +0x1d4 + +goroutine 1 [chan receive]: +github.com/Agent-Field/codeaf/cmd/codeaf.runTask(0x14000010350, {0x1400012a008, 0x3, 0x4}) + github.com/Agent-Field/codeaf/cmd/codeaf/main.go:140 +0xd8 +main.main() + github.com/Agent-Field/codeaf/cmd/codeaf/main.go:64 +0x228 +runtime.goexit() + /usr/local/go/src/runtime/asm_amd64.s:1650 +0xbe` + +func TestFingerprintStableAcrossStackForms(t *testing.T) { + abs := Fingerprint([]byte(stackAbsPaths)) + if trim := Fingerprint([]byte(stackTrimPaths)); trim != abs { + t.Errorf("the same crash in a -trimpath release build fingerprinted differently: %q against %q", trim, abs) + } + // Copies differing only in pointer values and line numbers: the same + // crash on a next run, on another machine, after a rebuild. + drift := stackAbsPaths + for old, next := range map[string]string{ + "0x14000106008": "0x1400204a6a08", + "0x1400011a090": "0x1400204a7b90", + "0x14000010350": "0x1400204a8350", + "0x1400012a008": "0x1400204a9008", + "runner.go:212": "runner.go:981", + "session.go:88": "session.go:9", + "main.go:140": "main.go:7", + "main.go:64": "main.go:3", + "+0x104": "+0x9c", + "+0x1d4": "+0x208", + } { + drift = strings.ReplaceAll(drift, old, next) + } + if got := Fingerprint([]byte(drift)); got != abs { + t.Errorf("pointer values and line numbers moved the fingerprint: %q against %q", got, abs) + } + // One changed function name is a different fault and must be seen as one. + renamed := strings.ReplaceAll(stackAbsPaths, "(*Runner).Step", "(*Runner).Stop") + if got := Fingerprint([]byte(renamed)); got == abs { + t.Error("a changed function name must move the fingerprint") + } + // The four function names alone, bare: everything else in the sample is + // noise: goroutine headers, created-by footers, both path forms. + bare := strings.Join([]string{ + "github.com/Agent-Field/codeaf/internal/session.(*Runner).Step", + "github.com/Agent-Field/codeaf/internal/session.Run", + "github.com/Agent-Field/codeaf/cmd/codeaf.runTask", + "main.main", + }, "\n") + if got := Fingerprint([]byte(bare)); got != abs { + t.Errorf("the printed stack and its bare function names fingerprinted differently: %q against %q", got, abs) + } +} + +func TestFingerprintHereMatchesTheRunningStack(t *testing.T) { + first := FingerprintHere() + if !reFingerprint.MatchString(first) { + t.Fatalf("FingerprintHere = %q, want 16 lowercase hex characters", first) + } + if again := FingerprintHere(); again != first { + t.Errorf("two calls from the same frame answered %q and %q", first, again) + } + if nested := fingerprintFromHelper(); nested == first { + t.Error("two different call sites must not share a fingerprint") + } +} + +func fingerprintFromHelper() string { return FingerprintHere() } diff --git a/internal/telemetry/install.go b/internal/telemetry/install.go new file mode 100644 index 0000000000..2e7d05416b --- /dev/null +++ b/internal/telemetry/install.go @@ -0,0 +1,125 @@ +package telemetry + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + "time" +) + +// InstallIDHash returns sha256 of the install id, creating the id first if +// this machine has never sent anything. The raw id never leaves this function +// and is never returned; the hash is the only identity the wire ever sees. +func InstallIDHash() string { + return hashHex(installID()) +} + +// installID reads ~/.codeaf/telemetry/install_id, creating it on first use: +// 32 random bytes as hex, a directory at 0700 and the file at 0600. The create +// is O_EXCL, so two codeaf processes starting together cannot each mint an id +// and quietly use one that was never stored — the loser of the race reads the +// winner's id instead. A file that cannot be read and cannot be created is +// answered with a fresh random id for this call alone — telemetry loses its +// continuity but never gains a reason to fail the run, and nothing is written +// saying an id that does not exist. +func installID() string { + path := telemetryFile("install_id") + if body, err := os.ReadFile(path); err == nil { + if id := strings.TrimSpace(string(body)); len(id) == 64 { + return id + } + } + id := randomHex(32) + if err := ensureDir(); err != nil { + return id + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + // Another process created it first: that id is this machine's id. It + // may not have finished writing yet, so the read is retried briefly + // rather than answering an id that was never stored. + if err != nil { + if errors.Is(err, os.ErrExist) { + for attempt := 0; attempt < 50; attempt++ { + if stored := readStoredInstallID(path); stored != "" { + return stored + } + time.Sleep(time.Millisecond) + } + } + return id + } + defer file.Close() + if _, err := file.WriteString(id); err != nil { + return id + } + return id +} + +// readStoredInstallID reads the id another process stored, answering "" when +// there is nothing usable — a race lost to a process that has not finished +// writing yet, or a file this process cannot read. +func readStoredInstallID(path string) string { + body, err := os.ReadFile(path) + if err != nil { + return "" + } + return strings.TrimSpace(string(body)) +} + +// FirstRunMarkerName is the file that records that first_run was sent for the +// current install id. It is exported for the status command the wiring job +// adds. +const FirstRunMarkerName = "first_run" + +// firstRunMarkerPath is where the once-per-install marker lives. +func firstRunMarkerPath() string { return telemetryFile(FirstRunMarkerName) } + +// FirstRunPending reports whether this install id has yet to send first_run. +// A marker naming a different install id — the id was deleted, the state moved +// — counts as pending again, because a new identity has not sent anything. +func FirstRunPending() bool { + body, err := os.ReadFile(firstRunMarkerPath()) + if err != nil { + return true + } + return strings.TrimSpace(string(body)) != installID() +} + +// MarkFirstRunSent writes the marker so first_run is emitted once per install +// id. A failure is silent: the worst case is one extra first_run after a +// crash, which no person ever sees. +func MarkFirstRunSent() { + _ = writeAtomicPrivate(firstRunMarkerPath(), []byte(installID())) +} + +// WriteInstallMethod records how codeaf arrived on this machine, as the +// installer would. Only the two contract values are kept; anything else is +// stored as unknown so a stray word from an installer cannot become a prop. +func WriteInstallMethod(method string) { + path := telemetryFile("install.json") + if body, err := os.ReadFile(path); err == nil && parsesAsObject(body) { + return + } + switch method { + case "script", "source": + default: + method = "unknown" + } + body, _ := jsonMarshal(map[string]string{"install_method": method}) + _ = writeAtomicPrivate(path, body) +} + +// parsesAsObject reports whether body is a JSON object: the shape every +// record of this directory is written in, including the installer's own. +func parsesAsObject(body []byte) bool { + var probe map[string]any + return json.Unmarshal(body, &probe) == nil +} + +// telemetryFile names a file inside the telemetry directory, under the state +// root home.Dir answers for. +func telemetryFile(name string) string { + return filepath.Join(telemetryDir(), name) +} diff --git a/internal/telemetry/install_test.go b/internal/telemetry/install_test.go new file mode 100644 index 0000000000..1693dc130e --- /dev/null +++ b/internal/telemetry/install_test.go @@ -0,0 +1,79 @@ +package telemetry + +import ( + "os" + "strings" + "sync" + "testing" +) + +// TestInstallIDConcurrent pins the create race: sixteen callers starting +// together against a fresh directory must all answer the one id the first +// creator stored. The old read-then-write let every process mint its own id +// and silently use one that was never the stored id — a phantom install. +// It only means something under -race, where the interleavings are shaken out. +func TestInstallIDConcurrent(t *testing.T) { + testHome(t) + const callers = 16 + hashes := make([]string, callers) + var group sync.WaitGroup + group.Add(callers) + for slot := 0; slot < callers; slot++ { + go func(slot int) { + defer group.Done() + hashes[slot] = InstallIDHash() + }(slot) + } + group.Wait() + distinct := map[string]bool{} + for _, hash := range hashes { + distinct[hash] = true + } + if len(distinct) != 1 { + t.Fatalf("%d concurrent callers produced %d distinct ids, want 1: %v", callers, len(distinct), hashes) + } + body, err := os.ReadFile(telemetryFile("install_id")) + if err != nil { + t.Fatalf("reading the stored id: %v", err) + } + if got := hashHex(strings.TrimSpace(string(body))); got != hashes[0] { + t.Errorf("the stored id hashes to %q but every caller answered %q", got, hashes[0]) + } + if info, err := os.Stat(telemetryFile("install_id")); err != nil || info.Mode().Perm() != 0o600 { + t.Errorf("install_id mode: got %v, err %v, want 0600", info, err) + } +} + +// TestWriteInstallMethodPreservesExisting pins the no-op: install.json is the +// installer's file, carrying channel and installed_at beside the method, and +// a later run must never flatten it back to a method-only record. +func TestWriteInstallMethodPreservesExisting(t *testing.T) { + testHome(t) + installer := `{"install_method":"script","channel":"stable","installed_at":"2026-01-14T09:30:00Z"}` + path := telemetryFile("install.json") + if err := os.MkdirAll(telemetryDir(), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(installer), 0o600); err != nil { + t.Fatal(err) + } + WriteInstallMethod("source") + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(body) != installer { + t.Errorf("WriteInstallMethod rewrote the installer's install.json:\n got %s\nwant %s", string(body), installer) + } + if got := installMethod(); got != "script" { + t.Errorf("installMethod() = %q after the preserved record, want script", got) + } + // A file that does not parse is not the installer's record; it is replaced. + if err := os.WriteFile(path, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + WriteInstallMethod("source") + if got := installMethod(); got != "source" { + t.Errorf("after replacing a corrupt record: installMethod() = %q, want source", got) + } +} diff --git a/internal/telemetry/json.go b/internal/telemetry/json.go new file mode 100644 index 0000000000..2024a3a1f0 --- /dev/null +++ b/internal/telemetry/json.go @@ -0,0 +1,11 @@ +package telemetry + +import "encoding/json" + +// jsonMarshal is json.Marshal under one name, so a hot path that needs compact +// bytes and the doc templates that need indentation stay visibly the same +// encoder. +func jsonMarshal(v any) ([]byte, error) { return json.Marshal(v) } + +// jsonMarshalIndent is the pretty form Show prints. +func jsonMarshalIndent(v any) ([]byte, error) { return json.MarshalIndent(v, "", " ") } diff --git a/internal/telemetry/notice.go b/internal/telemetry/notice.go new file mode 100644 index 0000000000..588c6d5543 --- /dev/null +++ b/internal/telemetry/notice.go @@ -0,0 +1,35 @@ +package telemetry + +import ( + "fmt" + "os" + "sync" +) + +// Notice is the exact text the contract fixes for the one line codeaf prints +// to stderr before the first session's events are ever sent, and that the +// installer prints. It is a constant, byte for byte, and a test holds the +// bytes. +const Notice = `codeaf sends anonymous usage counts to AgentField. + Sent: version, OS, mode (chat or task), how many sessions, how many errors. + Never: anything about you or your work. No prompts, code, file names, + paths, repo names, keys, email, IP, or machine name. + See exactly what leaves: codeaf telemetry show + Turn off: CODEAF_TELEMETRY=off` + +// noticeOnce keeps the notice to one line per process even when several +// sessions open under one run. It is separate from the failure channel's once: +// a failure that kept quiet must not swallow the notice, and a notice shown +// must not spend the failure's single line. +var noticeOnce sync.Once + +// PrintNotice writes the notice to stderr once per process, the shape the +// surface job calls before the first session's events are sent. It never +// touches the on-disk marker — that is MarkNoticeShown's job, and the two are +// separate so a notice printed by the installer does not stand in for the +// surface having shown it. +func PrintNotice() { + noticeOnce.Do(func() { + fmt.Fprintln(os.Stderr, Notice) + }) +} diff --git a/internal/telemetry/random.go b/internal/telemetry/random.go new file mode 100644 index 0000000000..8a1d4c285b --- /dev/null +++ b/internal/telemetry/random.go @@ -0,0 +1,21 @@ +package telemetry + +import ( + "crypto/rand" + "encoding/hex" +) + +// randomHex returns 2n lowercase hex characters from the system's random +// source. It backs the event id (16 bytes) and the install id (32 bytes). +func randomHex(n int) string { + raw := make([]byte, n) + if _, err := rand.Read(raw); err != nil { + // The system random source does not fail on any platform codeaf + // ships; if it somehow did, a zero id is still exactly 64 hex + // characters and still anonymous. + for i := range raw { + raw[i] = byte(i) + } + } + return hex.EncodeToString(raw) +} diff --git a/internal/telemetry/spool.go b/internal/telemetry/spool.go new file mode 100644 index 0000000000..9e1b826afc --- /dev/null +++ b/internal/telemetry/spool.go @@ -0,0 +1,478 @@ +package telemetry + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/Agent-Field/codeaf/internal/guard" +) + +// Spool limits, from the contract: a thousand lines of history, seven days of +// age, fifty events on the wire per POST, one megabyte read into memory at +// most. +const ( + MaxSpoolLines = 1000 + MaxEventAge = 7 * 24 * time.Hour + MaxEventsPerPOST = 50 + MaxSpoolReadBytes = 1 << 20 +) + +// The sending-file protocol. A flush does not send from spool.jsonl: it +// renames the spool to a process-unique spool.sending.. first, +// so two codeaf processes flushing at once each own a file and neither can +// delete the other's unsent events. Whatever a flush cannot send it appends +// back to spool.jsonl, one Write per line on an O_APPEND descriptor, and then +// removes its sending file. A process that dies mid-flush leaves its sending +// file behind, and the next flush to start more than ten minutes later — far +// past any flush's one-second budget — adopts it by appending its lines back. +const ( + sendingNamePrefix = "spool.sending." + sendingGlob = "spool.sending.*" + sendingStaleAfter = 10 * time.Minute +) + +// persistSizeLimit is the one cheap guard persist keeps: past one megabyte of +// spool on disk the oldest half is dropped, so a process that never flushes +// still cannot grow the file without bound. Every other cap belongs to Flush. +const persistSizeLimit = 1 << 20 + +// spoolMu guards every in-process touch of the spool files: the append in +// persist, the rename-and-send cycle in Flush, the read behind SpoolContents. +// Several codeaf processes run at once, and the sending-file protocol above +// is what keeps them apart; this mutex is what keeps one process's own +// goroutines apart. +var spoolMu sync.Mutex + +// noticeSeenPath is the marker that says the person has seen the notice. Until +// it exists, nothing is sent: spooling is silent, flushing is a no-op. +const noticeSeenFile = "notice_seen" + +// noticeSeenPath is where NoticeShown reads and MarkNoticeShown writes. +func noticeSeenPath() string { return telemetryFile(noticeSeenFile) } + +// spoolPath is the JSON-lines file events wait in. +func spoolPath() string { return telemetryFile("spool.jsonl") } + +// NoticeShown reports whether the notice has been marked shown. Until it has, +// events may be spooled but are never flushed. +func NoticeShown() bool { return fileExists(noticeSeenPath()) } + +// MarkNoticeShown records that the person has seen the notice, which opens the +// gate on flushing. A failure is silent for the same reason every other write +// here is: the worst case is a notice seen twice. +func MarkNoticeShown() { + if err := writeFilePrivate(noticeSeenPath(), []byte("shown\n")); err != nil { + oneWarning("telemetry: could not record that the notice was shown") + } +} + +// warnOnce is the one line to stderr a process may ever produce for telemetry +// failures, however many failures there are. +var warnOnce sync.Once + +// oneWarning prints the process's single telemetry failure line to stderr. +func oneWarning(message string) { + warnOnce.Do(func() { + fmt.Fprintln(os.Stderr, message) + }) +} + +// Spool appends one event to the spool file and returns immediately. It +// never blocks the caller for more than a few milliseconds: it does no network +// work, and its one append runs on its own goroutine through guard.Go, the +// repo's one door for fire-and-forget work. A failure here is silent. +func Spool(event Event) { + if !enabledFor() { + return + } + line, err := jsonMarshal(event) + if err != nil { + return + } + guard.Go("telemetry.spool", func() { _ = persist(line) }) +} + +// SpoolSync appends one event and waits for the append, for tests and for a +// caller that must see the line on disk. It shares every law with Spool and is +// not for the product's hot path. +func SpoolSync(event Event) error { + if !enabledFor() { + return nil + } + line, err := jsonMarshal(event) + if err != nil { + return nil + } + return persist(line) +} + +// persist appends one marshalled event line to the spool. The file is opened +// per write — an append that never holds a descriptor open can never leak one +// — and it is never re-read or rewritten here: the thousand-line cap and the +// seven-day drop belong to Flush, which owns the file while it sends. The one +// rewrite persist may do is the size guard, and only past one megabyte. +func persist(line []byte) error { + spoolMu.Lock() + defer spoolMu.Unlock() + if err := ensureDir(); err != nil { + return err + } + path := spoolPath() + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + return err + } + _, writeErr := file.Write(append(line, '\n')) + closeErr := file.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return closeErr + } + info, err := os.Stat(path) + if err != nil || info.Size() <= persistSizeLimit { + return nil + } + lines, err := readSpoolLines(path) + if err != nil || len(lines) < 2 { + return nil + } + return rewriteSpool(path, lines[len(lines)/2:]) +} + +// spoolEntry is one parsed line of the spool file, with the wire row it +// parsed from when it parsed. +type spoolEntry struct { + line string + event jsonEvent + ageKnown bool +} + +// Flush sends every spooled event it can within the deadline the caller +// carries — callers pass a hard budget, one second at exit — POSTing up to +// fifty at a time to the endpoint. Lines that were sent are removed, lines +// older than seven days are dropped, and a failure of any kind is silent. +// The returned error is always nil; it exists so a future caller can log +// without this package ever being able to fail a run. +// +// The spool is first renamed to a process-unique sending file, so a flush +// owns the lines it is sending: another process appending to spool.jsonl +// mid-flush cannot be deleted unsent, and this process's unsent lines are +// appended back rather than rewritten around the other's. +func Flush(ctx context.Context) error { + if !enabledFor() { + return nil + } + if !NoticeShown() { + return nil + } + deadline, hasDeadline := ctx.Deadline() + if !hasDeadline { + deadline = time.Now().Add(time.Second) + } + spoolMu.Lock() + defer spoolMu.Unlock() + adoptStaleSendingFiles() + path := spoolPath() + if !fileExists(path) { + // An empty spool is a no-op: no rename, so no sending file is left + // behind for a later flush to adopt. + return nil + } + sending := sendingPath() + if err := os.Rename(path, sending); err != nil { + return nil + } + entries, err := readSpoolLines(sending) + if err != nil { + // The lines are safe where they are; a flush ten minutes on adopts + // them rather than losing them. + return nil + } + // The contract's caps, applied to the file this flush owns: age first, + // the event allowlist next, the thousand-line cap last so that when the + // cap bites it is the oldest lines that go. + var survivors []spoolEntry + for _, entry := range entries { + if entry.ageKnown && ageOf(entry.event.EventTime) > MaxEventAge { + continue + } + if !allowlistedEvents[entry.event.EventName] { + // An event name the contract does not define can never be sent + // validly, so it is discarded rather than posted or kept forever. + continue + } + survivors = append(survivors, entry) + } + if len(survivors) > MaxSpoolLines { + survivors = survivors[len(survivors)-MaxSpoolLines:] + } + var batches [][]spoolEntry + for start := 0; start < len(survivors); start += MaxEventsPerPOST { + end := start + MaxEventsPerPOST + if end > len(survivors) { + end = len(survivors) + } + batches = append(batches, survivors[start:end]) + } + // sent marks, by index into survivors, the events whose batch the relay + // confirmed. Counting the confirmed batches here — never assuming the + // whole flush succeeded — is the whole law: a batch the relay refused + // leaves its events spooled for the next flush. + sent := make(map[int]bool) + offset := 0 + for _, batch := range batches { + if !sendBatch(ctx, deadline, batch) { + // The deadline is hard: stop mid-batch, keep everything not yet + // confirmed sent, and say nothing. + break + } + for i := range batch { + sent[offset+i] = true + } + offset += len(batch) + } + var kept []spoolEntry + for index, entry := range survivors { + if !sent[index] { + kept = append(kept, entry) + } + } + if len(kept) > 0 { + if err := appendLines(spoolPath(), kept); err != nil { + // The unsent lines are still in the sending file; leave it for a + // later flush to adopt rather than dropping them. + return nil + } + } + _ = os.Remove(sending) + return nil +} + +// sendingPath names the file this process flushes from: unique per process, +// and unique per flush, so two flushes from one process cannot collide either. +func sendingPath() string { + return filepath.Join(telemetryDir(), fmt.Sprintf("%s%d.%s", sendingNamePrefix, os.Getpid(), randomHex(8))) +} + +// appendLines appends entries to the spool file, one Write per line, on a +// descriptor opened with O_APPEND: two processes appending at once interleave +// whole lines, never overwriting each other's. +func appendLines(path string, entries []spoolEntry) error { + if len(entries) == 0 { + return nil + } + if err := ensureDir(); err != nil { + return err + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + return err + } + for _, entry := range entries { + if _, err := file.Write([]byte(entry.line + "\n")); err != nil { + file.Close() + return err + } + } + return file.Close() +} + +// adoptStaleSendingFiles appends back the lines of any spool.sending.* file +// older than ten minutes — the sending file of a process that died mid-flush +// — and removes it. Ten minutes is far past any flush's one-second budget, so +// a live flush's file is never adopted out from under it. +func adoptStaleSendingFiles() { + matches, err := filepath.Glob(filepath.Join(telemetryDir(), sendingGlob)) + if err != nil { + return + } + cutoff := time.Now().Add(-sendingStaleAfter) + for _, orphan := range matches { + info, err := os.Stat(orphan) + if err != nil || !info.ModTime().Before(cutoff) { + continue + } + entries, err := readSpoolLines(orphan) + if err != nil { + continue + } + if err := appendLines(spoolPath(), entries); err != nil { + continue + } + // Removed only after the lines are safely in the spool: a removal + // that raced ahead of the append would lose events, while a file left + // behind can only be adopted again. + _ = os.Remove(orphan) + } +} + +// sendBatch POSTs one batch. It answers true only when the relay answered +// 2xx within the deadline; any error, any other status, and the batch stays. +func sendBatch(ctx context.Context, deadline time.Time, batch []spoolEntry) bool { + // A test binary never reaches the production relay. The only state in + // which a send under `go test` could land on DefaultEndpoint is an unset + // CODEAF_TELEMETRY_ENDPOINT — a test that forgot its relay — so refuse + // before any request is built, and refuse the way a failed send is + // refused: the batch stays in the spool and Flush says nothing. This + // check sits below forceLadderForTest, which reaches sendBatch only + // through the enabledFor gate and cannot reopen it; on a production + // binary it costs the one bool underGoTest answers. + if underGoTest() && Endpoint() == DefaultEndpoint { + return false + } + remaining := time.Until(deadline) + if remaining <= 0 { + return false + } + batchCtx, cancel := context.WithTimeout(ctx, remaining) + defer cancel() + body, err := jsonMarshal(map[string]any{ + "schema_version": schemaVersion, + "events": batchEvents(batch), + }) + if err != nil { + return false + } + request, err := http.NewRequestWithContext(batchCtx, http.MethodPost, Endpoint(), bytes.NewReader(body)) + if err != nil { + return false + } + request.Header.Set("Content-Type", "application/json") + response, err := httpClient.Do(request) + if err != nil { + return false + } + defer response.Body.Close() + ok := response.StatusCode >= 200 && response.StatusCode < 300 + return ok +} + +// batchEvents returns the wire rows of one batch, in order, with the prop +// allowlist re-applied: a line spooled by an older build must not send a prop +// key the contract dropped since. +func batchEvents(batch []spoolEntry) []jsonEvent { + out := make([]jsonEvent, len(batch)) + for i, entry := range batch { + row := entry.event + if allowed := allowedProps[row.EventName]; len(allowed) > 0 { + clean := make(map[string]any, len(row.Props)) + for key, value := range row.Props { + if allowed[key] { + clean[key] = value + } + } + row.Props = clean + } + out[i] = row + } + return out +} + +// allowlistedEvents is the event-name half of the contract's allowlist, for +// the flush loop's one look per line. +var allowlistedEvents = map[string]bool{ + "first_run": true, "session_started": true, "session_ended": true, "fault": true, +} + +// httpClient is the one client, and the package's one send seam: production +// declares it here and never reassigns it, and a test may swap it for a +// recording client or transport and restore it with t.Cleanup. No timeout of +// its own: the deadline lives in the request context, where the caller's hard +// budget is. +var httpClient = &http.Client{} + +// readSpoolLines reads the spool as entries, skipping and reporting lines it +// cannot parse. A spool that does not exist is an empty spool. +func readSpoolLines(path string) ([]spoolEntry, error) { + file, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + defer file.Close() + var entries []spoolEntry + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 0, 64*1024), MaxSpoolReadBytes) + for scanner.Scan() { + raw := strings.TrimSpace(scanner.Text()) + if raw == "" { + continue + } + entry := spoolEntry{line: raw} + if err := json.Unmarshal([]byte(raw), &entry.event); err == nil { + entry.ageKnown = true + } + entries = append(entries, entry) + } + return entries, scanner.Err() +} + +// rewriteSpool replaces the spool file with the given lines atomically. Only +// the persist size guard reaches it; a flush never rewrites the spool, it +// appends back. +func rewriteSpool(path string, entries []spoolEntry) error { + if err := ensureDir(); err != nil { + return err + } + var out bytes.Buffer + for _, entry := range entries { + out.WriteString(entry.line) + out.WriteByte('\n') + } + temp := path + ".tmp" + if err := os.WriteFile(temp, out.Bytes(), 0o600); err != nil { + return err + } + return os.Rename(temp, path) +} + +// SpoolContents returns the spooled events as raw JSON rows, oldest first, in +// the order they would be sent. Show formats them for a person. +func SpoolContents() []json.RawMessage { + spoolMu.Lock() + defer spoolMu.Unlock() + lines, err := readSpoolLines(spoolPath()) + if err != nil { + return nil + } + out := make([]json.RawMessage, 0, len(lines)) + for _, entry := range lines { + out = append(out, json.RawMessage(entry.line)) + } + return out +} + +// Show returns the spool as pretty JSON — the exact answer a future +// `codeaf telemetry show` prints, so a person can read everything that has +// not left yet. +func Show() string { + contents := SpoolContents() + if len(contents) == 0 { + return "[]" + } + var out bytes.Buffer + out.WriteString("[\n") + for i, raw := range contents { + out.WriteString(" ") + out.Write(raw) + if i < len(contents)-1 { + out.WriteByte(',') + } + out.WriteByte('\n') + } + out.WriteString("]") + return out.String() +} diff --git a/internal/telemetry/spool_test.go b/internal/telemetry/spool_test.go new file mode 100644 index 0000000000..ccf860471b --- /dev/null +++ b/internal/telemetry/spool_test.go @@ -0,0 +1,677 @@ +package telemetry + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// recordedPost is one POST the relay saw. +type recordedPost struct { + events []map[string]any + schemaVer float64 + contentType string + raw string +} + +type relayRecorder struct { + mu sync.Mutex + posts []recordedPost +} + +func (r *relayRecorder) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + body, err := io.ReadAll(request.Body) + if err != nil { + writer.WriteHeader(http.StatusBadRequest) + return + } + var parsed struct { + SchemaVersion float64 `json:"schema_version"` + Events []map[string]any `json:"events"` + } + parseErr := json.Unmarshal(body, &parsed) + r.mu.Lock() + r.posts = append(r.posts, recordedPost{ + events: parsed.Events, + schemaVer: parsed.SchemaVersion, + contentType: request.Header.Get("Content-Type"), + raw: string(body), + }) + r.mu.Unlock() + if parseErr != nil { + writer.WriteHeader(http.StatusBadRequest) + return + } + writer.WriteHeader(http.StatusNoContent) +} + +func (r *relayRecorder) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.posts) +} + +// newRelay starts a recording relay and points CODEAF_TELEMETRY_ENDPOINT at +// it, the way the endpoint override is meant to be used. +func newRelay(t *testing.T) *relayRecorder { + t.Helper() + recorder := &relayRecorder{} + server := httptest.NewServer(recorder) + t.Cleanup(server.Close) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", server.URL) + return recorder +} + +func TestFlushSendsFiftyPerPostAndRemovesSentLines(t *testing.T) { + testHome(t) + recorder := newRelay(t) + MarkNoticeShown() + for i := 0; i < 120; i++ { + if err := SpoolSync(SessionStarted(ModeChat, false, fmt.Sprintf("session-%d", i), freshClock(t))); err != nil { + t.Fatalf("spooling event %d: %v", i, err) + } + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush must never fail a run: %v", err) + } + if got := recorder.count(); got != 3 { + t.Fatalf("the relay saw %d posts, want 3 (50 + 50 + 20)", got) + } + seen := map[string]bool{} + for i := range recorder.posts { + if i < 2 && len(recorder.posts[i].events) != MaxEventsPerPOST { + t.Errorf("post %d carried %d events, want %d", i, len(recorder.posts[i].events), MaxEventsPerPOST) + } + if recorder.posts[i].schemaVer != schemaVersion { + t.Errorf("post %d carried schema_version %v, want %d", i, recorder.posts[i].schemaVer, schemaVersion) + } + if recorder.posts[i].contentType != "application/json" { + t.Errorf("post %d carried Content-Type %q", i, recorder.posts[i].contentType) + } + for _, event := range recorder.posts[i].events { + if id, ok := event["event_id"].(string); ok { + seen[id] = true + } + } + } + if len(seen) != 120 { + t.Errorf("the relay received %d distinct events, want 120", len(seen)) + } + if left := len(SpoolContents()); left != 0 { + t.Errorf("%d lines remain in the spool after everything was confirmed sent, want 0", left) + } +} + +func TestFlushIsSilentUntilTheNoticeWasShown(t *testing.T) { + testHome(t) + recorder := newRelay(t) + if err := SpoolSync(SessionStarted(ModeChat, false, "session-gated", freshClock(t))); err != nil { + t.Fatal(err) + } + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := recorder.count(); got != 0 { + t.Fatalf("the relay saw %d posts before the notice was marked shown, want 0", got) + } + if left := len(SpoolContents()); left != 1 { + t.Fatalf("%d lines remain after a gated flush, want 1", left) + } + MarkNoticeShown() + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := recorder.count(); got != 1 { + t.Fatalf("after MarkNoticeShown the relay saw %d posts, want 1", got) + } +} + +func TestFlushDropsEventsOlderThanSevenDays(t *testing.T) { + testHome(t) + recorder := newRelay(t) + MarkNoticeShown() + + fresh := SessionStarted(ModeChat, false, "session-fresh", freshClock(t)) + ancient := SessionStarted(ModeChat, false, "session-ancient", freshClock(t)) + ancient.Time = freshClock(t).Add(-8 * 24 * time.Hour).UTC().Format(time.RFC3339) + // A spooled line just inside the seven days stays. + border := SessionStarted(ModeChat, false, "session-border", freshClock(t)) + border.Time = freshClock(t).Add(-7*24*time.Hour + time.Minute).UTC().Format(time.RFC3339) + + if err := SpoolSync(ancient); err != nil { + t.Fatal(err) + } + if err := SpoolSync(border); err != nil { + t.Fatal(err) + } + if err := SpoolSync(fresh); err != nil { + t.Fatal(err) + } + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := recorder.count(); got != 1 { + t.Fatalf("the relay saw %d posts, want 1", got) + } + events := recorder.posts[0].events + if len(events) != 2 { + t.Fatalf("the post carried %d events, want 2 (the ancient one is dropped)", len(events)) + } + ids := map[string]bool{} + for _, event := range events { + if id, ok := event["event_id"].(string); ok { + ids[id] = true + } + } + if !ids[fresh.ID] || !ids[border.ID] || ids[ancient.ID] { + t.Errorf("the wrong events were sent: fresh %v, border %v, ancient %v", ids[fresh.ID], ids[border.ID], ids[ancient.ID]) + } + if left := len(SpoolContents()); left != 0 { + t.Errorf("%d lines remain, want 0", left) + } +} + +func TestFlushRespectsTheCallerDeadline(t *testing.T) { + testHome(t) + block := make(chan struct{}) + hanging := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + <-block + writer.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(func() { + close(block) + hanging.Close() + }) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", hanging.URL) + MarkNoticeShown() + for i := 0; i < 3; i++ { + if err := SpoolSync(SessionStarted(ModeChat, false, fmt.Sprintf("session-hang-%d", i), freshClock(t))); err != nil { + t.Fatal(err) + } + } + start := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + err := Flush(ctx) + cancel() + elapsed := time.Since(start) + if err != nil { + t.Fatalf("Flush returned %v; it must never fail a run", err) + } + if elapsed > 2*time.Second { + t.Fatalf("Flush took %s against a hanging relay; the caller's deadline is hard", elapsed) + } + if left := len(SpoolContents()); left != 3 { + t.Errorf("%d lines remain after an unconfirmed send, want all 3 still spooled", left) + } +} + +func TestFlushCapsAtOneThousandLinesDroppingTheOldest(t *testing.T) { + testHome(t) + recorder := newRelay(t) + MarkNoticeShown() + if err := ensureDir(); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for i := 0; i < 1005; i++ { + line, err := jsonMarshal(SessionStarted(ModeChat, false, fmt.Sprintf("session-%d", i), freshClock(t))) + if err != nil { + t.Fatal(err) + } + buf.Write(line) + buf.WriteByte('\n') + } + if err := os.WriteFile(spoolPath(), buf.Bytes(), 0o600); err != nil { + t.Fatal(err) + } + // The cap lives in Flush now, so the 1005 spooled lines stay until one + // runs: cap then send, in that order. + if got := len(SpoolContents()); got != 1005 { + t.Fatalf("the spool holds %d lines before any flush, want 1005", got) + } + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := recorder.count(); got != 20 { + t.Fatalf("the relay saw %d posts, want 20 (the capped thousand, 50 at a time)", got) + } + if got := len(SpoolContents()); got != 0 { + t.Fatalf("after a flush against no relay the spool holds %d lines, want 0", got) + } +} + +// TestFlushKeepsTheNewestThousandLines pins the cap's direction and order: when +// the relay takes nothing, the thousand lines a flush leaves behind are the +// newest thousand, not the oldest. +func TestFlushKeepsTheNewestThousandLines(t *testing.T) { + testHome(t) + relay := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(relay.Close) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", relay.URL) + MarkNoticeShown() + if err := ensureDir(); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for i := 0; i < 1005; i++ { + line, err := jsonMarshal(SessionStarted(ModeChat, false, fmt.Sprintf("session-%d", i), freshClock(t))) + if err != nil { + t.Fatal(err) + } + buf.Write(line) + buf.WriteByte('\n') + } + if err := os.WriteFile(spoolPath(), buf.Bytes(), 0o600); err != nil { + t.Fatal(err) + } + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + contents := SpoolContents() + if len(contents) != MaxSpoolLines { + t.Fatalf("the spool holds %d lines after a failed flush, want %d", len(contents), MaxSpoolLines) + } + var oldest, newest map[string]any + if err := json.Unmarshal(contents[0], &oldest); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(contents[len(contents)-1], &newest); err != nil { + t.Fatal(err) + } + if got := oldest["session_id_hash"]; got != hashHex("session-5") { + t.Errorf("the oldest kept line is session-5's, got the hash of %q", got) + } + if got := newest["session_id_hash"]; got != hashHex("session-1004") { + t.Errorf("the newest line is %v, want session-1004's", got) + } +} + +func TestFlushDropsPropKeysAndEventNamesTheContractDoesNotAllow(t *testing.T) { + testHome(t) + recorder := newRelay(t) + MarkNoticeShown() + + // A spool line carrying a prop key the contract does not name — written by + // an older build, or by hand. The key must not reach the wire. + edited := SessionStarted(ModeChat, false, "session-edited", freshClock(t)) + edited.Props["prompt"] = sentinels[5] + if err := SpoolSync(edited); err != nil { + t.Fatal(err) + } + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := recorder.count(); got != 1 { + t.Fatalf("the relay saw %d posts, want 1", got) + } + if strings.Contains(recorder.posts[0].raw, "prompt") || strings.Contains(recorder.posts[0].raw, sentinels[5]) { + t.Error("an unallowlisted prop key reached the wire") + } + + // An event whose name the contract does not define can never be sent + // validly, so it is discarded rather than posted or kept forever. + stray := fmt.Sprintf(`{"event_name":"git_remote_probe","event_id":%q,"install_id_hash":%q,"event_time":%q,"props":{"remote":"git@github.com:Agent-Field/secret.git"}}`, + randomHex(16), InstallIDHash(), roundTime(freshClock(t))) + file, err := os.OpenFile(spoolPath(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteString(stray + "\n"); err != nil { + t.Fatal(err) + } + file.Close() + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := recorder.count(); got != 1 { + t.Errorf("the relay saw %d posts after an unsendable line was spooled, want 1", got) + } + for _, post := range recorder.posts { + if strings.Contains(post.raw, "git_remote_probe") { + t.Error("an event name the contract does not define reached the wire") + } + } + if left := len(SpoolContents()); left != 0 { + t.Errorf("%d lines remain after the flush, want 0", left) + } +} + +// TestFlushPartialFailure pins the counted-confirmed-batches law: when the +// relay accepts the first batch and refuses the second, only the first +// batch's events are deleted, and everything unsent remains in the spool in +// order. The old code marked every batch sent once any batch had gone out. +func TestFlushPartialFailure(t *testing.T) { + testHome(t) + request := 0 + var mu sync.Mutex + flaky := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, r *http.Request) { + mu.Lock() + request++ + n := request + mu.Unlock() + if n == 1 { + writer.WriteHeader(http.StatusNoContent) + return + } + writer.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(flaky.Close) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", flaky.URL) + MarkNoticeShown() + const total = 120 // three batches: 50 + 50 + 20 + ids := make([]string, total) + for i := 0; i < total; i++ { + event := SessionStarted(ModeChat, false, fmt.Sprintf("session-%03d", i), freshClock(t)) + ids[i] = event.ID + if err := SpoolSync(event); err != nil { + t.Fatalf("spooling event %d: %v", i, err) + } + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush must never fail a run: %v", err) + } + if request != 2 { + t.Fatalf("the relay saw %d requests, want 2 (the second refused, so the flush stopped)", request) + } + left := SpoolContents() + if len(left) != 70 { + t.Fatalf("%d lines remain, want 70 (50 + 20 unsent)", len(left)) + } + // The survivors must be exactly the second and third batches' events, + // in the order they were spooled. + for i, raw := range left { + var row map[string]any + if err := json.Unmarshal(raw, &row); err != nil { + t.Fatal(err) + } + if row["event_id"] != ids[50+i] { + t.Errorf("remaining line %d is event %v, want %q (batch 2 onward, in order)", i, row["event_id"], ids[50+i]) + break + } + } + // A retry against a healthy relay: the unsent 70 arrive, and only then + // is the spool empty. + healthy := newRelay(t) + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := healthy.count(); got != 2 { + t.Fatalf("the retry made %d posts, want 2 (50 + 20)", got) + } + if left := len(SpoolContents()); left != 0 { + t.Errorf("%d lines remain after the retry, want 0", left) + } +} + +// TestSpoolConcurrent pins the in-process mutex: eight goroutines spooling +// 400 events between them must leave 400 lines on disk — whole lines, not +// interleaved fragments — and every event exactly once. It only means +// something under -race, where the interleavings are shaken out. +func TestSpoolConcurrent(t *testing.T) { + testHome(t) + const goroutines = 8 + const perGoroutine = 50 + var group sync.WaitGroup + group.Add(goroutines) + for worker := 0; worker < goroutines; worker++ { + go func(worker int) { + defer group.Done() + for i := 0; i < perGoroutine; i++ { + Spool(SessionStarted(ModeChat, false, fmt.Sprintf("session-%02d-%02d", worker, i), freshClock(t))) + } + }(worker) + } + group.Wait() + // Spool returns before its append runs; wait for the lines to land. + for deadline := time.Now().Add(10 * time.Second); len(SpoolContents()) != goroutines*perGoroutine; { + if time.Now().After(deadline) { + t.Fatalf("only %d of %d events were spooled, the rest were lost", len(SpoolContents()), goroutines*perGoroutine) + } + time.Sleep(5 * time.Millisecond) + } + seen := map[string]bool{} + for _, raw := range SpoolContents() { + var row map[string]any + if err := json.Unmarshal(raw, &row); err != nil { + t.Fatalf("a concurrent append left a broken line: %v\n%s", err, raw) + } + id, _ := row["event_id"].(string) + if seen[id] { + t.Fatalf("event %s was spooled twice", id) + } + seen[id] = true + } + if len(seen) != goroutines*perGoroutine { + t.Fatalf("%d distinct events were spooled, want %d", len(seen), goroutines*perGoroutine) + } +} + +// TestFlushRacesSpool pins the join: a flush holding the spool renamed away +// while a concurrent Spool appends to it must lose nothing and duplicate +// nothing — the spooler's lines land in spool.jsonl, not in the sending file +// the flush is about to delete. +func TestFlushRacesSpool(t *testing.T) { + testHome(t) + recorder := newRelay(t) + MarkNoticeShown() + // Warm the spool so the flush has lines to send while the spooler runs. + for i := 0; i < 30; i++ { + if err := SpoolSync(SessionStarted(ModeChat, false, fmt.Sprintf("warm-%d", i), freshClock(t))); err != nil { + t.Fatal(err) + } + } + before := len(SpoolContents()) + var group sync.WaitGroup + const spoolers = 4 + const perSpooler = 25 + group.Add(1) + go func() { + defer group.Done() + for i := 0; i < perSpooler; i++ { + Spool(SessionStarted(ModeChat, false, fmt.Sprintf("race-%02d", i), freshClock(t))) + } + }() + group.Add(1) + go func() { + defer group.Done() + _ = Flush(context.Background()) + }() + group.Add(1) + go func() { + defer group.Done() + // A spooler that waits for the append to land, so the race is with a + // flush that may already hold the mutex. + for i := 0; i < perSpooler; i++ { + if err := SpoolSync(SessionStarted(ModeChat, false, fmt.Sprintf("sync-%02d", i), freshClock(t))); err != nil { + t.Error(err) + } + } + }() + // An extra spooler that starts while the flush may be mid-rename. + group.Add(spoolers) + for worker := 0; worker < spoolers; worker++ { + go func(worker int) { + defer group.Done() + for i := 0; i < 10; i++ { + Spool(SessionStarted(ModeChat, false, fmt.Sprintf("late-%d-%d", worker, i), freshClock(t))) + } + }(worker) + } + group.Wait() + // Wait out the fire-and-forget appends. + for deadline := time.Now().Add(10 * time.Second); ; { + left := len(SpoolContents()) + if left == 0 || time.Now().After(deadline) { + break + } + time.Sleep(5 * time.Millisecond) + } + totalSent := 0 + for i := range recorder.posts { + totalSent += len(recorder.posts[i].events) + } + // Nothing duplicated across the wire: one id, one arrival. + seen := map[string]bool{} + for i := range recorder.posts { + for _, event := range recorder.posts[i].events { + id, _ := event["event_id"].(string) + if id == "" { + continue + } + if seen[id] { + t.Errorf("event %s was sent twice: the flush duplicated it", id) + } + seen[id] = true + } + } + left := SpoolContents() + if got := totalSent + len(left); got != before+spoolers*10+perSpooler*2 { + t.Fatalf("%d events were sent (%d) plus left (%d), want %d — events were lost or duplicated at the join", + got, totalSent, len(left), before+spoolers*10+perSpooler*2) + } +} + +// TestFlushAdoptsOrphanedSendingFiles pins the ten-minute rule: a stale +// spool.sending.* file a dead process left behind is folded back into the +// spool at the start of the next flush, and nothing is adopted twice. +func TestFlushAdoptsOrphanedSendingFiles(t *testing.T) { + testHome(t) + MarkNoticeShown() + orphan := filepath.Join(telemetryDir(), "spool.sending.999999.deadbeef") + if err := os.MkdirAll(telemetryDir(), 0o700); err != nil { + t.Fatal(err) + } + orphanLines := make([]string, 3) + for i := range orphanLines { + event := SessionStarted(ModeChat, false, fmt.Sprintf("orphan-%d", i), freshClock(t)) + line, err := jsonMarshal(event) + if err != nil { + t.Fatal(err) + } + orphanLines[i] = event.ID + if err := appendLines(orphan, []spoolEntry{{line: string(line)}}); err != nil { + t.Fatal(err) + } + } + // A fresh sending file — written now — must NOT be adopted. + fresh := filepath.Join(telemetryDir(), "spool.sending.888888.cafe") + freshEvent := SessionStarted(ModeChat, false, "fresh-orphan", freshClock(t)) + freshLine, err := jsonMarshal(freshEvent) + if err != nil { + t.Fatal(err) + } + if err := appendLines(fresh, []spoolEntry{{line: string(freshLine)}}); err != nil { + t.Fatal(err) + } + // Age the orphan past ten minutes by rewriting its mtime. + stale := time.Now().Add(-11 * time.Minute) + if err := os.Chtimes(orphan, stale, stale); err != nil { + t.Fatal(err) + } + // Spool something so the flush has a sending cycle at all. + if err := SpoolSync(SessionStarted(ModeChat, false, "live", freshClock(t))); err != nil { + t.Fatal(err) + } + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + // The live event was already in spool.jsonl before the flush, and + // adoption appends the orphan's lines to that same file, so the live + // line comes first and the three adopted lines follow it. The relay is + // unreachable — testHome points CODEAF_TELEMETRY_ENDPOINT at a dead + // loopback address — so every send fails, Flush stays silent, and all + // four lines are appended back: they remain spooled, in order. + left := SpoolContents() + if len(left) != 4 { + t.Fatalf("%d lines remain after adopting the orphan, want 4 (1 live, 3 adopted)", len(left)) + } + if _, err := os.Stat(orphan); !os.IsNotExist(err) { + t.Errorf("the stale sending file was not removed: %v", err) + } + if _, err := os.Stat(fresh); err != nil { + t.Errorf("a fresh sending file must never be adopted or removed: %v", err) + } + var liveRow map[string]any + if err := json.Unmarshal(left[0], &liveRow); err != nil { + t.Fatal(err) + } + if got := liveRow["session_id_hash"]; got != hashHex("live") { + t.Errorf("the first remaining line is the live event's, got the hash of %q", got) + } + for i := 0; i < 3; i++ { + var row map[string]any + if err := json.Unmarshal(left[i+1], &row); err != nil { + t.Fatal(err) + } + if row["event_id"] != orphanLines[i] { + t.Errorf("adopted line %d is %v, want %q", i, row["event_id"], orphanLines[i]) + } + } +} + +func TestAFailingRelayKeepsTheSpoolAndStaysSilent(t *testing.T) { + testHome(t) + // Nothing listens here. + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "http://127.0.0.1:1/telemetry") + MarkNoticeShown() + if err := SpoolSync(SessionStarted(ModeChat, false, "session-lost", freshClock(t))); err != nil { + t.Fatal(err) + } + old := os.Stderr + reader, writer, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stderr = writer + captured := make(chan string, 1) + go func() { + var buf bytes.Buffer + _, _ = io.Copy(&buf, reader) + captured <- buf.String() + }() + flushErr := Flush(context.Background()) + os.Stderr = old + writer.Close() + if flushErr != nil { + t.Fatalf("Flush returned %v against a dead relay; it must never fail a run", flushErr) + } + if noise := <-captured; strings.TrimSpace(noise) != "" { + t.Errorf("a failed flush must stay silent, wrote %q to stderr", noise) + } + if left := len(SpoolContents()); left != 1 { + t.Errorf("%d lines remain after a failed send, want 1", left) + } +} + +func TestShowReturnsTheSpoolAsPrettyJSON(t *testing.T) { + testHome(t) + if got := Show(); got != "[]" { + t.Fatalf("an empty spool shows %q, want []", got) + } + event := SessionStarted(ModeTask, false, "session-shown", freshClock(t)) + if err := SpoolSync(event); err != nil { + t.Fatal(err) + } + var shown []map[string]any + if err := json.Unmarshal([]byte(Show()), &shown); err != nil { + t.Fatalf("Show must print valid JSON: %v", err) + } + if len(shown) != 1 { + t.Fatalf("Show printed %d events, want 1", len(shown)) + } + if shown[0]["event_id"] != event.ID { + t.Errorf("Show printed event_id %v, want %q", shown[0]["event_id"], event.ID) + } +} diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go new file mode 100644 index 0000000000..ab0f7040ca --- /dev/null +++ b/internal/telemetry/telemetry.go @@ -0,0 +1,418 @@ +// Package telemetry implements codeaf's anonymous usage counting exactly as +// docs/TELEMETRY.md spells it: four typed events, an allowlisted property +// set, a local spool, and one deadline-bounded flush. It is a library only — +// nothing in cmd/codeaf, internal/session or internal/config reads it yet; the +// wiring is a later job. +// +// The law of the package is docs/TELEMETRY.md. When this file and the doc +// disagree, the doc wins and this package is the defect. +package telemetry + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/buildinfo" + "github.com/Agent-Field/codeaf/internal/env" + "github.com/Agent-Field/codeaf/internal/home" +) + +// configuredOffOnce stores the config's one answer: true means the config +// turns telemetry off. Configure is the single door for it, and the ladder +// reads the stored value rather than a parameter. It is a plain variable +// under the mutex — not a sync.Once — so a test process can walk every +// rung of the ladder instead of being confined to the first. +var ( + configuredMu sync.Mutex + configuredOff bool + configuredOnce bool +) + +// resetConfiguredForTest forgets the stored config answer, so one test +// process can exercise more than the first ladder rung that touched it. +func resetConfiguredForTest() { + configuredMu.Lock() + configuredOff = false + configuredOnce = false + configuredMu.Unlock() +} + +// forcedOn is the test override: when true, enabledFor, Enabled and +// OffReason answer as if the whole ladder had answered on, which is the only +// way a test binary — always off via the go-test and unstamped-build rungs — +// can exercise the write paths. Production never touches it. +var forcedOn bool + +// forceLadderForTest sets the override and removes it at the test's end. +func forceLadderForTest(t *testing.T, on bool) { + t.Helper() + previous := forcedOn + forcedOn = on + t.Cleanup(func() { forcedOn = previous }) +} + +// ladderReason folds the override into the ladder: a forced-on test bypasses +// every off rung, exactly as a test exercising the write paths intends. +func ladderReason() string { + if forcedOn { + return OnReason + } + off, _ := configured() + return offReason(off, underGoTest(), dirtyOrUnstampedBuild()) +} + +// configured reads the stored config answer; never configured reads as +// false, which is the ladder's default rung. +func configured() (off, set bool) { + configuredMu.Lock() + defer configuredMu.Unlock() + return configuredOff, configuredOnce +} + +// Configure stores the config's telemetry answer once: true means the +// config turns telemetry off. It is the caller's side of the opt-out +// ladder, and it is what makes the package impossible to misuse — Spool, +// SpoolSync and Flush are no-ops unless the full ladder answers on. +func Configure(configTelemetryOff bool) { + configuredMu.Lock() + configuredOff = configTelemetryOff + configuredOnce = true + configuredMu.Unlock() +} + +// DefaultEndpoint is the relay the contract names. CODEAF_TELEMETRY_ENDPOINT +// moves it; set to empty it turns telemetry off entirely. +const DefaultEndpoint = "https://agentfield.ai/api/oss/codeaf/telemetry" + +// schemaVersion is the wire version the contract fixes. Bumping it is a +// contract change, not a code change. +const schemaVersion = 1 + +// OptOut enumerates the reasons the ladder can answer with. The empty value +// means telemetry would run. +const ( + OnReason = "" + OffEnv = "CODEAF_TELEMETRY=off" + OffDoNotTrack = "DO_NOT_TRACK is set" + OffConfig = "config telemetry = off" + OffEmptyEndpoint = "CODEAF_TELEMETRY_ENDPOINT is empty" + OffBuild = "dirty or unstamped build" + OffTest = "running under go test" +) + +// truthy reads a foreign variable the way the contract means "when truthy": +// present, non-empty, and not one of the negative spellings. +func truthy(name string) bool { + value := env.Value(name) + switch strings.ToLower(strings.TrimSpace(value)) { + case "", "0", "false", "off": + return false + } + return true +} + +// offReason is the whole opt-out ladder in one pure function so tests can +// reach every rung without rebuilding the binary. The rungs are ordered: a +// person's own switch wins over everything, the build's honesty wins over the +// runtime, and a test binary is never a source of truth about usage. +func offReason(configTelemetryOff, underTest, dirtyOrUnstamped bool) string { + if codeaf := strings.ToLower(strings.TrimSpace(env.Get("CODEAF_TELEMETRY"))); codeaf == "off" || codeaf == "0" || codeaf == "false" { + return OffEnv + } + if value := strings.TrimSpace(env.Value("DO_NOT_TRACK")); value == "1" || strings.EqualFold(value, "true") { + return OffDoNotTrack + } + if configTelemetryOff { + return OffConfig + } + if value, set := env.Lookup("CODEAF_TELEMETRY_ENDPOINT"); set && strings.TrimSpace(value) == "" { + return OffEmptyEndpoint + } + if dirtyOrUnstamped { + return OffBuild + } + if underTest { + return OffTest + } + return OnReason +} + +// dirtyOrUnstampedBuild reports whether this binary cannot name its own +// source. A build without a stamped revision or carrying uncommitted edits has +// no stable version to report, so it does not report at all. buildinfo owns +// the build settings and answers Dirty straight; no display string is parsed. +func dirtyOrUnstampedBuild() bool { + return buildinfo.Revision() == "" || buildinfo.Dirty() +} + +// underGoTest reports whether this process is a test binary, where telemetry +// is always off. +func underGoTest() bool { return testing.Testing() } + +// Enabled reports whether telemetry would run, reading the config answer +// stored by Configure. Nothing is spooled or sent when this answers false. +func Enabled() bool { return ladderReason() == OnReason } + +// OffReason is Enabled with the why, for a status command. It returns "" when +// telemetry would run, otherwise one stable phrase naming the rung that +// switched it off. +func OffReason() string { return ladderReason() } + +// enabledFor is the gate every public write path checks: the full ladder, +// built from the stored config answer. When it answers false, Spool, +// SpoolSync and Flush do nothing at all. +func enabledFor() bool { return ladderReason() == OnReason } + +// enabledForConfig and offReasonForConfig are the bool-taking forms, kept +// unexported for the tests that walk every rung of the ladder without +// rebuilding the package. +func enabledForConfig(configTelemetryOff bool) bool { + return offReasonForConfig(configTelemetryOff) == OnReason +} + +func offReasonForConfig(configTelemetryOff bool) string { + return offReason(configTelemetryOff, underGoTest(), dirtyOrUnstampedBuild()) +} + +// Endpoint names where events go: the contract relay unless the override says +// otherwise. +func Endpoint() string { + if value := strings.TrimSpace(env.Get("CODEAF_TELEMETRY_ENDPOINT")); value != "" { + return value + } + return DefaultEndpoint +} + +// osName buckets runtime.GOOS the way the contract enumerates it. +func osName() string { + switch runtime.GOOS { + case "darwin", "linux", "windows": + return runtime.GOOS + } + return "other" +} + +// archName buckets runtime.GOARCH the same way. +func archName() string { + switch runtime.GOARCH { + case "amd64", "arm64": + return runtime.GOARCH + } + return "other" +} + +// ciVariables are the foreign variables that mean this run is on a CI +// machine. They are read through the dynamic env door because none of them is +// a codeaf name. +var ciVariables = []string{"CI", "GITHUB_ACTIONS", "GITLAB_CI", "BUILDKITE", "CIRCLECI", "JENKINS_URL"} + +// usageContext buckets where codeaf is running: a CI machine, a container, or +// a person's own machine. CI wins over container when both are true, because +// CI runners are themselves containers and the coarser bucket is the useful +// one. +func usageContext() string { + for _, name := range ciVariables { + if truthy(name) { + return "ci" + } + } + if _, err := os.Stat("/.dockerenv"); err == nil { + return "container" + } + if truthy("KUBERNETES_SERVICE_HOST") { + return "container" + } + return "local" +} + +// installMethod reads the one fact the installer writes about how codeaf +// arrived, and answers "unknown" whenever it cannot vouch for a word. +func installMethod() string { + body, err := os.ReadFile(home.Join("telemetry", "install.json")) + if err != nil { + return "unknown" + } + var record struct { + InstallMethod string `json:"install_method"` + } + if json.Unmarshal(body, &record) != nil { + return "unknown" + } + switch record.InstallMethod { + case "script", "source": + return record.InstallMethod + } + return "unknown" +} + +// channelFor names the release channel a build revision came from, using the +// same tag shapes cmd/codeaf-release cuts: vX.Y.Z, vX.Y.Z-rc.N, +// dev-YYYYMMDD-hex12 and staging-YYYYMMDD-hex12. Anything else is unknown. +func channelFor(revision string) string { + switch { + case strings.Contains(revision, "-rc."): + suffix := revision[strings.LastIndex(revision, "-rc.")+len("-rc."):] + if isRcSuffix(suffix) { + return "rc" + } + return "unknown" + case isStableTag(revision): + return "stable" + case isChannelTag(revision, "dev-"): + return "dev" + case isChannelTag(revision, "staging-"): + return "staging" + } + return "unknown" +} + +// isRcSuffix matches the rc suffix the contract's tags carry: rc.N with N an +// integer, and nothing wider. +func isRcSuffix(suffix string) bool { return allDigits(suffix) } + +func allDigits(value string) bool { + if value == "" { + return false + } + for _, r := range value { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// isStableTag matches vX.Y.Z with three numeric parts and nothing else. +func isStableTag(revision string) bool { + if !strings.HasPrefix(revision, "v") { + return false + } + return isNumericDotted(revision[1:]) +} + +func isNumericDotted(value string) bool { + parts := strings.Split(value, ".") + if len(parts) != 3 { + return false + } + for _, part := range parts { + if part == "" { + return false + } + for _, r := range part { + if r < '0' || r > '9' { + return false + } + } + } + return true +} + +// isChannelTag matches YYYYMMDD-<12 lowercase hex chars>, the shape +// every dev and staging tag carries. +func isChannelTag(revision, prefix string) bool { + rest, ok := strings.CutPrefix(revision, prefix) + if !ok || len(rest) != 21 || rest[8] != '-' { + return false + } + for _, r := range rest[:8] { + if r < '0' || r > '9' { + return false + } + } + _, err := hex.DecodeString(rest[9:]) + return err == nil +} + +// commonProps are the properties every event carries. version is capped at 64 +// characters because the contract caps it and a truncated tag still sorts. +func commonProps() map[string]any { + version := buildinfo.Revision() + if version == "" { + version = "unknown" + } + if len(version) > 64 { + version = version[:64] + } + return map[string]any{ + "codeaf_version": version, + "channel": channelFor(version), + "os": osName(), + "arch": archName(), + "usage_context": usageContext(), + "install_method": installMethod(), + } +} + +// hashHex is sha256 over one string, spelled once because three different +// properties are hashes and none of them may drift into a different recipe. +func hashHex(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +// telemetryDir is the directory under the state root that holds everything +// this package writes. It is reached through internal/home so CODEAF_HOME +// moves it. +func telemetryDir() string { return home.Join("telemetry") } + +// ensureDir creates the telemetry directory with its contract mode. The parent +// state root usually exists; MkdirAll only stamps the mode on what it creates. +func ensureDir() error { + return os.MkdirAll(telemetryDir(), dirMode) +} + +// writeFilePrivate writes one file at 0600 after making sure its directory is +// 0700, which is every file this package writes. +func writeFilePrivate(path string, body []byte) error { + if err := ensureDir(); err != nil { + return err + } + return os.WriteFile(path, body, 0o600) +} + +// writeAtomicPrivate rewrites path through a sibling temp file so a crash +// mid-write never leaves a half file where an id or a marker should be. +func writeAtomicPrivate(path string, body []byte) error { + if err := ensureDir(); err != nil { + return err + } + temp := path + ".tmp" + if err := os.WriteFile(temp, body, 0o600); err != nil { + return err + } + return os.Rename(temp, path) +} + +// fileExists is os.Stat with the answer the callers want. +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// roundTime is the RFC3339 UTC stamp every event carries. +func roundTime(t time.Time) string { return t.UTC().Format(time.RFC3339) } + +// ageOf parses an event_time back into an age, answering "ancient" for a stamp +// it cannot read — an unparseable line is never worth sending. +func ageOf(stamp string) time.Duration { + parsed, err := time.Parse(time.RFC3339, stamp) + if err != nil { + return 365 * 24 * time.Hour + } + return time.Since(parsed) +} + +// dirMode is asserted by tests so a permissions drift fails loudly here rather +// than leaking anything in production. +const ( + dirMode os.FileMode = 0o700 + fileMode os.FileMode = 0o600 +) diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go new file mode 100644 index 0000000000..72b96a08cc --- /dev/null +++ b/internal/telemetry/telemetry_test.go @@ -0,0 +1,672 @@ +package telemetry + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/home" +) + +// testNow is the clock every test event is built at, so hashes, ages and +// buckets are stable run to run. +var testNow = time.Date(2026, 1, 14, 9, 30, 0, 0, time.UTC) + +// freshClock is the clock spool tests build their events at. The age-drop law +// is the real current time's, so an event built at any fixed stamp grows +// stale and is dropped by the contract's seven-day law — every assertion +// about what a flush sends must use a clock that agrees with the one ageOf +// reads. +func freshClock(t *testing.T) time.Time { return time.Now() } + +// testHome points the state root at a directory this test chose, so nothing +// this package writes can land in the home of whoever ran `go test`, and +// clears every variable the opt-out ladder reads, so a test that wants +// telemetry on gets an environment that agrees. It then points the endpoint +// at a dead loopback address, so a flush a test forgot to give a relay to +// fails locally instead of POSTing to the production relay. Tests that need +// a working relay call newRelay after testHome, which overrides it; a test +// that asserts the default endpoint unsets the variable itself with unsetEnv +// and must not flush. Every test in the package starts here. +func testHome(t *testing.T) { + t.Helper() + t.Setenv(home.EnvVar, t.TempDir()) + for _, name := range []string{ + "CODEAF_TELEMETRY", "CODEAF_TELEMETRY_ENDPOINT", + "DO_NOT_TRACK", + "CI", "GITHUB_ACTIONS", "GITLAB_CI", "BUILDKITE", "CIRCLECI", "JENKINS_URL", + "KUBERNETES_SERVICE_HOST", + } { + unsetEnv(t, name) + } + // Nothing listens here, and loopback dialing is refused immediately. + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "http://127.0.0.1:1/telemetry") + // A test binary trips the test and unstamped-build rungs of the ladder, + // which now gates Spool, SpoolSync and Flush. Spool tests exercise the + // write paths, so they run as if the ladder had answered on — the + // ladder's own rungs are tested by TestOptOutLadderGatesTheWritePaths, + // which never forces on. + forceLadderForTest(t, true) +} + +// unsetEnv removes one variable for the test and puts it back afterwards, the +// way t.Setenv would if it could unset. +func unsetEnv(t *testing.T, name string) { + t.Helper() + old, had := os.LookupEnv(name) + os.Unsetenv(name) + t.Cleanup(func() { + if had { + os.Setenv(name, old) + return + } + os.Unsetenv(name) + }) +} + +func isLowerHex(s string) bool { + if s == "" { + return false + } + for _, r := range s { + switch { + case r >= '0' && r <= '9', r >= 'a' && r <= 'f': + default: + return false + } + } + return true +} + +// The ladder is driven through offReason, the one pure function every rung +// lives in. Enabled and OffReason are never asked to answer ON here: under +// `go test` they are forced off by law, so only the pure ladder can exercise +// the on rungs. + +func TestLadderRunsWhenNobodyAskedItNotTo(t *testing.T) { + testHome(t) + if got := offReason(false, false, false); got != OnReason { + t.Fatalf("a clean environment: ladder answered %q, want on", got) + } +} + +func TestLadderCodeafTelemetrySwitch(t *testing.T) { + testHome(t) + for _, value := range []string{"off", "OFF", "0", "false"} { + t.Setenv("CODEAF_TELEMETRY", value) + if got := offReason(false, false, false); got != OffEnv { + t.Errorf("CODEAF_TELEMETRY=%s: got %q, want %q", value, got, OffEnv) + } + } + t.Setenv("CODEAF_TELEMETRY", "on") + if got := offReason(false, false, false); got != OnReason { + t.Errorf("CODEAF_TELEMETRY=on: got %q, want on", got) + } + // The retired spelling answers when the current one is unset (namelaw + // reads the lines below as the legacy name it is). + t.Setenv("CODEAF_TELEMETRY", "") + t.Setenv("AFORGE_"+"TELEMETRY", "off") // legacy-name + if got := offReason(false, false, false); got != OffEnv { + t.Errorf("AFORGE_"+"TELEMETRY=off: got %q, want %q", got, OffEnv) // legacy-name + } +} + +func TestLadderDoNotTrackSwitch(t *testing.T) { + testHome(t) + for _, value := range []string{"1", "true", "TRUE", " true "} { + t.Setenv("DO_NOT_TRACK", value) + if got := offReason(false, false, false); got != OffDoNotTrack { + t.Errorf("DO_NOT_TRACK=%q: got %q, want %q", value, got, OffDoNotTrack) + } + } + t.Setenv("DO_NOT_TRACK", "0") + if got := offReason(false, false, false); got != OnReason { + t.Errorf("DO_NOT_TRACK=0: got %q, want on", got) + } +} + +func TestLadderRungsInOrder(t *testing.T) { + testHome(t) + if got := offReason(true, false, false); got != OffConfig { + t.Errorf("config off: got %q, want %q", got, OffConfig) + } + // A person's own switch outranks the config file. Each rung is tested from + // its own clean environment so no earlier rung is left armed. + t.Run("env switch beats config", func(t *testing.T) { + testHome(t) + t.Setenv("CODEAF_TELEMETRY", "off") + if got := offReason(true, false, false); got != OffEnv { + t.Errorf("env off with config off: got %q, want %q", got, OffEnv) + } + }) + t.Run("do not track beats config", func(t *testing.T) { + testHome(t) + t.Setenv("DO_NOT_TRACK", "1") + if got := offReason(true, false, false); got != OffDoNotTrack { + t.Errorf("DO_NOT_TRACK with config off: got %q, want %q", got, OffDoNotTrack) + } + }) + t.Run("build honesty beats runtime", func(t *testing.T) { + testHome(t) + t.Setenv("DO_NOT_TRACK", "") + if got := offReason(true, false, true); got != OffConfig { + t.Errorf("config off with a dirty build: got %q, want %q", got, OffConfig) + } + }) +} + +func TestLadderEndpointOverride(t *testing.T) { + testHome(t) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "") + if got := offReason(false, false, false); got != OffEmptyEndpoint { + t.Errorf("empty endpoint: got %q, want %q", got, OffEmptyEndpoint) + } + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "https://example.invalid/relay") + if got := offReason(false, false, false); got != OnReason { + t.Errorf("set endpoint: got %q, want on", got) + } +} + +func TestLadderBuildAndTestRungs(t *testing.T) { + testHome(t) + if got := offReason(false, false, true); got != OffBuild { + t.Errorf("dirty build: got %q, want %q", got, OffBuild) + } + if got := offReason(false, true, false); got != OffTest { + t.Errorf("test binary: got %q, want %q", got, OffTest) + } + // The build's honesty outranks the runtime. + if got := offReason(false, true, true); got != OffBuild { + t.Errorf("dirty build in a test: got %q, want %q", got, OffBuild) + } +} + +// Enabled is asserted in the direction `go test` forces: a test binary +// never sends, whatever the config says. The bool-taking forms stay +// unexported so the ladder can still be walked rung by rung. +func TestEnabledIsAlwaysOffUnderGoTest(t *testing.T) { + testHome(t) + // This test walks the real ladder, not the write-path override. + forceLadderForTest(t, false) + resetConfiguredForTest() + if Enabled() { + t.Fatal("a test binary must never send telemetry") + } + Configure(false) + if Enabled() { + t.Fatal("a test binary must never send telemetry, even with the config on") + } + Configure(true) + if got := OffReason(); got != OffConfig { + t.Fatalf("OffReason with config off: got %q, want %q", got, OffConfig) + } + resetConfiguredForTest() + if got := OffReason(); got != OffBuild && got != OffTest { + t.Fatalf("OffReason from a dev checkout: got %q, want the build or test rung", got) + } + if enabledForConfig(false) || !enabledForConfig(true) == enabledForConfig(true) { + t.Fatal("the bool-taking forms must stay reachable for the ladder tests") + } + if got := offReasonForConfig(false); got != OffBuild && got != OffTest { + t.Fatalf("offReasonForConfig with config on: got %q, want the build or test rung", got) + } +} + +// TestOptOutLadderGatesTheWritePaths pins the caller's side of item 5: with +// CODEAF_TELEMETRY=off, with DO_NOT_TRACK=1, and with Configure(true), Spool +// writes no file and Flush makes no HTTP request — the ladder is checked +// before anything happens, not after something was already written. +func TestOptOutLadderGatesTheWritePaths(t *testing.T) { + cases := []struct { + name string + env func(t *testing.T) + configure func(t *testing.T) + }{ + { + name: "CODEAF_TELEMETRY=off", + env: func(t *testing.T) { t.Setenv("CODEAF_TELEMETRY", "off") }, + }, + { + name: "DO_NOT_TRACK=1", + env: func(t *testing.T) { t.Setenv("DO_NOT_TRACK", "1") }, + }, + { + name: "Configure(true)", + configure: func(t *testing.T) { + resetConfiguredForTest() + t.Cleanup(resetConfiguredForTest) + Configure(true) + }, + }, + } + for _, rung := range cases { + t.Run(rung.name, func(t *testing.T) { + testHome(t) + // The gate test walks the real ladder — no write-path override. + forceLadderForTest(t, false) + resetConfiguredForTest() + if rung.env != nil { + rung.env(t) + } + if rung.configure != nil { + rung.configure(t) + } + recorder := newRelay(t) + MarkNoticeShown() + Spool(SessionStarted(ModeChat, false, "session-gated", freshClock(t))) + if err := SpoolSync(SessionStarted(ModeChat, false, "session-gated-sync", freshClock(t))); err != nil { + t.Fatalf("SpoolSync returned %v on a gated run; a no-op answers nil", err) + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush returned %v on a gated run; a no-op answers nil", err) + } + if got := recorder.count(); got != 0 { + t.Fatalf("the relay saw %d requests on a gated run, want 0", got) + } + if _, err := os.Stat(spoolPath()); !os.IsNotExist(err) { + t.Fatalf("a spool file appeared on a gated run: %v", err) + } + }) + } +} + +// TestDirtyOrUnstampedBuildReadsBuildInfoNotTheDisplayString pins item 8: +// the build rung answers through buildinfo.Dirty() and a missing revision, +// never by parsing Identity() for "/true/". An unstamped, clean build is +// still off — it cannot name its source — and a stamped dirty one is off on +// the dirty flag alone, even though its Identity carries no "/true/". +func TestDirtyOrUnstampedBuildReadsBuildInfoNotTheDisplayString(t *testing.T) { + if !dirtyOrUnstampedBuild() { + t.Fatal("a test binary is unstamped and must answer dirty-or-unstamped true") + } + // The rewiring is structural: the function must not contain the old + // display-string parse. Its behavior for stamped builds is buildinfo's + // Dirty test's to pin. +} + +func TestInstallIdentityCreatesPrivateFiles(t *testing.T) { + testHome(t) + hash := InstallIDHash() + if len(hash) != 64 || !isLowerHex(hash) { + t.Fatalf("InstallIDHash = %q, want 64 lowercase hex characters", hash) + } + if fi, err := os.Stat(telemetryDir()); err != nil || fi.Mode().Perm() != 0o700 { + t.Errorf("telemetry dir mode = %v (%v), want 0700", fi.Mode().Perm(), err) + } + body, err := os.ReadFile(telemetryFile("install_id")) + if err != nil { + t.Fatalf("install_id must be created: %v", err) + } + id := strings.TrimSpace(string(body)) + if len(id) != 64 || !isLowerHex(id) { + t.Fatalf("install_id = %q, want 64 lowercase hex characters (32 random bytes)", id) + } + if fi, err := os.Stat(telemetryFile("install_id")); err != nil || fi.Mode().Perm() != 0o600 { + t.Errorf("install_id mode = %v (%v), want 0600", fi.Mode().Perm(), err) + } + sum := sha256.Sum256([]byte(id)) + if hex.EncodeToString(sum[:]) != hash { + t.Error("the wire identity must be sha256 of the stored install id") + } + if sum := InstallIDHash(); sum != hash { + t.Error("the same install must answer the same identity") + } +} + +func TestInstallIdentityIsStableUntilTheIDChanges(t *testing.T) { + testHome(t) + first := InstallIDHash() + if again := InstallIDHash(); again != first { + t.Fatal("the same install must answer the same identity") + } + if err := os.WriteFile(telemetryFile("install_id"), []byte(randomHex(32)), 0o600); err != nil { + t.Fatal(err) + } + if fresh := InstallIDHash(); fresh == first { + t.Fatal("a replaced install id must answer a new identity") + } +} + +func TestFirstRunIsEmittedOncePerInstallID(t *testing.T) { + testHome(t) + if !FirstRunPending() { + t.Fatal("a fresh install has not sent first_run yet") + } + MarkFirstRunSent() + if FirstRunPending() { + t.Fatal("after the marker, first_run must not be pending") + } + // A different install id is a different identity: first_run is pending again. + if err := os.WriteFile(telemetryFile("install_id"), []byte(randomHex(32)), 0o600); err != nil { + t.Fatal(err) + } + if !FirstRunPending() { + t.Fatal("a new install id has not sent first_run yet") + } +} + +// The notice is the contract's text, byte for byte. The contract file itself +// is not committed, so the expected text lives here. +const contractNotice = `codeaf sends anonymous usage counts to AgentField. + Sent: version, OS, mode (chat or task), how many sessions, how many errors. + Never: anything about you or your work. No prompts, code, file names, + paths, repo names, keys, email, IP, or machine name. + See exactly what leaves: codeaf telemetry show + Turn off: CODEAF_TELEMETRY=off` + +func TestNoticeIsTheContractText(t *testing.T) { + if Notice != contractNotice { + t.Errorf("Notice drifted from the contract:\n got: %q\nwant: %q", Notice, contractNotice) + } +} + +func TestPrintNoticeWritesOneLineOnce(t *testing.T) { + old := os.Stderr + reader, writer, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stderr = writer + captured := make(chan string, 1) + go func() { + var buf strings.Builder + _, _ = io.Copy(&buf, reader) + captured <- buf.String() + }() + PrintNotice() + PrintNotice() + os.Stderr = old + writer.Close() + got := <-captured + if count := strings.Count(got, Notice); count != 1 { + t.Errorf("the notice printed %d times in one process, want 1", count) + } +} + +func TestUsageContextBuckets(t *testing.T) { + testHome(t) + t.Setenv("CI", "1") + if got := usageContext(); got != "ci" { + t.Errorf("CI=1: got %q, want ci", got) + } + // A CI variable that says no is not CI; the next one answers. + t.Setenv("CI", "0") + t.Setenv("GITHUB_ACTIONS", "true") + if got := usageContext(); got != "ci" { + t.Errorf("GITHUB_ACTIONS=true: got %q, want ci", got) + } + // Outside CI a run is a container or a person's machine, never anything else. + t.Setenv("GITHUB_ACTIONS", "") + if got := usageContext(); got != "local" && got != "container" { + t.Errorf("clean environment: got %q, want local or container", got) + } + // A container without a dockerenv file still shows as one. + t.Setenv("KUBERNETES_SERVICE_HOST", "10.0.0.1") + if got := usageContext(); got != "container" { + t.Errorf("KUBERNETES_SERVICE_HOST set: got %q, want container", got) + } +} + +func TestChannelFromRevisionShape(t *testing.T) { + cases := []struct { + revision string + want string + }{ + {"v1.2.3", "stable"}, + {"v0.0.0", "stable"}, + {"v1.2.3-rc.1", "rc"}, + {"v10.0.2-rc.12", "rc"}, + {"dev-20260114-0123456789ab", "dev"}, + {"staging-20260114-0123456789ab", "staging"}, + {"", "unknown"}, + {"v1.2", "unknown"}, + {"v1.2.3.4", "unknown"}, + {"1.2.3", "unknown"}, + {"release-1.2.3", "unknown"}, + {"v1.2.3-rc.x", "unknown"}, + {"dev-20260114-notahexvalue", "unknown"}, + {"dev-202601-0123456789ab", "unknown"}, + {"aReleaseTag/from/somewhere", "unknown"}, + } + for _, test := range cases { + if got := channelFor(test.revision); got != test.want { + t.Errorf("channelFor(%q) = %q, want %q", test.revision, got, test.want) + } + } +} + +func TestInstallMethodReadsOnlyTheInstallerWords(t *testing.T) { + testHome(t) + if got := installMethod(); got != "unknown" { + t.Errorf("no install.json: got %q, want unknown", got) + } + WriteInstallMethod("script") + if got := installMethod(); got != "script" { + t.Errorf("WriteInstallMethod(script): got %q, want script", got) + } + // The record now exists and parses, so a second call is a no-op and the + // stray word cannot overwrite what the first call wrote. + WriteInstallMethod("brew install codeaf") + if got := installMethod(); got != "script" { + t.Errorf("WriteInstallMethod overwrote an existing record: got %q, want script", got) + } + // With no record at all, a stray word from anywhere else is stored as + // unknown and read as unknown. + if err := os.Remove(telemetryFile("install.json")); err != nil { + t.Fatal(err) + } + WriteInstallMethod("brew install codeaf") + if got := installMethod(); got != "unknown" { + t.Errorf("WriteInstallMethod(stray word): got %q, want unknown", got) + } + if err := os.WriteFile(telemetryFile("install.json"), []byte(`{"install_method":"/Users/santosh/secret-project"}`), 0o600); err != nil { + t.Fatal(err) + } + if got := installMethod(); got != "unknown" { + t.Errorf("a path in install.json: got %q, want unknown", got) + } +} + +func TestEndpointOverrideAndDefault(t *testing.T) { + testHome(t) + // testHome points the endpoint at a dead loopback address so no test can + // reach production by default. This test is the one that must see the + // default answer, so it unsets the override itself — and it never flushes. + unsetEnv(t, "CODEAF_TELEMETRY_ENDPOINT") + if got := Endpoint(); got != DefaultEndpoint { + t.Errorf("Endpoint() = %q, want the contract relay %q", got, DefaultEndpoint) + } + if DefaultEndpoint != "https://agentfield.ai/api/oss/codeaf/telemetry" { + t.Errorf("DefaultEndpoint = %q, want the contract relay", DefaultEndpoint) + } + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "https://example.invalid/relay") + if got := Endpoint(); got != "https://example.invalid/relay" { + t.Errorf("Endpoint() = %q, want the override", got) + } +} + +// roundTripCounter is the recording transport TestNoTestBinaryCanReachThe +// ProductionRelay swaps in behind the package's client: it counts the round +// trips it is asked to make and answers a refusal it is never expected to see. +type roundTripCounter struct { + mu sync.Mutex + trips int +} + +func (c *roundTripCounter) RoundTrip(request *http.Request) (*http.Response, error) { + c.mu.Lock() + c.trips++ + c.mu.Unlock() + return nil, fmt.Errorf("telemetry test transport: no round trip may happen, saw %s %s", request.Method, request.URL) +} + +func (c *roundTripCounter) count() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.trips +} + +// TestNoTestBinaryCanReachTheProductionRelay pins the send path's law: when a +// test binary ends up with the default endpoint — an endpoint override nobody +// set — the flush refuses before building any request, and does so even with +// the ladder forced on. The recording transport proves no HTTP round trip was +// attempted, not merely that the spool survived; the event stays spooled the +// way a failed send leaves it, and nothing reaches the real relay. +func TestNoTestBinaryCanReachTheProductionRelay(t *testing.T) { + testHome(t) + // The default is what must be watched here: unset the dead loopback + // address testHome set, so Endpoint() answers the production relay. + unsetEnv(t, "CODEAF_TELEMETRY_ENDPOINT") + if got := Endpoint(); got != DefaultEndpoint { + t.Fatalf("Endpoint() = %q, want the default the law watches for", got) + } + counter := &roundTripCounter{} + previous := httpClient + httpClient = &http.Client{Transport: counter} + t.Cleanup(func() { httpClient = previous }) + MarkNoticeShown() + event := SessionStarted(ModeChat, false, "session-law", freshClock(t)) + if err := SpoolSync(event); err != nil { + t.Fatalf("SpoolSync returned %v; a spool that is never sent answers nil", err) + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush returned %v; a refused send must stay silent", err) + } + if trips := counter.count(); trips != 0 { + t.Fatalf("the send path made %d round trips under go test against the default endpoint, want 0", trips) + } + left := SpoolContents() + if len(left) != 1 { + t.Fatalf("%d lines remain in the spool, want the refused event still spooled", len(left)) + } + var row map[string]any + if err := json.Unmarshal(left[0], &row); err != nil { + t.Fatal(err) + } + if row["event_id"] != event.ID { + t.Errorf("the remaining event is %v, want the one that was spooled", row["event_id"]) + } +} + +// TestNewRelayAfterTestHomeOverridesTheDeadLoopback pins the relay seam's +// ordering: testHome leaves the endpoint at the dead loopback address, and a +// relay started after it must replace that override — otherwise every +// relay-using test would flush at a port nothing listens on and pass by +// accident. The recorder receiving the event is the proof: a real round trip +// happened, to this server and no other. +func TestNewRelayAfterTestHomeOverridesTheDeadLoopback(t *testing.T) { + testHome(t) + if got := Endpoint(); got != "http://127.0.0.1:1/telemetry" { + t.Fatalf("after testHome the endpoint is %q, want the dead loopback it sets", got) + } + recorder := newRelay(t) + if override := os.Getenv("CODEAF_TELEMETRY_ENDPOINT"); override != Endpoint() { + t.Fatalf("after newRelay Endpoint() answers %q but the override is %q", Endpoint(), override) + } + MarkNoticeShown() + event := SessionStarted(ModeChat, false, "session-relay", freshClock(t)) + if err := SpoolSync(event); err != nil { + t.Fatalf("SpoolSync returned %v; a spool that is sent answers nil", err) + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush returned %v; it must never fail a run", err) + } + if got := recorder.count(); got != 1 { + t.Fatalf("the relay saw %d posts after the override, want 1 — the flush must reach the relay newRelay started", got) + } + if len(recorder.posts[0].events) != 1 || recorder.posts[0].events[0]["event_id"] != event.ID { + t.Errorf("the relay received %v, want the one spooled event %q", recorder.posts[0].events, event.ID) + } + if left := len(SpoolContents()); left != 0 { + t.Errorf("%d lines remain after a confirmed send, want 0", left) + } +} + +// TestFlushAgainstTheDefaultEndpointRefusesLikeADeadRelay pins the refusal as +// Flush meets it, without the relay: with the endpoint left at the production +// default, a flush holding one sendable event must behave exactly as it does +// against a dead relay — nothing said, the event still spooled. Flush's nil +// proves nothing on its own; the counter's zero is the assertion that carries. +func TestFlushAgainstTheDefaultEndpointRefusesLikeADeadRelay(t *testing.T) { + testHome(t) + // testHome's dead loopback must not soften this: the refusal is about the + // default, so the override is removed and the default watched instead. + unsetEnv(t, "CODEAF_TELEMETRY_ENDPOINT") + if got := Endpoint(); got != DefaultEndpoint { + t.Fatalf("Endpoint() = %q, want the production default the refusal watches", got) + } + counter := &roundTripCounter{} + previous := httpClient + httpClient = &http.Client{Transport: counter} + t.Cleanup(func() { httpClient = previous }) + MarkNoticeShown() + event := SessionStarted(ModeChat, false, "session-refused", freshClock(t)) + if err := SpoolSync(event); err != nil { + t.Fatalf("SpoolSync returned %v; a spool that is never sent answers nil", err) + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush returned %v; a refused send must stay silent, as with a dead relay", err) + } + if trips := counter.count(); trips != 0 { + t.Fatalf("the send path made %d round trips against the default endpoint, want 0", trips) + } + left := SpoolContents() + if len(left) != 1 { + t.Fatalf("%d lines remain after the refusal, want the event still spooled", len(left)) + } + var row map[string]any + if err := json.Unmarshal(left[0], &row); err != nil { + t.Fatal(err) + } + if row["event_id"] != event.ID { + t.Errorf("the remaining line is %v, want the spooled event %q", row["event_id"], event.ID) + } +} + +// TestForcingTheLadderOnCannotReopenTheDefaultEndpointPath pins where the +// refusal sits: below forceLadderForTest. testHome forces the ladder on for +// every spool test, so a refusal the ladder could override would gate nothing +// this package ever runs. The ladder is forced on again, explicitly, and the +// flush must still make zero round trips and keep the event. +func TestForcingTheLadderOnCannotReopenTheDefaultEndpointPath(t *testing.T) { + testHome(t) + forceLadderForTest(t, true) + unsetEnv(t, "CODEAF_TELEMETRY_ENDPOINT") + if got := Endpoint(); got != DefaultEndpoint { + t.Fatalf("Endpoint() = %q, want the production default", got) + } + counter := &roundTripCounter{} + previous := httpClient + httpClient = &http.Client{Transport: counter} + t.Cleanup(func() { httpClient = previous }) + MarkNoticeShown() + event := SessionStarted(ModeChat, false, "session-forced-ladder", freshClock(t)) + if err := SpoolSync(event); err != nil { + t.Fatalf("SpoolSync returned %v; a spool that is never sent answers nil", err) + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush returned %v; a refused send must stay silent", err) + } + if trips := counter.count(); trips != 0 { + t.Fatalf("with the ladder forced on the send path made %d round trips against the default endpoint, want 0", trips) + } + if left := len(SpoolContents()); left != 1 { + t.Fatalf("%d lines remain after the forced-ladder flush, want the event still spooled", left) + } + var row map[string]any + if err := json.Unmarshal(SpoolContents()[0], &row); err != nil { + t.Fatal(err) + } + if row["event_id"] != event.ID { + t.Errorf("the remaining line is %v, want the spooled event %q", row["event_id"], event.ID) + } +} diff --git a/internal/telemetry/testhook.go b/internal/telemetry/testhook.go new file mode 100644 index 0000000000..f88c7cbd95 --- /dev/null +++ b/internal/telemetry/testhook.go @@ -0,0 +1,41 @@ +package telemetry + +// The go-test rung is a production rule: a test binary is never a source of +// truth about usage, and it must stay unreachable by any environment variable +// or config value. But the wiring above this package — cmd/codeaf's lifecycle +// tests — runs inside a test binary too, and it has to exercise the spool +// paths for real or prove nothing. +// +// So the ONE door past the rung is a package-level func variable, set from a +// test only, restored by that test, and never touched by production code: +// no env var, no config key, no build flag can reach it. Setting it is what +// a test does, and a test is the only thing that can. + +import "testing" + +// EnableForTest turns the ladder on (on=true) or back to its honest answer +// (on=false) for the duration of one test, restoring the previous value when +// the test ends. It is the only way past the go-test and unstamped-build +// rungs of [offReason]. +// +// It is deliberately exported and deliberately in the testhook file rather +// than a _test.go one, because the lifecycle tests in cmd/codeaf need it from +// their own package — and it is still a test-only surface by construction: +// nothing in the production call graph reads it, and a caller that is not a +// test has nothing to restore it with. +func EnableForTest(t testing.TB, on bool) { + t.Helper() + previous := forcedOn + forcedOn = on + t.Cleanup(func() { forcedOn = previous }) +} + +// ResetCountersForTest zeroes the process-wide session counters for one test +// and again when it ends, so a test that asserts exact bands is not reading +// what an earlier test in the same binary counted. Test-only for the same +// reason EnableForTest is: nothing in the production call graph reaches it. +func ResetCountersForTest(t testing.TB) { + t.Helper() + resetCountersForTest() + t.Cleanup(resetCountersForTest) +} diff --git a/internal/tui3/settings.go b/internal/tui3/settings.go index 46c78d28c5..d42a183aa5 100644 --- a/internal/tui3/settings.go +++ b/internal/tui3/settings.go @@ -588,6 +588,12 @@ var settingUI = map[string]settingMeta{ about: "remembers the messages you send, so the up arrow walks them back " + "in a later session.", }, + config.KeyTelemetry: { + tab: tabDisplay, label: "telemetry", widget: widgetToggle, + about: "sends anonymous usage counts (version, OS, mode, session and error " + + "counts) after a notice has been printed once; never prompts, code, paths " + + "or names. Off sends nothing.", + }, config.KeyDraftPersist: { tab: tabDisplay, label: "keep drafts", widget: widgetToggle, about: "keeps the half-typed message in the box across a restart, per directory.", diff --git a/scripts/install.sh b/scripts/install.sh index 16a41aad77..72203e338c 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -6,6 +6,16 @@ REPOSITORY="Agent-Field/codeaf" LEGACY_REPOSITORY="Agent-Field/aforge-v2" # Remove after the one-release repository fallback. # legacy-name CHANNEL="${CHANNEL:-stable}" VERSION="${VERSION:-}" +# The telemetry notice, verbatim from docs/TELEMETRY.md. +# The installer only writes a local install marker and prints this text; it +# never sends telemetry, and it makes no request that the download steps did +# not already make. +TELEMETRY_NOTICE='codeaf sends anonymous usage counts to AgentField. + Sent: version, OS, mode (chat or task), how many sessions, how many errors. + Never: anything about you or your work. No prompts, code, file names, + paths, repo names, keys, email, IP, or machine name. + See exactly what leaves: codeaf telemetry show + Turn off: CODEAF_TELEMETRY=off' VERBOSE="${VERBOSE:-0}" NO_MODIFY_PATH="${CODEAF_NO_MODIFY_PATH:-${AFORGE_NO_MODIFY_PATH:-0}}" # legacy-name INSTALL_DIR="${CODEAF_INSTALL_DIR:-${AFORGE_INSTALL_DIR:-${HOME}/.codeaf/bin}}" # legacy-name @@ -40,6 +50,7 @@ Environment: CHANNEL, VERSION, CODEAF_INSTALL_DIR, CODEAF_NO_MODIFY_PATH, VERBOSE GITHUB_TOKEN or GH_TOKEN: GitHub answers anonymous API calls sixty times an hour per address; a token raises that. CODEAF_GITHUB_API and CODEAF_GITHUB_DOWNLOAD for mirrors and tests + CODEAF_TELEMETRY=off, or DO_NOT_TRACK=1, turns the anonymous usage counts off EOF } @@ -54,6 +65,56 @@ usage_error() { exit 2 } +telemetry_off() { + local value + case "${DO_NOT_TRACK:-}" in + 1|[Tt]|[Tt][Rr][Uu][Ee]) return 0 ;; + esac + value=$(echo "${CODEAF_TELEMETRY:-}" | tr '[:upper:]' '[:lower:]') + case "$value" in + off|0|false) return 0 ;; + esac + return 1 +} + +# The install marker is a local record only: it names how and when this +# machine's codeaf was installed so the usage counts can bucket by channel, +# never who installed it. Nothing here contacts the network. +write_install_marker() { + local directory="$1" + local file + # The channel lands in JSON verbatim and CHANNEL can come from the + # environment, so only a known channel is recorded; anything else is + # "unknown" instead of malformed JSON. + local channel + case "${CHANNEL:-}" in + stable|rc|staging|dev) channel="$CHANNEL" ;; + *) channel="unknown" ;; + esac + if ! mkdir -p "$directory/telemetry" 2>/dev/null; then + printf 'codeaf: could not create %s/telemetry; skipping the install marker\n' "$directory" >&2 + return 0 + fi + chmod 0700 "$directory/telemetry" + file="$directory/telemetry/install.json" + if ! { printf '{"install_method":"script","channel":"%s","installed_at":"%s"}' \ + "$channel" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"; } > "$file" 2>/dev/null; then + printf 'codeaf: could not write %s; skipping the install marker\n' "$file" >&2 + return 0 + fi + chmod 0600 "$file" +} + +# Printed once, at the very end of a successful install. The binary repeats it +# before the first session's counts are ever sent. +print_telemetry_notice() { + if telemetry_off; then + printf 'codeaf: anonymous usage counts are off (CODEAF_TELEMETRY=off or DO_NOT_TRACK=1)\n' >&2 + return 0 + fi + printf '\n%s\n' "$TELEMETRY_NOTICE" >&2 +} + while [[ $# -gt 0 ]]; do case "$1" in --stable) CHANNEL="stable"; shift ;; @@ -427,3 +488,12 @@ if [[ "$RUN_BOOT_ADOPTION" == "1" ]]; then else CODEAF_HOME="$STATE_ROOT" "$INSTALL_DIR/codeaf${extension}" version fi + +# The install marker lives under the state root, and a custom install outside +# it must not create the login's state folders: the marker is written when the +# install is inside the state root or the root already exists, and skipped +# otherwise (the binary then reports install_method "unknown"). +if [[ "$RUN_BOOT_ADOPTION" == "1" || -d "$STATE_ROOT" ]]; then + write_install_marker "$STATE_ROOT" +fi +print_telemetry_notice diff --git a/scripts/release_downloads.py b/scripts/release_downloads.py new file mode 100644 index 0000000000..9e285c8cdb --- /dev/null +++ b/scripts/release_downloads.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +"""Report release download counts to PostHog. + +The numbers come from public GitHub release data: the GitHub API exposes a +running download count for every asset on a release, and this script reads +that count and sends one PostHog event per binary asset so the download +trend is visible next to the other codeaf telemetry. It runs daily from +.github/workflows/release-downloads.yml, and runs locally too: + + python3 scripts/release_downloads.py --dry-run + +Nothing is inferred and nothing about the requester is known: an asset's +download count is public, the event carries the release tag, the asset's +platform, and that count. Only assets named codeaf--[.exe] are +reported; sidecars like checksums.txt are not releases of the binary. + +An asset is reported once per day even if the run repeats: the event's id is +derived from the tag, the asset and the UTC date, so PostHog's event +deduplication makes a same-day rerun land on the first copy instead of +counting twice. +""" + +import argparse +import datetime +import json +import os +import re +import sys +import urllib.request +import uuid + +POSTHOG_EVENT = "codeaf:release_downloads" +DISTINCT_ID = "codeaf-release-downloads" +DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com" +GITHUB_API = "https://api.github.com" +NAMESPACE = uuid.NAMESPACE_URL +# codeaf--[.exe]. os/arch follow the release.yml build matrix, so +# anything outside this set is a file this script has no opinion about. +ASSET_RE = re.compile( + r"^codeaf-(?Pdarwin|linux|windows)-" + r"(?Pamd64|arm64)(?P\.exe)?$" +) +# Tag grammar, matching cmd/codeaf-release/version.go exactly: stable and +# rc tags are vMAJOR.MINOR.PATCH (rc with an -rc.N counter), and the dev and +# staging channels are date-and-commit tags. Anything else is no channel this +# script can name, so it is reported as "unknown". +STABLE_TAG_RE = re.compile(r"^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$") +RC_TAG_RE = re.compile( + r"^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)-rc\.([1-9][0-9]*)$" +) +DEV_TAG_RE = re.compile(r"^dev-[0-9]{8}-[0-9a-f]{12}$") +STAGING_TAG_RE = re.compile(r"^staging-[0-9]{8}-[0-9a-f]{12}$") +PROPS = ( + "release_tag", + "channel", + "prerelease", + "os", + "arch", + "download_count", + "snapshot_date", +) + + +def release_channel(tag: str) -> str: + """The channel of a tag: stable, rc, dev or staging, else "unknown".""" + if STABLE_TAG_RE.match(tag): + return "stable" + if RC_TAG_RE.match(tag): + return "rc" + if DEV_TAG_RE.match(tag): + return "dev" + if STAGING_TAG_RE.match(tag): + return "staging" + return "unknown" + + +def asset_os_arch(name: str): + """(os, arch) for an asset this script reports, else None.""" + match = ASSET_RE.match(name) + if not match: + return None + return match.group("os"), match.group("arch") + + +def event_uuid(tag: str, asset: str, snapshot_date: str) -> str: + """One id per release, asset and day, so a same-day rerun deduplicates.""" + return str( + uuid.uuid5( + NAMESPACE, f"{DISTINCT_ID}|{tag}|{asset}|{snapshot_date}" + ) + ) + + +def fetch_releases(repository: str, token: str): + """Every release of the repository, following GitHub's pagination.""" + releases = [] + page = 1 + while True: + request = urllib.request.Request( + f"{GITHUB_API}/repos/{repository}/releases" + f"?per_page=100&page={page}", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "codeaf-release-downloads", + }, + ) + with urllib.request.urlopen(request, timeout=30) as response: + batch = json.load(response) + if not isinstance(batch, list): + raise ValueError(f"unexpected GitHub API response on page {page}") + releases.extend(batch) + if len(batch) < 100: + return releases + page += 1 + + +def build_events(releases, snapshot_date: str): + """One PostHog event per binary asset, with the properties it carries.""" + events = [] + for release in releases: + if release.get("draft"): + continue + tag = release["tag_name"] + for asset in release.get("assets") or []: + name = asset.get("name") or "" + platform = asset_os_arch(name) + if platform is None: + continue + os_name, arch = platform + events.append( + { + "event": POSTHOG_EVENT, + "uuid": event_uuid(tag, name, snapshot_date), + "distinct_id": DISTINCT_ID, + "properties": { + "release_tag": tag, + "channel": release_channel(tag), + "prerelease": bool(release.get("prerelease")), + "os": os_name, + "arch": arch, + # The cumulative total the GitHub API reports. + "download_count": asset["download_count"], + "snapshot_date": snapshot_date, + # The sender is a CI runner; its location is meaningless. + "$process_person_profile": False, + "$geoip_disable": True, + }, + } + ) + return events + + +def build_payload(events, project_key: str): + return {"api_key": project_key, "batch": events} + + +def send(payload, posthog_host: str) -> None: + body = json.dumps(payload).encode() + request = urllib.request.Request( + f"{posthog_host.rstrip('/')}/batch/", + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=30) as response: + response.read() + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--dry-run", + action="store_true", + help="print the batch JSON instead of sending it", + ) + parser.add_argument( + "--fixture", + metavar="PATH", + help="read the releases API response from a file instead of the network", + ) + parser.add_argument( + "--repository", + default=os.environ.get("GITHUB_REPOSITORY", "Agent-Field/codeaf"), + help="owner/name of the repository whose releases are read", + ) + args = parser.parse_args(argv) + + project_key = os.environ.get("CODEAF_POSTHOG_PROJECT_KEY", "") + if not project_key and not args.dry_run: + print( + "notice: CODEAF_POSTHOG_PROJECT_KEY is not set; nothing sent", + file=sys.stderr, + ) + return 0 + # --dry-run is the local and test path: it must work without a key, so a + # missing key is printed as a placeholder rather than stopping the run. + if not project_key: + project_key = "" + + if args.fixture: + with open(args.fixture) as fixture: + releases = json.load(fixture) + else: + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN", "") + if not token: + print( + "notice: GITHUB_TOKEN is not set; nothing sent", + file=sys.stderr, + ) + return 0 + releases = fetch_releases(args.repository, token) + + snapshot_date = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y-%m-%d" + ) + events = build_events(releases, snapshot_date) + payload = build_payload(events, project_key) + + if args.dry_run: + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + + send(payload, os.environ.get("POSTHOG_HOST") or DEFAULT_POSTHOG_HOST) + print(f"sent {len(events)} events to PostHog") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_release_downloads.py b/scripts/test_release_downloads.py new file mode 100644 index 0000000000..4d1c1604c3 --- /dev/null +++ b/scripts/test_release_downloads.py @@ -0,0 +1,435 @@ +#!/usr/bin/env python3 +"""Tests for scripts/release_downloads.py: dry-run mode against a recorded +fixture of the GitHub releases API, plus the identifiers the wire spec spells +out, the workflow that schedules the script, and the docs paragraph. +""" + +import ast +import datetime +import importlib.util +import json +import os +import pathlib +import re +import subprocess +import sys +import unittest +import uuid +from unittest import mock + +HERE = pathlib.Path(__file__).resolve().parent +SCRIPT = HERE / "release_downloads.py" +FIXTURE = HERE / "testdata" / "github_releases.json" +WORKFLOW = HERE.parent / ".github" / "workflows" / "release-downloads.yml" +DOCS = HERE.parent / "docs" + +_spec = importlib.util.spec_from_file_location("release_downloads", SCRIPT) +module = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(module) + + +class FakeResponse: + """Stand-in for urlopen's return: a context manager whose read() is JSON.""" + + def __init__(self, payload): + self._payload = ( + payload if isinstance(payload, bytes) else json.dumps(payload).encode() + ) + + def __enter__(self): + return self + + def __exit__(self, *exception): + return False + + def read(self): + return self._payload + + +try: + import yaml # type: ignore +except ImportError: # pragma: no cover - depends on the machine + yaml = None + + +EVENT_PROPERTIES = ( + "release_tag", + "channel", + "prerelease", + "os", + "arch", + "download_count", + "snapshot_date", +) + + +def run_script(*args, env_extra=None): + env = dict(os.environ) + env.pop("CODEAF_POSTHOG_PROJECT_KEY", None) + env.update(env_extra or {}) + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + env=env, + ) + + +def dry_run_payload(env_extra=None): + result = run_script( + "--dry-run", "--fixture", str(FIXTURE), env_extra=env_extra + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + +class ReleaseDownloadsTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.payload = dry_run_payload({"CODEAF_POSTHOG_PROJECT_KEY": "phx_test"}) + cls.batch = cls.payload["batch"] + + def test_one_event_per_binary_asset(self): + # 6 assets x 2 full releases plus one asset on each of the dev, staging + # and beta tags; checksums.txt and any other sidecar on the fixture is + # not a binary asset and gets no event. + self.assertEqual(len(self.batch), 15) + tags = {event["properties"]["release_tag"] for event in self.batch} + self.assertEqual( + tags, + { + "v1.2.0", + "v1.3.0-rc.1", + "dev-20260917-09299019bcdc", + "staging-20260916-ed5fe58eb546", + "v1.4.0-beta.1", + }, + ) + + def test_payload_shape(self): + self.assertEqual( + self.payload["api_key"], "phx_test" + ) + for event in self.batch: + self.assertEqual(event["event"], "codeaf:release_downloads") + self.assertEqual(event["distinct_id"], "codeaf-release-downloads") + + def test_event_properties(self): + for event in self.batch: + properties = event["properties"] + self.assertEqual( + sorted(properties), + sorted( + EVENT_PROPERTIES + + ("$process_person_profile", "$geoip_disable") + ), + ) + self.assertIs(properties["$process_person_profile"], False) + # The sender is a CI runner; its location is meaningless. + self.assertIs(properties["$geoip_disable"], True) + self.assertEqual( + properties["snapshot_date"], + datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), + ) + self.assertIsInstance(properties["prerelease"], bool) + self.assertIsInstance(properties["download_count"], int) + + def test_channel_and_prerelease(self): + channels = { + event["properties"]["release_tag"]: ( + event["properties"]["channel"], + event["properties"]["prerelease"], + ) + for event in self.batch + } + self.assertEqual(channels["v1.2.0"], ("stable", False)) + self.assertEqual(channels["v1.3.0-rc.1"], ("rc", True)) + self.assertEqual(channels["dev-20260917-09299019bcdc"], ("dev", True)) + self.assertEqual( + channels["staging-20260916-ed5fe58eb546"], ("staging", True) + ) + # Not one of the four tag shapes this repository publishes. + self.assertEqual(channels["v1.4.0-beta.1"], ("unknown", True)) + + def test_platforms_and_counts(self): + counts = { + (event["properties"]["os"], event["properties"]["arch"]): event[ + "properties" + ]["download_count"] + for event in self.batch + if event["properties"]["release_tag"] == "v1.2.0" + } + self.assertEqual( + counts, + { + ("linux", "amd64"): 15010, + ("linux", "arm64"): 11240, + ("darwin", "amd64"): 18930, + ("darwin", "arm64"): 22471, + ("windows", "amd64"): 4109, + ("windows", "arm64"): 2051, + }, + ) + + def test_event_uuid_format_and_uniqueness(self): + uuids = [event["uuid"] for event in self.batch] + self.assertEqual(len(set(uuids)), len(uuids)) + for value in uuids: + self.assertRegex( + value, + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + ) + self.assertEqual(uuid.UUID(value).version, 5) + + def test_same_day_rerun_is_deduplicated(self): + # The id is derived from the tag, the asset and the UTC date, so a + # rerun on the same day produces the same ids and PostHog drops the + # repeat instead of counting the downloads twice. + again = dry_run_payload({"CODEAF_POSTHOG_PROJECT_KEY": "phx_test"}) + self.assertEqual( + [event["uuid"] for event in again["batch"]], + [event["uuid"] for event in self.batch], + ) + + def test_uuid_depends_on_snapshot_date(self): + # A different day is a new event, not a repeat of yesterday's. + tag, asset, distinct = "v1.2.0", "codeaf-linux-amd64", "codeaf-release-downloads" + self.assertNotEqual( + uuid.uuid5(uuid.NAMESPACE_URL, f"{distinct}|{tag}|{asset}|2026-02-11"), + uuid.uuid5(uuid.NAMESPACE_URL, f"{distinct}|{tag}|{asset}|2026-02-12"), + ) + + def test_no_checksums_or_asset_names_in_payload(self): + # Sidecars are not binary releases: their names and counts never + # appear in the payload at all. + text = json.dumps(self.payload) + self.assertNotIn("checksums.txt", text) + for event in self.batch: + self.assertNotIn("name", event["properties"]) + + def test_dry_run_without_project_key_prints_the_batch(self): + # --dry-run is the local and test path: no key in the environment is no + # obstacle, the key is printed as a placeholder and the batch still + # comes out, with no notice. + result = run_script("--dry-run", "--fixture", str(FIXTURE)) + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(payload["api_key"], "") + self.assertEqual(len(payload["batch"]), 15) + + def test_sending_without_project_key_is_a_notice_and_success(self): + # Only the sending path stops on a missing key; it never reaches the + # network or the PostHog host. + result = run_script() + self.assertEqual(result.returncode, 0) + self.assertIn("notice", result.stderr) + self.assertIn("CODEAF_POSTHOG_PROJECT_KEY", result.stderr) + self.assertEqual(result.stdout, "") + + +class PaginationTest(unittest.TestCase): + def test_all_pages_are_followed(self): + page_one = [ + { + "tag_name": f"v1.{number}.0", + "draft": False, + "prerelease": False, + "assets": [{"name": "codeaf-linux-amd64", "download_count": 1}], + } + for number in range(100) + ] + page_two = [ + { + "tag_name": "v0.1.0", + "draft": False, + "prerelease": False, + "assets": [{"name": "codeaf-linux-arm64", "download_count": 2}], + } + ] + with mock.patch( + "urllib.request.urlopen", + side_effect=[FakeResponse(page_one), FakeResponse(page_two)], + ) as urlopen: + releases = module.fetch_releases("Agent-Field/codeaf", "token") + self.assertEqual(urlopen.call_count, 2) + self.assertIn("page=1", urlopen.call_args_list[0].args[0].full_url) + self.assertIn("page=2", urlopen.call_args_list[1].args[0].full_url) + self.assertEqual(len(releases), 101) + self.assertEqual(releases[0]["tag_name"], "v1.0.0") + self.assertEqual(releases[-1]["tag_name"], "v0.1.0") + + def test_posthog_host_is_overridable(self): + with mock.patch("urllib.request.urlopen") as urlopen: + urlopen.return_value = FakeResponse(b"") + module.send({}, "https://telemetry.example.com") + request = urlopen.call_args.args[0] + self.assertEqual(request.full_url, "https://telemetry.example.com/batch/") + + +class WorkflowTest(unittest.TestCase): + """The workflow that schedules the script, asserted as text so the check + reads the same file GitHub Actions does.""" + + @classmethod + def setUpClass(cls): + cls.text = WORKFLOW.read_text() + + def test_workflow_file_exists(self): + self.assertTrue(WORKFLOW.exists()) + + def test_workflow_is_scheduled_daily_and_dispatchable(self): + self.assertIn("workflow_dispatch", self.text) + self.assertIn("'0 6 * * *'", self.text) + + def test_workflow_permissions_are_contents_read_only(self): + self.assertIn("permissions:", self.text) + self.assertIn("contents: read", self.text) + + def test_workflow_runs_the_script(self): + self.assertIn("python3 scripts/release_downloads.py", self.text) + self.assertIn("runs-on: ubuntu-latest", self.text) + self.assertIn("actions/checkout", self.text) + + def test_workflow_uses_builtin_token_secret_and_host_variable(self): + self.assertIn("GH_TOKEN: ${{ github.token }}", self.text) + self.assertIn( + "CODEAF_POSTHOG_PROJECT_KEY: ${{ secrets.CODEAF_POSTHOG_PROJECT_KEY }}", + self.text, + ) + self.assertIn("POSTHOG_HOST: ${{ vars.POSTHOG_HOST }}", self.text) + + def test_workflow_parses_as_yaml(self): + if yaml is None: + self.skipTest("PyYAML is not installed; the text checks above stand") + document = yaml.safe_load(self.text) + triggers = document[True] + self.assertEqual(triggers["schedule"][0]["cron"], "0 6 * * *") + self.assertIn("workflow_dispatch", triggers) + self.assertEqual(document["permissions"], {"contents": "read"}) + + def test_workflow_and_script_are_the_only_nonstandard_files(self): + # The feature is workflow + script + fixture + test + docs; no Go + # source anywhere carries it. + text = self.text + self.assertNotIn("go run", text) + self.assertNotIn("go build", text) + + +class DocsTest(unittest.TestCase): + def test_download_counts_paragraph_exists(self): + text = (DOCS / "TELEMETRY.md").read_text() + flat = " ".join(text.split()) + self.assertIn("## Download counts", text) + self.assertIn("public GitHub release data", flat) + + +class DocsAndWorkflowMixTest(unittest.TestCase): + def test_docs_paragraph_names_workflow_and_script(self): + text = (DOCS / "TELEMETRY.md").read_text() + self.assertIn("## Download counts", text) + self.assertIn("release-downloads.yml", text) + self.assertIn("scripts/release_downloads.py", text) + self.assertIn("checksums.txt", text) + + +class ScriptSourceTest(unittest.TestCase): + """The script is python3 standard library only, and imports prove it.""" + + def test_script_imports_stdlib_only(self): + tree = ast.parse(SCRIPT.read_text()) + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + imported.add(node.module.split(".")[0]) + self.assertEqual( + imported, + {"argparse", "datetime", "json", "os", "re", "sys", "urllib", "uuid"}, + ) + # Nothing outside the standard library is imported, so the script + # runs anywhere python3 does, including the workflow runner. + # sys.stdlib_module_names exists from Python 3.10; on older interpreters + # the pinned import set above is the check that stands. + stdlib = getattr(sys, "stdlib_module_names", None) + if stdlib is not None: + self.assertLessEqual(imported, set(stdlib)) + + +class ChannelTest(unittest.TestCase): + """The four tag shapes of cmd/codeaf-release/version.go, and nothing + else: any other tag is reported as "unknown" rather than mislabelled.""" + + def test_stable_tags(self): + self.assertEqual(module.release_channel("v1.2.0"), "stable") + self.assertEqual(module.release_channel("v0.0.1"), "stable") + self.assertEqual(module.release_channel("v10.20.30"), "stable") + + def test_rc_tags(self): + self.assertEqual(module.release_channel("v1.3.0-rc.1"), "rc") + self.assertEqual(module.release_channel("v2.0.0-rc.12"), "rc") + + def test_dev_tags(self): + self.assertEqual( + module.release_channel("dev-20260917-09299019bcdc"), "dev" + ) + self.assertEqual(module.release_channel("dev-20250101-abcdef012345"), "dev") + + def test_staging_tags(self): + self.assertEqual( + module.release_channel("staging-20260916-ed5fe58eb546"), "staging" + ) + self.assertEqual( + module.release_channel("staging-20250102-000000000abc"), "staging" + ) + + def test_unrecognised_tags_are_unknown(self): + for tag in ( + "v1.4.0-beta.1", + "v1.3.0-rc.0", + "v1.3.0-rc", + "v01.2.0", + "1.2.0", + "v1.2", + "dev-20260917-09299019bcdcX", + "staging-20260916-ED5FE58EB546", + "nightly-20260917-09299019bcdc", + "", + ): + self.assertEqual(module.release_channel(tag), "unknown", tag) + + +class SendEndpointTest(unittest.TestCase): + def test_default_host_is_posthog_us(self): + with mock.patch("urllib.request.urlopen") as urlopen: + urlopen.return_value = FakeResponse(b"") + module.send({}, "https://us.i.posthog.com") + self.assertEqual( + urlopen.call_args.args[0].full_url, "https://us.i.posthog.com/batch/" + ) + + +class AssetMatchingTest(unittest.TestCase): + def test_sidecars_and_nonmatching_names_are_skipped(self): + self.assertIsNone(module.asset_os_arch("checksums.txt")) + self.assertIsNone(module.asset_os_arch("codeaf-linux-amd64.tar.gz")) + self.assertIsNone(module.asset_os_arch("codeaf-solaris-sparc")) + self.assertEqual(module.asset_os_arch("codeaf-linux-amd64"), ("linux", "amd64")) + self.assertEqual( + module.asset_os_arch("codeaf-windows-amd64.exe"), ("windows", "amd64") + ) + + +class DraftTest(unittest.TestCase): + def test_draft_releases_produce_no_events(self): + draft = { + "tag_name": "v9.9.9", + "draft": True, + "prerelease": False, + "assets": [{"name": "codeaf-linux-amd64", "download_count": 5}], + } + self.assertEqual(module.build_events([draft], "2026-02-11"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/testdata/github_releases.json b/scripts/testdata/github_releases.json new file mode 100644 index 0000000000..fef8af259f --- /dev/null +++ b/scripts/testdata/github_releases.json @@ -0,0 +1,105 @@ +[ + { + "tag_name": "v1.3.0-rc.1", + "prerelease": true, + "draft": false, + "assets": [ + { + "name": "codeaf-linux-amd64", + "download_count": 4021 + }, + { + "name": "codeaf-linux-arm64", + "download_count": 3110 + }, + { + "name": "codeaf-darwin-amd64", + "download_count": 5243 + }, + { + "name": "codeaf-darwin-arm64", + "download_count": 6032 + }, + { + "name": "codeaf-windows-amd64.exe", + "download_count": 1247 + }, + { + "name": "codeaf-windows-arm64.exe", + "download_count": 604 + }, + { + "name": "checksums.txt", + "download_count": 812 + } + ] + }, + { + "tag_name": "v1.2.0", + "prerelease": false, + "draft": false, + "assets": [ + { + "name": "codeaf-linux-amd64", + "download_count": 15010 + }, + { + "name": "codeaf-linux-arm64", + "download_count": 11240 + }, + { + "name": "codeaf-darwin-amd64", + "download_count": 18930 + }, + { + "name": "codeaf-darwin-arm64", + "download_count": 22471 + }, + { + "name": "codeaf-windows-amd64.exe", + "download_count": 4109 + }, + { + "name": "codeaf-windows-arm64.exe", + "download_count": 2051 + }, + { + "name": "checksums.txt", + "download_count": 3302 + } + ] + }, + { + "tag_name": "dev-20260917-09299019bcdc", + "prerelease": true, + "draft": false, + "assets": [ + { + "name": "codeaf-linux-amd64", + "download_count": 37 + } + ] + }, + { + "tag_name": "staging-20260916-ed5fe58eb546", + "prerelease": true, + "draft": false, + "assets": [ + { + "name": "codeaf-darwin-arm64", + "download_count": 24 + } + ] + }, + { + "tag_name": "v1.4.0-beta.1", + "prerelease": true, + "draft": false, + "assets": [ + { + "name": "codeaf-windows-amd64.exe", + "download_count": 9 + } + ] + } +] diff --git a/test/installer-telemetry.sh b/test/installer-telemetry.sh new file mode 100755 index 0000000000..b4d29871e8 --- /dev/null +++ b/test/installer-telemetry.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# THE INSTALLER'S TELEMETRY DUTIES, PROVED WITHOUT A NETWORK. +# +# The notice text lives once, byte for byte, in docs/TELEMETRY.md and is +# quoted in three places — the binary, this installer and the README — and +# only a test notices when one of them drifts. The installer's main body +# downloads a release, so this test never sources it whole: it lifts out the +# three telemetry functions and runs them against a temporary state root. +# Nothing here opens a socket. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" + +script=scripts/install.sh +doc=docs/TELEMETRY.md +test -f "$doc" || { echo "docs/TELEMETRY.md is missing; this test reads the notice from it"; exit 1; } + +eval "$(awk '/^TELEMETRY_NOTICE=/{f=1} /^VERBOSE=/{f=0} f' "$script")" +eval "$(sed -n '/^telemetry_off()/,/^}/p; /^write_install_marker()/,/^}/p; /^print_telemetry_notice()/,/^}/p' "$script")" +type telemetry_off >/dev/null +type write_install_marker >/dev/null +type print_telemetry_notice >/dev/null + +pass=0 +fail=0 +ok() { + if eval "$2"; then + pass=$((pass + 1)) + else + fail=$((fail + 1)) + printf 'FAIL %s\n' "$1" + fi +} + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +unset CODEAF_TELEMETRY DO_NOT_TRACK || true + +# ok() evals these assertion strings, so they stay single-quoted and resolve +# every variable at run time; nothing below expands its quotes early. +# shellcheck disable=SC2016,SC2034 # ok() evals these strings; quoting is deliberate +{ + +# --- install.json ----------------------------------------------------------- + +CHANNEL=stable write_install_marker "$tmp/state" +f="$tmp/state/telemetry/install.json" +ok "marker exists" '[ -f "$f" ]' +# GNU stat first: on Linux `stat -f` succeeds too, but reports the filesystem. +mode_of() { stat -c '%a' "$1" 2>/dev/null || stat -f '%Lp' "$1"; } +json_valid() { + if command -v python3 >/dev/null 2>&1; then + python3 -m json.tool "$f" >/dev/null 2>&1 + elif command -v jq >/dev/null 2>&1; then + jq -e . "$f" >/dev/null 2>&1 + else + grep -Eq '"channel":"(stable|rc|staging|dev|unknown)"' "$f" + fi +} +ok "telemetry dir is 0700" '[ "$(mode_of "$tmp/state/telemetry")" = 700 ]' +ok "install.json is 0600" '[ "$(mode_of "$f")" = 600 ]' +ok "install_method is script" 'grep -q "\"install_method\":\"script\"" "$f"' +ok "channel recorded" 'grep -q "\"channel\":\"stable\"" "$f"' +ok "installed_at is UTC RFC3339" 'grep -Eq "\"installed_at\":\"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z\"" "$f"' +ok "exactly the three contract keys" '[ "$(tr "," "\n" < "$f" | grep -c "\":")" = 3 ]' +CHANNEL=dev write_install_marker "$tmp/state" +ok "marker follows the channel" 'grep -q "\"channel\":\"dev\"" "$f"' +ok "installer creates no install_id" '[ ! -e "$tmp/state/telemetry/install_id" ]' +: > "$tmp/state/telemetry/install_id" +CHANNEL=rc write_install_marker "$tmp/state" +ok "existing install_id left untouched" '[ ! -s "$tmp/state/telemetry/install_id" ]' +# The channel lands in JSON verbatim and CHANNEL can come from the environment: +# a known channel is recorded as-is, anything else becomes "unknown", and the +# marker parses either way. +ok "valid channel recorded" 'grep -q "\"channel\":\"rc\"" "$f"' +ok "valid channel marker is valid JSON" 'json_valid' +CHANNEL='we"ird' write_install_marker "$tmp/state" +ok "unexpected channel written as unknown" 'grep -q "\"channel\":\"unknown\"" "$f"' +ok "unexpected-channel marker is valid JSON" 'json_valid' +ok "unwritable state root does not fail the install" 'write_install_marker /proc/nonexistent-root' + +# --- the notice ------------------------------------------------------------- + +notice="$tmp/notice.txt" +print_telemetry_notice 2> "$notice" +expected=$(awk ' + /^## The notice$/ {f=1; next} + f && /^```$/ {f++; next} + f == 2 {print} +' "$doc") +body=$(sed 1d "$notice") +ok "one blank line before the notice" '[ -z "$(head -n 1 "$notice")" ]' +ok "notice matches docs/TELEMETRY.md verbatim" '[ "$body" = "$expected" ]' +readme_block=$(awk ' + /^```text$/ {f = 1; buf = ""; next} + /^```$/ {if (f && buf ~ /codeaf sends anonymous usage counts/) {print buf; exit} f = 0; next} + f {buf = buf $0 "\n"} +' README.md) +ok "README quotes the notice verbatim" '[ -n "$readme_block" ] && [ "$(printf "%s\n" "$expected")" = "$readme_block" ]' +ok "notice goes to stderr, nothing to stdout" '[ -z "$(print_telemetry_notice 2>/dev/null)" ]' + +for v in off 0 false OFF False; do + export CODEAF_TELEMETRY="$v" + out=$( print_telemetry_notice 2>&1 ) + ok "CODEAF_TELEMETRY=$v opts out" 'case "$out" in *"off"*) true;; *) false;; esac' + ok "opt-out prints no notice body" 'case "$out" in *"anonymous usage counts to AgentField"*) false;; *) true;; esac' + unset CODEAF_TELEMETRY +done +for v in 1 true TRUE; do + export DO_NOT_TRACK="$v" + out=$( print_telemetry_notice 2>&1 ) + ok "DO_NOT_TRACK=$v opts out" 'case "$out" in *"off"*) true;; *) false;; esac' + unset DO_NOT_TRACK +done +out=$( print_telemetry_notice 2>&1 ) +ok "unset prints the notice" 'case "$out" in *"anonymous usage counts to AgentField"*) true;; *) false;; esac' +export CODEAF_TELEMETRY=1 +out=$( print_telemetry_notice 2>&1 ) +ok "CODEAF_TELEMETRY=1 prints the notice" 'case "$out" in *"anonymous usage counts to AgentField"*) true;; *) false;; esac' +unset CODEAF_TELEMETRY + +# --- nothing new on the wire -------------------------------------------------- + +ok "no telemetry request anywhere in the installer" '! grep -qE "agentfield\.ai/api|POST" "$script"' +} + +printf 'installer telemetry: %d passed, %d failed\n' "$pass" "$fail" +[ "$fail" -eq 0 ]