From f9ddbd63307b828acb07bd9c6601a9009310bfa3 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Wed, 16 Sep 2026 23:18:12 -0400 Subject: [PATCH 01/36] telemetry: the package behind anonymous usage counting, per the wire contract Four typed events (first_run, session_started, session_ended, fault) with the contract's allowlisted, bucketed props; the opt-out ladder with its off reasons; a hashed install identity at 0700/0600; a fault fingerprint over codeaf function names only; a JSON-lines spool with a deadline-bounded flush of up to 50 events per POST, 7-day age drops and a 1000-line cap; the notice constant with NoticeShown/MarkNoticeShown gating the flush; and Show() for a future codeaf telemetry show. A library only: nothing in cmd/codeaf, internal/session or internal/config reads it yet. Co-Authored-By: codeaf --- internal/telemetry/events.go | 343 ++++++++++++++++++++++++++++++ internal/telemetry/fingerprint.go | 80 +++++++ internal/telemetry/install.go | 80 +++++++ internal/telemetry/json.go | 11 + internal/telemetry/notice.go | 35 +++ internal/telemetry/random.go | 21 ++ internal/telemetry/spool.go | 322 ++++++++++++++++++++++++++++ internal/telemetry/telemetry.go | 332 +++++++++++++++++++++++++++++ 8 files changed, 1224 insertions(+) create mode 100644 internal/telemetry/events.go create mode 100644 internal/telemetry/fingerprint.go create mode 100644 internal/telemetry/install.go create mode 100644 internal/telemetry/json.go create mode 100644 internal/telemetry/notice.go create mode 100644 internal/telemetry/random.go create mode 100644 internal/telemetry/spool.go create mode 100644 internal/telemetry/telemetry.go diff --git a/internal/telemetry/events.go b/internal/telemetry/events.go new file mode 100644 index 0000000000..806c35607f --- /dev/null +++ b/internal/telemetry/events.go @@ -0,0 +1,343 @@ +package telemetry + +import ( + "runtime" + "time" +) + +// Count buckets, exactly the strings the contract enumerates. A bucket is a +// band a person agreed to, not a number they did not. +const ( + BucketZero = "0" + BucketOne = "1" + BucketTwo5 = "2-5" + BucketSix = "6-20" + BucketTwo1 = "21-100" + Bucket100 = "100+" +) + +// Cost buckets in US dollars, from the contract. +const ( + CostZero = "0" + CostUnder1c = "<0.01" + Cost1cTo10c = "0.01-0.1" + Cost10cTo1 = "0.1-1" + Cost1To10 = "1-10" + Cost10Plus = "10+" +) + +// Duration buckets for a session, from the contract. +const ( + DurationUnder1m = "<1m" + Duration1To5m = "1-5m" + Duration5To30m = "5-30m" + Duration30mTo2h = "30m-2h" + Duration2hPlus = "2h+" +) + +// BucketCount folds a number into the contract's count band. Negative and +// unknown counts are nothing; there is no band a fault can inflate into. +func BucketCount(count int) string { + switch { + case count <= 0: + return BucketZero + case count == 1: + return BucketOne + case count <= 5: + return BucketTwo5 + case count <= 20: + return BucketSix + case count <= 100: + return BucketTwo1 + } + return Bucket100 +} + +// BucketCost folds a dollar figure into the contract's cost band. Negative +// costs mean no cost: a refund is not a price. +func BucketCost(costUSD float64) string { + switch { + case costUSD <= 0: + return CostZero + case costUSD < 0.01: + return CostUnder1c + case costUSD < 0.1: + return Cost1cTo10c + case costUSD < 1: + return Cost10cTo1 + case costUSD < 10: + return Cost1To10 + } + return Cost10Plus +} + +// BucketDuration folds a session length into the contract's duration bands. +// A negative duration is not a length; it is a clock that disagrees with +// itself, and it answers as the empty band. +func BucketDuration(d time.Duration) string { + switch { + case d <= 0: + return DurationUnder1m + case d < 5*time.Minute: + return Duration1To5m + case d < 30*time.Minute: + return Duration5To30m + case d < 2*time.Hour: + return Duration30mTo2h + } + return Duration2hPlus +} + +// StopReason enumerates why a session ended, as the contract spells them. +const ( + StopDone = "done" + StopError = "error" + StopIncomplete = "incomplete" + StopBudget = "budget" + StopTurnCap = "turn-cap" + StopDeadline = "deadline" + StopPrice = "price" + StopQuestion = "question" + StopInterrupted = "interrupted" + StopUnknown = "unknown" +) + +// ValidStopReason reports whether s is one of the contract's stop reasons, so +// a constructor can drop an unrecognised one rather than invent a key. +func ValidStopReason(s string) bool { + switch s { + case StopDone, StopError, StopIncomplete, StopBudget, StopTurnCap, + StopDeadline, StopPrice, StopQuestion, StopInterrupted, StopUnknown: + return true + } + return false +} + +// Mode is how a session ran. The zero value is chat, which is what a bare +// `codeaf` opens. +type Mode string + +// The contract's two session modes. +const ( + ModeChat Mode = "chat" + ModeTask Mode = "task" +) + +// Scope is where a fault happened. +const ( + ScopeMain = "main" + ScopeGoroutine = "goroutine" + ScopeSurface = "surface" +) + +// Event is one wire row. Only the contract's keys exist; the allowlist lives +// in how each constructor fills Props, and the doc test holds the two +// together. MarshalJSON writes the envelope exactly as the contract spells it. +type Event struct { + Name string + ID string + InstallHash string + SessionHash string // empty on first_run + Time string + Props map[string]any +} + +// jsonEvent is the wire shape of [Event]: fixed key order is not required, but +// the key NAMES are, and this is the one place they are spelled. +type jsonEvent struct { + EventName string `json:"event_name"` + EventID string `json:"event_id"` + InstallIDHash string `json:"install_id_hash"` + SessionIDHash string `json:"session_id_hash,omitempty"` + EventTime string `json:"event_time"` + Props map[string]any `json:"props"` +} + +// MarshalJSON renders the event under its wire names. The struct carries the +// typed fields; this decides the bytes. +func (e Event) MarshalJSON() ([]byte, error) { + return jsonMarshal(jsonEvent{ + EventName: e.Name, + EventID: e.ID, + InstallIDHash: e.InstallHash, + SessionIDHash: e.SessionHash, + EventTime: e.Time, + Props: e.Props, + }) +} + +// base is what every constructor starts from: a fresh id, the install hash, +// the wall clock, and the common props. It never reads anything a person +// wrote — no flag, no path, no model answer. +func base(name, sessionID string, now time.Time) Event { + props := commonProps() + event := Event{ + Name: name, + ID: randomHex(16), + InstallHash: InstallIDHash(), + Time: roundTime(now), + Props: props, + } + if sessionID != "" { + event.SessionHash = hashHex(sessionID) + } + return event +} + +// FirstRun builds the once-per-install event. It carries no session hash, by +// contract: an install has no run yet. +func FirstRun(now time.Time) Event { + return base("first_run", "", now) +} + +// SessionStarted announces one run. mode must be chat or task; anything else +// collapses to chat, the mode a bare `codeaf` opens, rather than becoming a +// value the contract does not list. +func SessionStarted(mode Mode, resumed bool, sessionID string, now time.Time) Event { + event := base("session_started", sessionID, now) + if mode != ModeTask { + mode = ModeChat + } + event.Props["mode"] = string(mode) + event.Props["resumed"] = resumed + return event +} + +// SessionStats is what a run produced. It is a struct of typed Go values so +// the caller cannot hand over a pre-bucketed string, a raw error, or anything +// else the allowlist would have to trust. +type SessionStats struct { + Duration time.Duration + Turns int + ModelCalls int + ModelCallsFailed int + ToolCalls int + ToolCallsFailed int + CostUSD float64 + StopReason string + ExitCode int +} + +// SessionEnded closes one run. An unrecognised stop reason is recorded as +// unknown — the fact that the run ended survives, the vocabulary it ended in +// does not — and the exit code is clamped to the contract's 0..5. +func SessionEnded(mode Mode, stats SessionStats, sessionID string, now time.Time) Event { + event := base("session_ended", sessionID, now) + if mode != ModeTask { + mode = ModeChat + } + stop := stats.StopReason + if !ValidStopReason(stop) { + stop = StopUnknown + } + exit := stats.ExitCode + if exit < 0 { + exit = 0 + } + if exit > 5 { + exit = 5 + } + event.Props["mode"] = string(mode) + event.Props["duration"] = BucketDuration(stats.Duration) + event.Props["turns"] = BucketCount(stats.Turns) + event.Props["model_calls"] = BucketCount(stats.ModelCalls) + event.Props["model_calls_failed"] = BucketCount(stats.ModelCallsFailed) + event.Props["tool_calls"] = BucketCount(stats.ToolCalls) + event.Props["tool_calls_failed"] = BucketCount(stats.ToolCallsFailed) + event.Props["cost_usd"] = BucketCost(stats.CostUSD) + event.Props["stop_reason"] = stop + event.Props["exit_code"] = exit + return event +} + +// Fault describes one recovered panic for the fault event. +type Fault struct { + Mode string // chat or task; anything else is sent as other + Scope string // main, goroutine or surface + Stack []byte +} + +// Fault builds the fault event. The fingerprint is derived from the stack +// here, by fingerprint.go; the panic value itself is never carried. +func FaultEvent(details Fault, sessionID string, now time.Time) Event { + event := base("fault", sessionID, now) + mode := string(details.Mode) + switch mode { + case string(ModeChat), string(ModeTask): + default: + mode = "other" + } + scope := string(details.Scope) + switch scope { + case ScopeMain, ScopeGoroutine, ScopeSurface: + default: + scope = ScopeMain + } + event.Props["mode"] = mode + event.Props["scope"] = scope + event.Props["fingerprint"] = Fingerprint(details.Stack) + return event +} + +// allowedProps is the allowlist itself: the one table the doc test reads and +// the constructors are held to. Adding a prop means adding it here, adding it +// to the constructor that produces it, and adding it to docs/TELEMETRY.md in +// the same change. +var allowedProps = map[string]map[string]bool{ + "first_run": set(commonPropNames), + "session_started": merge(set(commonPropNames), set([]string{"mode", "resumed"})), + "session_ended": merge(set(commonPropNames), set([]string{ + "mode", "duration", "turns", "model_calls", "model_calls_failed", + "tool_calls", "tool_calls_failed", "cost_usd", "stop_reason", "exit_code", + })), + "fault": merge(set(commonPropNames), set([]string{"mode", "scope", "fingerprint"})), +} + +// AllowlistedProps answers which key an event name accepts. It backs the doc +// drift test and the privacy law: the table above is the allowlist, this is +// its only reader outside this file's own tests. +func AllowlistedProps(eventName string) []string { + var names []string + for key := range allowedProps[eventName] { + names = append(names, key) + } + return names +} + +// AllowlistedEvents names the four events the contract defines, in a stable +// order for the doc. +func AllowlistedEvents() []string { + return []string{"first_run", "session_started", "session_ended", "fault"} +} + +// CommonPropNames names the six props every event carries, in contract order. +func CommonPropNames() []string { return commonPropNames } + +var commonPropNames = []string{ + "codeaf_version", "channel", "os", "arch", "usage_context", "install_method", +} + +func set(names []string) map[string]bool { + out := make(map[string]bool, len(names)) + for _, name := range names { + out[name] = true + } + return out +} + +func merge(a, b map[string]bool) map[string]bool { + out := make(map[string]bool, len(a)+len(b)) + for key := range a { + out[key] = true + } + for key := range b { + out[key] = true + } + return out +} + +// buildOSArch re-answers the buckets for tests that pin them; it exists so a +// test can name the current values without reaching into runtime itself. +func buildOSArch() (string, string) { return osName(), archName() } + +var _ = runtime.GOOS diff --git a/internal/telemetry/fingerprint.go b/internal/telemetry/fingerprint.go new file mode 100644 index 0000000000..7b876ff034 --- /dev/null +++ b/internal/telemetry/fingerprint.go @@ -0,0 +1,80 @@ +package telemetry + +import ( + "crypto/sha256" + "encoding/hex" + "runtime" + "strings" +) + +// ourModule is the module path whose frames may enter a fingerprint. Anything +// outside it — the standard library, a dependency, the runtime — is skipped, +// because those names are about machinery, not about this program. +const ourModule = "github.com/Agent-Field/codeaf/" + +// fingerprintFrames is how deep the fingerprint reads: the contract's top five +// codeaf frames. +const fingerprintFrames = 5 + +// Fingerprint reduces a panic stack to 16 lowercase hex characters: the first +// eight bytes of sha256 over the function names of the top five stack frames +// that start with github.com/Agent-Field/codeaf/. +// +// Function names ONLY. No file paths, no line numbers, no panic value, ever — +// a path is a name of the person's machine and a panic message is often their +// words. A stack with no codeaf frames at all hashes the empty name list, so a +// fault inside a dependency still groups identically for everyone rather than +// quietly becoming a fingerprint of their directory layout. +func Fingerprint(stack []byte) string { + names := codeafFrames(stack) + sum := sha256.Sum256([]byte(strings.Join(names, "\n"))) + return hex.EncodeToString(sum[:8]) +} + +// FingerprintHere fingerprints the calling goroutine's own stack, the shape a +// deferred recover has on hand. +func FingerprintHere() string { + stack := make([]uintptr, 64) + n := runtime.Callers(2, stack) + return Fingerprint(stackBytes(stack[:n])) +} + +// codeafFrames parses a printed stack and keeps the package-qualified function +// names under ourModule, dropping everything else — including the file and +// line halves of each stack line before they can reach the hash. +func codeafFrames(stack []byte) []string { + var names []string + for _, line := range strings.Split(string(stack), "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, ourModule) { + continue + } + // "pkg.Func" or "pkg.Func.Shape.Method" or "pkg.Func-fm" — never the + // tab-indented file:line half, which never starts with the module. + name := line + if idx := strings.IndexAny(name, " \t"); idx >= 0 { + name = name[:idx] + } + names = append(names, name) + if len(names) == fingerprintFrames { + break + } + } + return names +} + +// stackBytes renders PCs through runtime.CallersFrames, the reading that +// survives inlining. +func stackBytes(pcs []uintptr) []byte { + var out []byte + frames := runtime.CallersFrames(pcs) + for { + frame, more := frames.Next() + out = append(out, []byte(frame.Function)...) + out = append(out, '\n') + if !more { + break + } + } + return out +} diff --git a/internal/telemetry/install.go b/internal/telemetry/install.go new file mode 100644 index 0000000000..4a1d71bb58 --- /dev/null +++ b/internal/telemetry/install.go @@ -0,0 +1,80 @@ +package telemetry + +import ( + "os" + "path/filepath" + "strings" +) + +// InstallIDHash returns sha256 of the install id, creating the id first if +// this machine has never sent anything. The raw id never leaves this function +// and is never returned; the hash is the only identity the wire ever sees. +func InstallIDHash() string { + return hashHex(installID()) +} + +// installID reads ~/.codeaf/telemetry/install_id, creating it on first use: +// 32 random bytes as hex, a directory at 0700 and the file at 0600. A file +// that cannot be read and cannot be created is answered with a fresh random +// id for this call alone — telemetry loses its continuity but never gains a +// reason to fail the run, and nothing is written saying an id that does not +// exist. +func installID() string { + path := telemetryFile("install_id") + if body, err := os.ReadFile(path); err == nil { + if id := strings.TrimSpace(string(body)); len(id) == 64 { + return id + } + } + id := randomHex(32) + if err := writeFilePrivate(path, []byte(id)); err != nil { + return id + } + _ = os.Chmod(path, 0o600) + return id +} + +// FirstRunMarkerName is the file that records that first_run was sent for the +// current install id. It is exported for the status command the wiring job +// adds. +const FirstRunMarkerName = "first_run" + +// firstRunMarkerPath is where the once-per-install marker lives. +func firstRunMarkerPath() string { return telemetryFile(FirstRunMarkerName) } + +// FirstRunPending reports whether this install id has yet to send first_run. +// A marker naming a different install id — the id was deleted, the state moved +// — counts as pending again, because a new identity has not sent anything. +func FirstRunPending() bool { + body, err := os.ReadFile(firstRunMarkerPath()) + if err != nil { + return true + } + return strings.TrimSpace(string(body)) != installID() +} + +// MarkFirstRunSent writes the marker so first_run is emitted once per install +// id. A failure is silent: the worst case is one extra first_run after a +// crash, which no person ever sees. +func MarkFirstRunSent() { + _ = writeAtomicPrivate(firstRunMarkerPath(), []byte(installID())) +} + +// WriteInstallMethod records how codeaf arrived on this machine, as the +// installer would. Only the two contract values are kept; anything else is +// stored as unknown so a stray word from an installer cannot become a prop. +func WriteInstallMethod(method string) { + switch method { + case "script", "source": + default: + method = "unknown" + } + body, _ := jsonMarshal(map[string]string{"install_method": method}) + _ = writeAtomicPrivate(telemetryFile("install.json"), body) +} + +// telemetryFile names a file inside the telemetry directory, under the state +// root home.Dir answers for. +func telemetryFile(name string) string { + return filepath.Join(telemetryDir(), name) +} diff --git a/internal/telemetry/json.go b/internal/telemetry/json.go new file mode 100644 index 0000000000..2024a3a1f0 --- /dev/null +++ b/internal/telemetry/json.go @@ -0,0 +1,11 @@ +package telemetry + +import "encoding/json" + +// jsonMarshal is json.Marshal under one name, so a hot path that needs compact +// bytes and the doc templates that need indentation stay visibly the same +// encoder. +func jsonMarshal(v any) ([]byte, error) { return json.Marshal(v) } + +// jsonMarshalIndent is the pretty form Show prints. +func jsonMarshalIndent(v any) ([]byte, error) { return json.MarshalIndent(v, "", " ") } diff --git a/internal/telemetry/notice.go b/internal/telemetry/notice.go new file mode 100644 index 0000000000..588c6d5543 --- /dev/null +++ b/internal/telemetry/notice.go @@ -0,0 +1,35 @@ +package telemetry + +import ( + "fmt" + "os" + "sync" +) + +// Notice is the exact text the contract fixes for the one line codeaf prints +// to stderr before the first session's events are ever sent, and that the +// installer prints. It is a constant, byte for byte, and a test holds the +// bytes. +const Notice = `codeaf sends anonymous usage counts to AgentField. + Sent: version, OS, mode (chat or task), how many sessions, how many errors. + Never: anything about you or your work. No prompts, code, file names, + paths, repo names, keys, email, IP, or machine name. + See exactly what leaves: codeaf telemetry show + Turn off: CODEAF_TELEMETRY=off` + +// noticeOnce keeps the notice to one line per process even when several +// sessions open under one run. It is separate from the failure channel's once: +// a failure that kept quiet must not swallow the notice, and a notice shown +// must not spend the failure's single line. +var noticeOnce sync.Once + +// PrintNotice writes the notice to stderr once per process, the shape the +// surface job calls before the first session's events are sent. It never +// touches the on-disk marker — that is MarkNoticeShown's job, and the two are +// separate so a notice printed by the installer does not stand in for the +// surface having shown it. +func PrintNotice() { + noticeOnce.Do(func() { + fmt.Fprintln(os.Stderr, Notice) + }) +} diff --git a/internal/telemetry/random.go b/internal/telemetry/random.go new file mode 100644 index 0000000000..8a1d4c285b --- /dev/null +++ b/internal/telemetry/random.go @@ -0,0 +1,21 @@ +package telemetry + +import ( + "crypto/rand" + "encoding/hex" +) + +// randomHex returns 2n lowercase hex characters from the system's random +// source. It backs the event id (16 bytes) and the install id (32 bytes). +func randomHex(n int) string { + raw := make([]byte, n) + if _, err := rand.Read(raw); err != nil { + // The system random source does not fail on any platform codeaf + // ships; if it somehow did, a zero id is still exactly 64 hex + // characters and still anonymous. + for i := range raw { + raw[i] = byte(i) + } + } + return hex.EncodeToString(raw) +} diff --git a/internal/telemetry/spool.go b/internal/telemetry/spool.go new file mode 100644 index 0000000000..3b47d9221c --- /dev/null +++ b/internal/telemetry/spool.go @@ -0,0 +1,322 @@ +package telemetry + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "os" + "strings" + "sync" + "time" +) + +// Spool limits, from the contract: a thousand lines of history, seven days of +// age, fifty events on the wire per POST, one megabyte read into memory at +// most. +const ( + MaxSpoolLines = 1000 + MaxEventAge = 7 * 24 * time.Hour + MaxEventsPerPOST = 50 + MaxSpoolReadBytes = 1 << 20 +) + +// noticeSeenPath is the marker that says the person has seen the notice. Until +// it exists, nothing is sent: spooling is silent, flushing is a no-op. +const noticeSeenFile = "notice_seen" + +// noticeSeenPath is where NoticeShown reads and MarkNoticeShown writes. +func noticeSeenPath() string { return telemetryFile(noticeSeenFile) } + +// spoolPath is the JSON-lines file events wait in. +func spoolPath() string { return telemetryFile("spool.jsonl") } + +// NoticeShown reports whether the notice has been marked shown. Until it has, +// events may be spooled but are never flushed. +func NoticeShown() bool { return fileExists(noticeSeenPath()) } + +// MarkNoticeShown records that the person has seen the notice, which opens the +// gate on flushing. A failure is silent for the same reason every other write +// here is: the worst case is a notice seen twice. +func MarkNoticeShown() { + if err := writeFilePrivate(noticeSeenPath(), []byte("shown\n")); err != nil { + oneWarning("telemetry: could not record that the notice was shown") + } +} + +// warnOnce is the one line to stderr a process may ever produce for telemetry +// failures, however many failures there are. +var warnOnce sync.Once + +// oneWarning prints the process's single telemetry failure line to stderr. +func oneWarning(message string) { + warnOnce.Do(func() { + fmt.Fprintln(os.Stderr, message) + }) +} + +// Spool appends one event to the spool file and returns immediately. It never +// blocks the caller for more than a few milliseconds: it does no network work, +// and the one file append it does is a single WriteFile on a line that was +// already marshalled. A failure here is silent. +func Spool(event Event) { + line, err := jsonMarshal(event) + if err != nil { + return + } + go persist([]byte(line)) +} + +// SpoolSync appends one event and waits for the append, for tests and for a +// caller that must see the line on disk. It shares every law with Spool and is +// not for the product's hot path. +func SpoolSync(event Event) error { + line, err := jsonMarshal(event) + if err != nil { + return nil + } + return persist(line) +} + +// persist appends one marshalled event line to the spool. The file is opened +// per write: a spool of a handful of lines is a once-per-session append, and +// an append that never holds a descriptor open can never leak one. +func persist(line []byte) error { + if err := ensureDir(); err != nil { + return err + } + file, err := os.OpenFile(spoolPath(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + return err + } + defer file.Close() + if _, err := file.Write(append(line, '\n')); err != nil { + return err + } + return truncateSpoolToCap() +} + +// truncateSpoolToCap keeps the spool at the contract's line cap by dropping +// the OLDEST lines — the front of the file — once it has grown past 1000. +func truncateSpoolToCap() error { + path := spoolPath() + lines, err := readSpoolLines(path) + if err != nil { + return err + } + if len(lines) <= MaxSpoolLines { + return nil + } + return rewriteSpool(path, lines[len(lines)-MaxSpoolLines:]) +} + +// spoolEntry is one parsed line of the spool file, with the wire row it +// parsed from when it parsed. +type spoolEntry struct { + line string + event jsonEvent + ageKnown bool +} + +// Flush sends every spooled event it can within the deadline the caller +// carries — callers pass a hard budget, one second at exit — POSTing up to +// fifty at a time to the endpoint. Lines that were sent are removed, lines +// older than seven days are dropped, and a failure of any kind is silent. +// The returned error is always nil; it exists so a future caller can log +// without this package ever being able to fail a run. +func Flush(ctx context.Context) error { + if !NoticeShown() { + return nil + } + deadline, hasDeadline := ctx.Deadline() + if !hasDeadline { + deadline = time.Now().Add(time.Second) + } + path := spoolPath() + lines, err := readSpoolLines(path) + if err != nil { + return nil + } + kept := make([]spoolEntry, 0, len(lines)) + var batches [][]spoolEntry + var batch []spoolEntry + dropped := false + for _, entry := range lines { + if entry.ageKnown && ageOf(entry.event.EventTime) > MaxEventAge { + dropped = true + continue + } + kept = append(kept, entry) + batch = append(batch, entry) + if len(batch) == MaxEventsPerPOST { + batches = append(batches, batch) + batch = nil + } + } + if len(batch) > 0 { + batches = append(batches, batch) + } + for _, batch := range batches { + if !sendBatch(ctx, deadline, batch) { + // The deadline is hard: stop mid-batch, keep everything not yet + // confirmed sent, and say nothing. + break + } + dropped = true + } + if dropped { + _ = rewriteSpool(path, keptAfterSending(path, lines, kept, batches)) + } + return nil +} + +// keptAfterSending removes the lines whose batches were confirmed sent. It is +// a pure function of the original lines, the parsed survivors and the batches +// that went out, so a flush that sent nothing still rewrites the file with the +// age drops applied. +func keptAfterSending(path string, lines []spoolEntry, kept []spoolEntry, batches [][]spoolEntry) []spoolEntry { + sent := map[int]bool{} + // Count forward: batch i is the i-th group of MaxEventsPerPOST survivors, + // which is exactly how the batches above were cut. + for i := range batches { + for j := range batches[i] { + sent[i*MaxEventsPerPOST+j] = true + } + } + if len(sent) == 0 { + return kept + } + var survivors []spoolEntry + for index, entry := range kept { + if !sent[index] { + survivors = append(survivors, entry) + } + } + return survivors +} + +// sendBatch POSTs one batch. It answers true only when the relay answered +// 2xx within the deadline; any error, any other status, and the batch stays. +func sendBatch(ctx context.Context, deadline time.Time, batch []spoolEntry) bool { + remaining := time.Until(deadline) + if remaining <= 0 { + return false + } + batchCtx, cancel := context.WithTimeout(ctx, remaining) + defer cancel() + body, err := jsonMarshal(map[string]any{ + "schema_version": schemaVersion, + "events": batchEvents(batch), + }) + if err != nil { + return false + } + request, err := http.NewRequestWithContext(batchCtx, http.MethodPost, Endpoint(), bytes.NewReader(body)) + if err != nil { + return false + } + request.Header.Set("Content-Type", "application/json") + response, err := httpClient.Do(request) + if err != nil { + return false + } + defer response.Body.Close() + ok := response.StatusCode >= 200 && response.StatusCode < 300 + return ok +} + +// batchEvents returns the wire rows of one batch, in order. +func batchEvents(batch []spoolEntry) []jsonEvent { + out := make([]jsonEvent, len(batch)) + for i, entry := range batch { + out[i] = entry.event + } + return out +} + +// httpClient is the one client. No timeout of its own: the deadline lives in +// the request context, where the caller's hard budget is. +var httpClient = &http.Client{} + +// readSpoolLines reads the spool as entries, skipping and reporting lines it +// cannot parse. A spool that does not exist is an empty spool. +func readSpoolLines(path string) ([]spoolEntry, error) { + file, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + defer file.Close() + var entries []spoolEntry + scanner := bufio.NewScanner(file) + scanner.Buffer(make([]byte, 0, 64*1024), MaxSpoolReadBytes) + for scanner.Scan() { + raw := strings.TrimSpace(scanner.Text()) + if raw == "" { + continue + } + entry := spoolEntry{line: raw} + if err := json.Unmarshal([]byte(raw), &entry.event); err == nil { + entry.ageKnown = true + } + entries = append(entries, entry) + } + return entries, scanner.Err() +} + +// rewriteSpool replaces the spool file with the given lines atomically. +func rewriteSpool(path string, entries []spoolEntry) error { + if err := ensureDir(); err != nil { + return err + } + var out bytes.Buffer + for _, entry := range entries { + out.WriteString(entry.line) + out.WriteByte('\n') + } + temp := path + ".tmp" + if err := os.WriteFile(temp, out.Bytes(), 0o600); err != nil { + return err + } + return os.Rename(temp, path) +} + +// SpoolContents returns the spooled events as raw JSON rows, oldest first, in +// the order they would be sent. Show formats them for a person. +func SpoolContents() []json.RawMessage { + lines, err := readSpoolLines(spoolPath()) + if err != nil { + return nil + } + out := make([]json.RawMessage, 0, len(lines)) + for _, entry := range lines { + out = append(out, json.RawMessage(entry.line)) + } + return out +} + +// Show returns the spool as pretty JSON — the exact answer a future +// `codeaf telemetry show` prints, so a person can read everything that has +// not left yet. +func Show() string { + contents := SpoolContents() + if len(contents) == 0 { + return "[]" + } + var out bytes.Buffer + out.WriteString("[\n") + for i, raw := range contents { + out.WriteString(" ") + out.Write(raw) + if i < len(contents)-1 { + out.WriteByte(',') + } + out.WriteByte('\n') + } + out.WriteString("]") + return out.String() +} diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go new file mode 100644 index 0000000000..3300f8b89b --- /dev/null +++ b/internal/telemetry/telemetry.go @@ -0,0 +1,332 @@ +// Package telemetry implements codeaf's anonymous usage counting exactly as +// .telemetry-contract.md spells it: four typed events, an allowlisted property +// set, a local spool, and one deadline-bounded flush. It is a library only — +// nothing in cmd/codeaf, internal/session or internal/config reads it yet; the +// wiring is a later job. +// +// The law of the package is the contract at the repository root. When this file +// and the contract disagree, the contract wins and this package is the defect. +package telemetry + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "runtime" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/buildinfo" + "github.com/Agent-Field/codeaf/internal/env" + "github.com/Agent-Field/codeaf/internal/home" +) + +// DefaultEndpoint is the relay the contract names. CODEAF_TELEMETRY_ENDPOINT +// moves it; set to empty it turns telemetry off entirely. +const DefaultEndpoint = "https://agentfield.ai/api/oss/codeaf/telemetry" + +// schemaVersion is the wire version the contract fixes. Bumping it is a +// contract change, not a code change. +const schemaVersion = 1 + +// OptOut enumerates the reasons the ladder can answer with. The empty value +// means telemetry would run. +const ( + OnReason = "" + OffEnv = "CODEAF_TELEMETRY=off" + OffDoNotTrack = "DO_NOT_TRACK is set" + OffConfig = "config telemetry = off" + OffEmptyEndpoint = "CODEAF_TELEMETRY_ENDPOINT is empty" + OffBuild = "dirty or unstamped build" + OffTest = "running under go test" +) + +// truthy reads a foreign variable the way the contract means "when truthy": +// present, non-empty, and not one of the negative spellings. +func truthy(name string) bool { + value := env.Value(name) + switch strings.ToLower(strings.TrimSpace(value)) { + case "", "0", "false", "off": + return false + } + return true +} + +// offReason is the whole opt-out ladder in one pure function so tests can +// reach every rung without rebuilding the binary. The rungs are ordered: a +// person's own switch wins over everything, the build's honesty wins over the +// runtime, and a test binary is never a source of truth about usage. +func offReason(configTelemetryOff, underTest, dirtyOrUnstamped bool) string { + if codeaf := strings.ToLower(strings.TrimSpace(env.Get("CODEAF_TELEMETRY"))); codeaf == "off" || codeaf == "0" || codeaf == "false" { + return OffEnv + } + if value := strings.TrimSpace(env.Value("DO_NOT_TRACK")); value == "1" || strings.EqualFold(value, "true") { + return OffDoNotTrack + } + if configTelemetryOff { + return OffConfig + } + if value, set := env.Lookup("CODEAF_TELEMETRY_ENDPOINT"); set && strings.TrimSpace(value) == "" { + return OffEmptyEndpoint + } + if dirtyOrUnstamped { + return OffBuild + } + if underTest { + return OffTest + } + return OnReason +} + +// dirtyOrUnstampedBuild reports whether this binary cannot name its own +// source. A build without a stamped revision or carrying uncommitted edits has +// no stable version to report, so it does not report at all. +func dirtyOrUnstampedBuild() bool { + revision := buildinfo.Revision() + if revision == "" { + return true + } + // Identity spells a source with no slashes in it and falls back to + // "source/dirty/moment" when it has none; a dirty build is visible there + // without internal/buildinfo growing a second accessor. + return strings.Contains(buildinfo.Identity(), "/true/") +} + +// underGoTest reports whether this process is a test binary, where telemetry +// is always off. +func underGoTest() bool { return testing.Testing() } + +// Enabled reports whether telemetry would run, given the config value the +// caller read (true means the config turns it off). Nothing is spooled or +// sent when this answers false. +func Enabled(configTelemetryOff bool) bool { + return OffReason(configTelemetryOff) == OnReason +} + +// OffReason is Enabled with the why, for a status command. It returns "" when +// telemetry would run, otherwise one stable phrase naming the rung that +// switched it off. +func OffReason(configTelemetryOff bool) string { + return offReason(configTelemetryOff, underGoTest(), dirtyOrUnstampedBuild()) +} + +// Endpoint names where events go: the contract relay unless the override says +// otherwise. +func Endpoint() string { + if value := strings.TrimSpace(env.Get("CODEAF_TELEMETRY_ENDPOINT")); value != "" { + return value + } + return DefaultEndpoint +} + +// osName buckets runtime.GOOS the way the contract enumerates it. +func osName() string { + switch runtime.GOOS { + case "darwin", "linux", "windows": + return runtime.GOOS + } + return "other" +} + +// archName buckets runtime.GOARCH the same way. +func archName() string { + switch runtime.GOARCH { + case "amd64", "arm64": + return runtime.GOARCH + } + return "other" +} + +// ciVariables are the foreign variables that mean this run is on a CI +// machine. They are read through the dynamic env door because none of them is +// a codeaf name. +var ciVariables = []string{"CI", "GITHUB_ACTIONS", "GITLAB_CI", "BUILDKITE", "CIRCLECI", "JENKINS_URL"} + +// usageContext buckets where codeaf is running: a CI machine, a container, or +// a person's own machine. CI wins over container when both are true, because +// CI runners are themselves containers and the coarser bucket is the useful +// one. +func usageContext() string { + for _, name := range ciVariables { + if truthy(name) { + return "ci" + } + } + if _, err := os.Stat("/.dockerenv"); err == nil { + return "container" + } + if truthy("KUBERNETES_SERVICE_HOST") { + return "container" + } + return "local" +} + +// installMethod reads the one fact the installer writes about how codeaf +// arrived, and answers "unknown" whenever it cannot vouch for a word. +func installMethod() string { + body, err := os.ReadFile(home.Join("telemetry", "install.json")) + if err != nil { + return "unknown" + } + var record struct { + InstallMethod string `json:"install_method"` + } + if json.Unmarshal(body, &record) != nil { + return "unknown" + } + switch record.InstallMethod { + case "script", "source": + return record.InstallMethod + } + return "unknown" +} + +// channelFor names the release channel a build revision came from, using the +// same tag shapes cmd/codeaf-release cuts: vX.Y.Z, vX.Y.Z-rc.N, +// dev-YYYYMMDD-hex12 and staging-YYYYMMDD-hex12. Anything else is unknown. +func channelFor(revision string) string { + switch { + case strings.Contains(revision, "-rc."): + if isNumericDotted(strings.TrimPrefix(strings.TrimSuffix(revision, revision[strings.LastIndex(revision, "-rc."):]), "v")) { + return "rc" + } + return "unknown" + case isStableTag(revision): + return "stable" + case isChannelTag(revision, "dev-"): + return "dev" + case isChannelTag(revision, "staging-"): + return "staging" + } + return "unknown" +} + +// isStableTag matches vX.Y.Z with three numeric parts and nothing else. +func isStableTag(revision string) bool { + if !strings.HasPrefix(revision, "v") { + return false + } + return isNumericDotted(revision[1:]) +} + +func isNumericDotted(value string) bool { + parts := strings.Split(value, ".") + if len(parts) != 3 { + return false + } + for _, part := range parts { + if part == "" { + return false + } + for _, r := range part { + if r < '0' || r > '9' { + return false + } + } + } + return true +} + +// isChannelTag matches YYYYMMDD-<12 lowercase hex chars>, the shape +// every dev and staging tag carries. +func isChannelTag(revision, prefix string) bool { + rest, ok := strings.CutPrefix(revision, prefix) + if !ok || len(rest) != 21 || rest[8] != '-' { + return false + } + for _, r := range rest[:8] { + if r < '0' || r > '9' { + return false + } + } + _, err := hex.DecodeString(rest[9:]) + return err == nil +} + +// commonProps are the properties every event carries. version is capped at 64 +// characters because the contract caps it and a truncated tag still sorts. +func commonProps() map[string]any { + version := buildinfo.Revision() + if version == "" { + version = "unknown" + } + if len(version) > 64 { + version = version[:64] + } + return map[string]any{ + "codeaf_version": version, + "channel": channelFor(version), + "os": osName(), + "arch": archName(), + "usage_context": usageContext(), + "install_method": installMethod(), + } +} + +// hashHex is sha256 over one string, spelled once because three different +// properties are hashes and none of them may drift into a different recipe. +func hashHex(value string) string { + sum := sha256.Sum256([]byte(value)) + return hex.EncodeToString(sum[:]) +} + +// telemetryDir is the directory under the state root that holds everything +// this package writes. It is reached through internal/home so CODEAF_HOME +// moves it. +func telemetryDir() string { return home.Join("telemetry") } + +// ensureDir creates the telemetry directory with its contract mode. The parent +// state root usually exists; MkdirAll only stamps the mode on what it creates. +func ensureDir() error { + return os.MkdirAll(telemetryDir(), dirMode) +} + +// writeFilePrivate writes one file at 0600 after making sure its directory is +// 0700, which is every file this package writes. +func writeFilePrivate(path string, body []byte) error { + if err := ensureDir(); err != nil { + return err + } + return os.WriteFile(path, body, 0o600) +} + +// writeAtomicPrivate rewrites path through a sibling temp file so a crash +// mid-write never leaves a half file where an id or a marker should be. +func writeAtomicPrivate(path string, body []byte) error { + if err := ensureDir(); err != nil { + return err + } + temp := path + ".tmp" + if err := os.WriteFile(temp, body, 0o600); err != nil { + return err + } + return os.Rename(temp, path) +} + +// fileExists is os.Stat with the answer the callers want. +func fileExists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +// roundTime is the RFC3339 UTC stamp every event carries. +func roundTime(t time.Time) string { return t.UTC().Format(time.RFC3339) } + +// ageOf parses an event_time back into an age, answering "ancient" for a stamp +// it cannot read — an unparseable line is never worth sending. +func ageOf(stamp string) time.Duration { + parsed, err := time.Parse(time.RFC3339, stamp) + if err != nil { + return 365 * 24 * time.Hour + } + return time.Since(parsed) +} + +// dirMode is asserted by tests so a permissions drift fails loudly here rather +// than leaking anything in production. +const ( + dirMode os.FileMode = 0o700 + fileMode os.FileMode = 0o600 +) From 8ee3758458b1eb76be39a37493d2ce9410f78481 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 00:04:19 -0400 Subject: [PATCH 02/36] telemetry: tests, doc, and contract fixes for the usage-count package MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the package's tests — bucketing edges, the opt-out ladder, install identity, the notice, fingerprinting, the spool against an httptest relay (cap, age drop, 50-per-POST splitting, a hard deadline against a hanging server, silence on failure) and the privacy law that no sentinel string reaches the wire. docs/TELEMETRY.md documents the notice, what is sent and how to turn it off, with a test that fails if the doc's property table drifts from the Go allowlist. Fixes four drifts from the wire contract found running those tests: the <1m duration bucket was missing, the fingerprint's argument stripping cut too much, the rc channel suffix must be a single integer, and under go test a config-off rung outranks the test rung in the reason reported. Co-Authored-By: codeaf --- docs/TELEMETRY.md | 87 ++++++ internal/telemetry/doc_test.go | 122 ++++++++ internal/telemetry/events.go | 23 +- internal/telemetry/events_test.go | 337 ++++++++++++++++++++++ internal/telemetry/fingerprint.go | 40 ++- internal/telemetry/fingerprint_test.go | 103 +++++++ internal/telemetry/spool.go | 40 ++- internal/telemetry/spool_test.go | 359 +++++++++++++++++++++++ internal/telemetry/telemetry.go | 19 +- internal/telemetry/telemetry_test.go | 378 +++++++++++++++++++++++++ 10 files changed, 1491 insertions(+), 17 deletions(-) create mode 100644 docs/TELEMETRY.md create mode 100644 internal/telemetry/doc_test.go create mode 100644 internal/telemetry/events_test.go create mode 100644 internal/telemetry/fingerprint_test.go create mode 100644 internal/telemetry/spool_test.go create mode 100644 internal/telemetry/telemetry_test.go diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md new file mode 100644 index 0000000000..5b048a9bcd --- /dev/null +++ b/docs/TELEMETRY.md @@ -0,0 +1,87 @@ +# Telemetry + +codeaf counts how it is used — how often, in which modes, on which platforms — +so the parts people rely on get the work. The counts are anonymous: nothing +about you or your work ever leaves this machine. + +## The notice + +Before the first session's events are sent, codeaf prints this to stderr once; +the installer prints it too: + +``` +codeaf sends anonymous usage counts to AgentField. + Sent: version, OS, mode (chat or task), how many sessions, how many errors. + Never: anything about you or your work. No prompts, code, file names, + paths, repo names, keys, email, IP, or machine name. + See exactly what leaves: codeaf telemetry show + Turn off: CODEAF_TELEMETRY=off +``` + +## What is sent + +Exactly four events. Each carries the every-event properties; three of them +add more. Every value is a count, a band, or a word from a fixed list. The +table is generated from the same allowlist the code is held to, and a test +fails the build if the two ever drift apart. + +| Event | Property | What it is | +| --- | --- | --- | +| every event | codeaf_version | the release tag this binary was built from, at most 64 characters | +| every event | channel | stable, rc, staging, dev, or unknown | +| every event | os | darwin, linux, windows, or other | +| every event | arch | amd64, arm64, or other | +| every event | usage_context | local (a person's machine), ci, or container | +| every event | install_method | script, source, or unknown | +| session_started | mode | chat or task | +| session_started | resumed | whether the session continued an earlier one | +| session_ended | mode | chat or task | +| session_ended | duration | a band: under 1m, 1-5m, 5-30m, 30m-2h, 2h or more | +| session_ended | turns | a count band | +| session_ended | model_calls | a count band | +| session_ended | model_calls_failed | a count band | +| session_ended | tool_calls | a count band | +| session_ended | tool_calls_failed | a count band | +| session_ended | cost_usd | a dollar band | +| session_ended | stop_reason | done, error, incomplete, budget, turn-cap, deadline, price, question, interrupted, or unknown | +| session_ended | exit_code | 0 to 5 | +| fault | mode | chat, task, or other | +| fault | scope | main, goroutine, or surface | +| fault | fingerprint | 16 hex characters hashed from codeaf function names in the stack | + +Count bands are 0, 1, 2-5, 6-20, 21-100 and 100+. Dollar bands are 0, under +0.01, 0.01-0.1, 0.1-1, 1-10 and 10+. + +first_run carries only the every-event properties and is sent once per +install. + +Each event also carries its identity as hashes: a random per-event id, sha256 +of the install id, and — except on first_run — sha256 of the run id. The raw +ids never leave this machine. + +## What is never sent + +Prompts, model replies, code, file names, paths, repo or directory names, git +remotes, hostnames, usernames, IP addresses, environment values, API keys, +email addresses, model names, error text, panic messages. A test builds every +event from inputs stuffed with exactly these and fails if any of them reach +the marshalled output. + +## Where events wait + +Events wait in ~/.codeaf/telemetry/spool.jsonl until they are sent: at most 50 +per request, nothing older than 7 days, at most 1000 lines kept, and nothing +sent before the notice has been shown. `codeaf telemetry show` prints exactly +what has not left yet. + +## How to turn it off + +Any one of these, before codeaf starts: + +- `CODEAF_TELEMETRY=off` (also `0` or `false`) +- `DO_NOT_TRACK=1` (also `true`) +- the config key `telemetry = off` +- an empty `CODEAF_TELEMETRY_ENDPOINT` + +A build that cannot name its own source — dirty or unstamped — never reports, +and neither does a test binary. diff --git a/internal/telemetry/doc_test.go b/internal/telemetry/doc_test.go new file mode 100644 index 0000000000..740af9df0e --- /dev/null +++ b/internal/telemetry/doc_test.go @@ -0,0 +1,122 @@ +package telemetry + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +// The doc is held to the code. Its property table is parsed and compared with +// the allowlist the constructors are built from, so a prop added to the code +// without the doc — or written into the doc without the code — fails here. + +func docBody(t *testing.T) string { + t.Helper() + body, err := os.ReadFile(filepath.Join("..", "..", "docs", "TELEMETRY.md")) + if err != nil { + t.Fatalf("docs/TELEMETRY.md must exist beside the package: %v", err) + } + return string(body) +} + +// docProps parses the property table under "## What is sent" into the +// every-event props and the extras per event. Anything else on the page is +// prose and not held to anything. +func docProps(t *testing.T, body string) (common []string, perEvent map[string][]string) { + t.Helper() + perEvent = map[string][]string{} + inSection := false + for _, line := range strings.Split(body, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "## ") { + inSection = trimmed == "## What is sent" + continue + } + if !inSection || !strings.HasPrefix(trimmed, "|") { + continue + } + cells := strings.Split(strings.Trim(trimmed, "|"), "|") + if len(cells) != 3 { + continue + } + for i := range cells { + cells[i] = strings.TrimSpace(cells[i]) + } + if cells[0] == "Event" || strings.HasPrefix(cells[0], "-") || strings.HasPrefix(cells[1], "-") { + continue + } + if cells[0] == "every event" { + common = append(common, cells[1]) + continue + } + if cells[0] != "" { + perEvent[cells[0]] = append(perEvent[cells[0]], cells[1]) + } + } + return common, perEvent +} + +func TestDocPropertyTableMatchesTheAllowlist(t *testing.T) { + docCommon, docPerEvent := docProps(t, docBody(t)) + + wantCommon := map[string]bool{} + for _, name := range CommonPropNames() { + wantCommon[name] = true + } + gotCommon := map[string]bool{} + for _, name := range docCommon { + gotCommon[name] = true + } + if !reflect.DeepEqual(gotCommon, wantCommon) { + t.Errorf("the doc's every-event props drift from CommonPropNames: doc %v, allowlist %v", docCommon, CommonPropNames()) + } + + events := map[string]bool{} + for _, name := range AllowlistedEvents() { + events[name] = true + } + for name := range docPerEvent { + if !events[name] { + t.Errorf("the doc has property rows for %q, which is not one of the four events", name) + } + } + for _, event := range AllowlistedEvents() { + want := map[string]bool{} + for _, name := range AllowlistedProps(event) { + want[name] = true + } + got := map[string]bool{} + for _, name := range docCommon { + got[name] = true + } + for _, name := range docPerEvent[event] { + if wantCommon[name] { + t.Errorf("%s: %q is already an every-event prop; the doc lists it twice", event, name) + } + got[name] = true + } + if !reflect.DeepEqual(got, want) { + t.Errorf("%s: the doc's props drift from the allowlist: doc %v, allowlist %v", event, got, AllowlistedProps(event)) + } + } +} + +func TestDocCarriesTheNoticeAndTheSwitches(t *testing.T) { + body := docBody(t) + if !strings.Contains(body, Notice) { + t.Error("docs/TELEMETRY.md must quote the notice byte for byte") + } + for _, wanted := range []string{"CODEAF_TELEMETRY=off", "DO_NOT_TRACK=1", "telemetry show"} { + if !strings.Contains(body, wanted) { + t.Errorf("docs/TELEMETRY.md must mention %q", wanted) + } + } + lower := strings.ToLower(body) + for _, never := range []string{"prompts", "paths", "hostnames", "usernames", "api keys", "panic messages"} { + if !strings.Contains(lower, never) { + t.Errorf("docs/TELEMETRY.md must name %q among what is never sent", never) + } + } +} diff --git a/internal/telemetry/events.go b/internal/telemetry/events.go index 806c35607f..05774eff04 100644 --- a/internal/telemetry/events.go +++ b/internal/telemetry/events.go @@ -76,7 +76,7 @@ func BucketCost(costUSD float64) string { // itself, and it answers as the empty band. func BucketDuration(d time.Duration) string { switch { - case d <= 0: + case d <= 0, d < time.Minute: return DurationUnder1m case d < 5*time.Minute: return Duration1To5m @@ -156,14 +156,29 @@ type jsonEvent struct { // MarshalJSON renders the event under its wire names. The struct carries the // typed fields; this decides the bytes. func (e Event) MarshalJSON() ([]byte, error) { - return jsonMarshal(jsonEvent{ + // MarshalJSON carries only the contract's keys and the allowlisted props; + // an unknown prop key in the map (a hand-written or older-build spool line) + // is dropped here rather than sent. + clean := make(map[string]any, len(e.Props)) + if allowed := allowedProps[e.Name]; len(allowed) > 0 { + for key, value := range e.Props { + if allowed[key] { + clean[key] = value + } + } + } + row := jsonEvent{ EventName: e.Name, EventID: e.ID, InstallIDHash: e.InstallHash, SessionIDHash: e.SessionHash, EventTime: e.Time, - Props: e.Props, - }) + Props: clean, + } + if e.Name == "first_run" { + row.SessionIDHash = "" + } + return jsonMarshal(row) } // base is what every constructor starts from: a fresh id, the install hash, diff --git a/internal/telemetry/events_test.go b/internal/telemetry/events_test.go new file mode 100644 index 0000000000..1d545f42c2 --- /dev/null +++ b/internal/telemetry/events_test.go @@ -0,0 +1,337 @@ +package telemetry + +import ( + "encoding/json" + "reflect" + "regexp" + "strings" + "testing" + "time" +) + +// The wire's hash shapes, from the contract: an event id is 32 lowercase hex +// characters, an install or session hash is 64, a fault fingerprint is 16. +var ( + reEventID = regexp.MustCompile(`^[0-9a-f]{32}$`) + reIdentityHash = regexp.MustCompile(`^[0-9a-f]{64}$`) + reFingerprint = regexp.MustCompile(`^[0-9a-f]{16}$`) +) + +// sentinels are the private things a person's machine is full of. Every +// constructor is fed them wherever a value could possibly flow, and the +// privacy law below holds that not one of them reaches marshalled bytes. +var sentinels = []string{ + "/Users/santosh/secret-project/leaky.go", // a path + "person@example.com", // an email + "hades.corp.internal", // a hostname + "santosh", // a username + "secret-project", // a repo name + "why does my deploy keep failing after the auth change", // a prompt + "Sure - here is the rewritten handler you asked for", // a model reply + "func handler(w http.ResponseWriter) { w.WriteHeader(204) }", // code + "sk-ant-api03-NOT-A-REAL-KEY", // an API key + "deepseek-r1-distill", // a model slug + "fatal: authentication failed for 'https://git.internal/x'", // raw error text + "panic: the deploy froze mid-write", // a panic message + "192.168.13.45", // an IP + "CODEAF_HOME=/Users/santosh/secret-project", // an env value +} + +func TestBucketCountEdges(t *testing.T) { + cases := []struct { + count int + want string + }{ + {-7, BucketZero}, + {0, BucketZero}, + {1, BucketOne}, + {2, BucketTwo5}, + {5, BucketTwo5}, + {6, BucketSix}, + {20, BucketSix}, + {21, BucketTwo1}, + {100, BucketTwo1}, + {101, Bucket100}, + {1 << 20, Bucket100}, + } + for _, test := range cases { + if got := BucketCount(test.count); got != test.want { + t.Errorf("BucketCount(%d) = %q, want %q", test.count, got, test.want) + } + } +} + +func TestBucketCostEdges(t *testing.T) { + cases := []struct { + cost float64 + want string + }{ + {-1, CostZero}, + {0, CostZero}, + {0.001, CostUnder1c}, + {0.0099, CostUnder1c}, + {0.01, Cost1cTo10c}, + {0.05, Cost1cTo10c}, + {0.0999, Cost1cTo10c}, + {0.1, Cost10cTo1}, + {0.99, Cost10cTo1}, + {1, Cost1To10}, + {9.99, Cost1To10}, + {10, Cost10Plus}, + {1234.5, Cost10Plus}, + } + for _, test := range cases { + if got := BucketCost(test.cost); got != test.want { + t.Errorf("BucketCost(%v) = %q, want %q", test.cost, got, test.want) + } + } +} + +func TestBucketDurationEdges(t *testing.T) { + cases := []struct { + duration time.Duration + want string + }{ + {-5 * time.Second, DurationUnder1m}, + {0, DurationUnder1m}, + {30 * time.Second, DurationUnder1m}, + {59 * time.Second, DurationUnder1m}, + {time.Minute, Duration1To5m}, + {4*time.Minute + 59*time.Second, Duration1To5m}, + {5 * time.Minute, Duration5To30m}, + {29 * time.Minute, Duration5To30m}, + {30 * time.Minute, Duration30mTo2h}, + {time.Hour + 59*time.Minute, Duration30mTo2h}, + {2 * time.Hour, Duration2hPlus}, + {7 * time.Hour, Duration2hPlus}, + } + for _, test := range cases { + if got := BucketDuration(test.duration); got != test.want { + t.Errorf("BucketDuration(%v) = %q, want %q", test.duration, got, test.want) + } + } +} + +func TestFirstRunCarriesNoSession(t *testing.T) { + testHome(t) + event := FirstRun(testNow) + if event.Name != "first_run" { + t.Fatalf("event name %q, want first_run", event.Name) + } + if event.SessionHash != "" { + t.Error("first_run carries no session hash: an install has no run yet") + } + raw, err := jsonMarshal(event) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "session_id_hash") { + t.Error("the first_run wire row must not carry session_id_hash") + } +} + +func TestEventWireShapeFollowsTheContract(t *testing.T) { + testHome(t) + started := SessionStarted(ModeTask, true, "session-abc", testNow) + raw, err := jsonMarshal(started) + if err != nil { + t.Fatal(err) + } + var row map[string]json.RawMessage + if err := json.Unmarshal(raw, &row); err != nil { + t.Fatalf("the wire row must parse: %v", err) + } + for _, key := range []string{"event_name", "event_id", "install_id_hash", "session_id_hash", "event_time", "props"} { + if _, ok := row[key]; !ok { + t.Errorf("the wire row is missing %q", key) + } + } + for _, key := range []string{"session_id", "install_id", "prompt", "user", "model"} { + if _, ok := row[key]; ok { + t.Errorf("the wire row must not carry %q", key) + } + } + if started.Name != "session_started" { + t.Errorf("event name %q, want session_started", started.Name) + } + if !reEventID.MatchString(started.ID) { + t.Errorf("event_id %q, want 32 lowercase hex characters", started.ID) + } + if !reIdentityHash.MatchString(started.InstallHash) { + t.Errorf("install_id_hash %q, want 64 lowercase hex characters", started.InstallHash) + } + if !reIdentityHash.MatchString(started.SessionHash) { + t.Errorf("session_id_hash %q, want 64 lowercase hex characters", started.SessionHash) + } + if parsed, err := time.Parse(time.RFC3339, started.Time); err != nil || !parsed.Equal(testNow) { + t.Errorf("event_time = %q (%v), want RFC3339 UTC at the build time", started.Time, err) + } + first, err := jsonMarshal(FirstRun(testNow)) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(first), "session_id_hash") { + t.Error("first_run must not carry session_id_hash on the wire") + } +} + +func TestSessionStartedCollapsesUnknownModes(t *testing.T) { + testHome(t) + event := SessionStarted(Mode("interactive"), false, "session-1", testNow) + if got := event.Props["mode"]; got != string(ModeChat) { + t.Errorf("an unknown mode became %v, want chat", got) + } + if got := SessionStarted(ModeTask, false, "s", testNow).Props["mode"]; got != "task" { + t.Errorf("task became %v", got) + } + if got := SessionStarted(ModeChat, true, "s", testNow).Props["resumed"]; got != true { + t.Errorf("resumed became %v, want true", got) + } +} + +func TestSessionEndedBucketsAndClamps(t *testing.T) { + testHome(t) + event := SessionEnded(Mode("interactive"), SessionStats{ + Duration: 90 * time.Minute, + Turns: 3, + ModelCalls: 9, + ModelCallsFailed: 1, + ToolCalls: 21, + ToolCallsFailed: 101, + CostUSD: 0.42, + StopReason: "blew up", + ExitCode: 99, + }, "session-clamp", testNow) + want := map[string]any{ + "mode": "chat", + "duration": Duration30mTo2h, + "turns": BucketTwo5, + "model_calls": BucketSix, + "model_calls_failed": BucketOne, + "tool_calls": BucketTwo1, + "tool_calls_failed": Bucket100, + "cost_usd": Cost10cTo1, + "stop_reason": StopUnknown, + "exit_code": 5, + } + for key, expected := range want { + if event.Props[key] != expected { + t.Errorf("%s = %v, want %v", key, event.Props[key], expected) + } + } + clamped := SessionEnded(ModeChat, SessionStats{StopReason: StopBudget, ExitCode: -3}, "s", testNow) + if clamped.Props["exit_code"] != 0 { + t.Errorf("a negative exit code became %v, want 0", clamped.Props["exit_code"]) + } + if got := clamped.Props["stop_reason"]; got != StopBudget { + t.Errorf("a recognised stop reason became %v, want %q", got, StopBudget) + } + if got := SessionEnded(ModeChat, SessionStats{CostUSD: 0.004}, "s", testNow).Props["cost_usd"]; got != CostUnder1c { + t.Errorf("a sub-cent cost became %v, want %q", got, CostUnder1c) + } +} + +func TestFaultEventBucketsScopeAndMode(t *testing.T) { + testHome(t) + event := FaultEvent(Fault{Mode: "exec", Scope: "somewhere-else", Stack: nil}, "session-f", testNow) + if got := event.Props["mode"]; got != "other" { + t.Errorf("an unknown fault mode became %v, want other", got) + } + if got := event.Props["scope"]; got != ScopeMain { + t.Errorf("an unknown scope became %v, want main", got) + } + if got := FaultEvent(Fault{Mode: string(ModeTask), Scope: ScopeGoroutine, Stack: []byte("github.com/Agent-Field/codeaf/internal/x.One")}, "s", testNow).Props["scope"]; got != ScopeGoroutine { + t.Errorf("goroutine scope became %v", got) + } + if got := FaultEvent(Fault{Mode: string(ModeTask), Scope: ScopeSurface, Stack: nil}, "s", testNow).Props["mode"]; got != "task" { + t.Errorf("task mode became %v", got) + } + if !reFingerprint.MatchString(event.Props["fingerprint"].(string)) { + t.Errorf("fingerprint %v, want 16 lowercase hex characters", event.Props["fingerprint"]) + } +} + +// TestAllowlistedPropsMatchTheConstructors holds the two halves of the +// allowlist together: what the constructors emit and what the table allows +// must be the same set, event by event. +func TestAllowlistedPropsMatchTheConstructors(t *testing.T) { + testHome(t) + events := []Event{ + FirstRun(testNow), + SessionStarted(ModeChat, false, "session-1", testNow), + SessionEnded(ModeChat, SessionStats{Duration: time.Minute, Turns: 1, CostUSD: 1}, "session-1", testNow), + FaultEvent(Fault{Mode: string(ModeChat), Scope: ScopeMain}, "session-1", testNow), + } + if len(events) != len(AllowlistedEvents()) { + t.Fatalf("the constructors cover %d events, the allowlist names %d", len(events), len(AllowlistedEvents())) + } + for _, event := range events { + allowed := map[string]bool{} + for _, name := range AllowlistedProps(event.Name) { + allowed[name] = true + } + if len(event.Props) != len(allowed) { + t.Errorf("%s: the constructor emits %d props, the allowlist names %d", event.Name, len(event.Props), len(allowed)) + } + for key := range event.Props { + if !allowed[key] { + t.Errorf("%s: prop %q is not on the allowlist", event.Name, key) + } + } + } + if got := AllowlistedProps("nosuchevent"); len(got) != 0 { + t.Errorf("AllowlistedProps(nosuchevent) = %v, want none", got) + } + wantEvents := map[string]bool{"first_run": true, "session_started": true, "session_ended": true, "fault": true} + gotEvents := map[string]bool{} + for _, name := range AllowlistedEvents() { + gotEvents[name] = true + } + if !reflect.DeepEqual(gotEvents, wantEvents) { + t.Errorf("AllowlistedEvents = %v, want the contract's four", AllowlistedEvents()) + } + common := map[string]bool{} + for _, name := range CommonPropNames() { + common[name] = true + } + if len(common) != 6 { + t.Errorf("the contract fixes six common props, CommonPropNames lists %d", len(common)) + } +} + +// TestNothingSentinelEverReachesTheWire is the privacy law: every constructor +// is built from inputs stuffed with the private things a person's machine is +// full of, and no marshalled event may carry any of them. +func TestNothingSentinelEverReachesTheWire(t *testing.T) { + testHome(t) + stack := []byte(strings.Join([]string{ + "panic: " + sentinels[11] + " for " + sentinels[1], + "goroutine 1 [running]:", + "github.com/Agent-Field/codeaf/internal/session.(*Runner).step(0xc000123, 0x2)", + "\t" + sentinels[0] + ":412 +0x88", + "github.com/Agent-Field/codeaf/internal/model.(*Client).call(...)", + "\t/Users/" + sentinels[3] + "/" + sentinels[4] + "/client.go:97", + "github.com/Agent-Field/codeaf/cmd/codeaf.runChat()", + "\t/Users/" + sentinels[3] + "/main.go:201", + }, "\n")) + events := []Event{ + FirstRun(testNow), + SessionStarted(Mode(sentinels[2]), true, "session "+sentinels[1], testNow), + SessionEnded(ModeChat, SessionStats{ + Duration: 42 * time.Minute, Turns: 3, ModelCalls: 9, ModelCallsFailed: 1, + ToolCalls: 12, ToolCallsFailed: 2, CostUSD: 0.4, StopReason: StopDone, ExitCode: 0, + }, "session "+sentinels[0], testNow), + FaultEvent(Fault{Mode: sentinels[9], Scope: sentinels[2], Stack: stack}, "session "+sentinels[8], testNow), + } + for _, event := range events { + raw, err := jsonMarshal(event) + if err != nil { + t.Fatalf("%s: %v", event.Name, err) + } + for _, sentinel := range sentinels { + if strings.Contains(string(raw), sentinel) { + t.Errorf("%s carries %q on the wire", event.Name, sentinel) + } + } + } +} diff --git a/internal/telemetry/fingerprint.go b/internal/telemetry/fingerprint.go index 7b876ff034..8f8866c202 100644 --- a/internal/telemetry/fingerprint.go +++ b/internal/telemetry/fingerprint.go @@ -31,6 +31,38 @@ func Fingerprint(stack []byte) string { return hex.EncodeToString(sum[:8]) } +// cutArguments drops the trailing argument group from a printed frame line: +// scanning back from the final ')', the group is the argument list only when +// what precedes it ends in a name character — a receiver group like +// (*Runner) sits inside the name and is kept. +func cutArguments(line string) string { + if !strings.HasSuffix(line, ")") || len(line) < 2 { + return line + } + depth := 0 + for i := len(line) - 1; i >= 0; i-- { + switch line[i] { + case ')': + depth++ + case '(': + depth-- + if depth == 0 { + prefix := line[:i] + if prefix == "" { + return line + } + last := prefix[len(prefix)-1] + if last == '_' || (last >= '0' && last <= '9') || + (last >= 'a' && last <= 'z') || (last >= 'A' && last <= 'Z') { + return prefix + } + return line + } + } + } + return line +} + // FingerprintHere fingerprints the calling goroutine's own stack, the shape a // deferred recover has on hand. func FingerprintHere() string { @@ -51,11 +83,9 @@ func codeafFrames(stack []byte) []string { } // "pkg.Func" or "pkg.Func.Shape.Method" or "pkg.Func-fm" — never the // tab-indented file:line half, which never starts with the module. - name := line - if idx := strings.IndexAny(name, " \t"); idx >= 0 { - name = name[:idx] - } - names = append(names, name) + // The argument list is dropped by balanced matching so a wide argument + // list cannot leave half an argument behind. + names = append(names, cutArguments(line)) if len(names) == fingerprintFrames { break } diff --git a/internal/telemetry/fingerprint_test.go b/internal/telemetry/fingerprint_test.go new file mode 100644 index 0000000000..cb31ad6c54 --- /dev/null +++ b/internal/telemetry/fingerprint_test.go @@ -0,0 +1,103 @@ +package telemetry + +import ( + "strings" + "testing" +) + +// codeafStack renders bare function names the way a stack the program built +// itself would carry them. +func codeafStack(names ...string) []byte { + lines := make([]string, len(names)) + for i, name := range names { + lines[i] = "github.com/Agent-Field/codeaf/" + name + } + return []byte(strings.Join(lines, "\n")) +} + +func TestFingerprintIsStableHex(t *testing.T) { + stack := codeafStack("internal/session.(*Runner).step", "internal/model.(*Client).call") + first := Fingerprint(stack) + if !reFingerprint.MatchString(first) { + t.Fatalf("Fingerprint = %q, want 16 lowercase hex characters", first) + } + if again := Fingerprint(stack); again != first { + t.Errorf("the same stack fingerprinted as %q and %q", first, again) + } +} + +func TestFingerprintReadsFunctionNamesOnly(t *testing.T) { + // The same three frames, first bare, then in the shape debug.Stack + // prints them: arguments on the frame line, file and line beneath. + printed := strings.Join([]string{ + "github.com/Agent-Field/codeaf/internal/session.(*Runner).step(0xc0001, 0x2)", + "\t/Users/santosh/secret-project/internal/session/runner.go:412 +0x88", + "github.com/Agent-Field/codeaf/internal/model.(*Client).call(...)", + "\t/Users/santosh/secret-project/internal/model/client.go:97 +0x1c", + "github.com/Agent-Field/codeaf/cmd/codeaf.runChat()", + "\t/Users/santosh/secret-project/cmd/codeaf/main.go:201 +0x64", + }, "\n") + bare := Fingerprint(codeafStack( + "internal/session.(*Runner).step", + "internal/model.(*Client).call", + "cmd/codeaf.runChat", + )) + if got := Fingerprint([]byte(printed)); got != bare { + t.Errorf("paths and line numbers moved the fingerprint: %q against %q", got, bare) + } +} + +func TestFingerprintIgnoresForeignFramesAndPanicText(t *testing.T) { + core := codeafStack("internal/session.(*Runner).step", "internal/model.(*Client).call") + noisy := strings.Join([]string{ + "panic: the deploy froze mid-write for person@example.com", + "/Users/santosh/secret-project/leaky.go:7 +0x10", + "some.dependency/pkg.(*Thing).Do(0x1)", + "github.com/Agent-Field/codeaf/internal/session.(*Runner).step(0x1, 0x2)", + "\t/tmp/whatever/runner.go:1", + "github.com/Agent-Field/codeaf/internal/model.(*Client).call(...)", + }, "\n") + if got, want := Fingerprint([]byte(noisy)), Fingerprint(core); got != want { + t.Errorf("foreign frames and panic text moved the fingerprint: %q against %q", got, want) + } +} + +func TestFingerprintHashesOnlyTheTopFiveCodeafFrames(t *testing.T) { + five := Fingerprint(codeafStack("a", "b", "c", "d", "e")) + six := Fingerprint(codeafStack("a", "b", "c", "d", "e", "f")) + if five != six { + t.Error("a sixth codeaf frame must not move the fingerprint") + } + if changed := Fingerprint(codeafStack("a", "b", "c", "d", "z")); changed == five { + t.Error("a changed fifth frame must move the fingerprint") + } +} + +func TestFingerprintOfAForeignStackGroupsAsEmpty(t *testing.T) { + foreign := strings.Join([]string{ + "main.main()", + "\t/tmp/other-project/main.go:9", + "runtime.main()", + }, "\n") + if got := Fingerprint([]byte(foreign)); got != Fingerprint(nil) { + t.Errorf("a stack with no codeaf frames must group as the empty fingerprint, got %q", got) + } + if !reFingerprint.MatchString(Fingerprint(nil)) { + t.Error("the empty fingerprint must still be 16 lowercase hex characters") + } +} + +func TestFingerprintHereMatchesTheRunningStack(t *testing.T) { + first := FingerprintHere() + if !reFingerprint.MatchString(first) { + t.Fatalf("FingerprintHere = %q, want 16 lowercase hex characters", first) + } + if again := FingerprintHere(); again != first { + t.Errorf("two calls from the same frame answered %q and %q", first, again) + } + if nested := fingerprintFromHelper(); nested == first { + t.Error("two different call sites must not share a fingerprint") + } +} + +func fingerprintFromHelper() string { return FingerprintHere() } diff --git a/internal/telemetry/spool.go b/internal/telemetry/spool.go index 3b47d9221c..258de410e8 100644 --- a/internal/telemetry/spool.go +++ b/internal/telemetry/spool.go @@ -11,6 +11,8 @@ import ( "strings" "sync" "time" + + "github.com/Agent-Field/codeaf/internal/guard" ) // Spool limits, from the contract: a thousand lines of history, seven days of @@ -57,16 +59,16 @@ func oneWarning(message string) { }) } -// Spool appends one event to the spool file and returns immediately. It never -// blocks the caller for more than a few milliseconds: it does no network work, -// and the one file append it does is a single WriteFile on a line that was -// already marshalled. A failure here is silent. +// Spool appends one event to the spool file and returns immediately. It +// never blocks the caller for more than a few milliseconds: it does no network +// work, and its one append runs on its own goroutine through guard.Go, the +// repo's one door for fire-and-forget work. A failure here is silent. func Spool(event Event) { line, err := jsonMarshal(event) if err != nil { return } - go persist([]byte(line)) + guard.Go("telemetry.spool", func() { _ = persist(line) }) } // SpoolSync appends one event and waits for the append, for tests and for a @@ -148,6 +150,12 @@ func Flush(ctx context.Context) error { dropped = true continue } + if !allowlistedEvents[entry.event.EventName] { + // An event name the contract does not define can never be sent + // validly, so it is discarded rather than posted or kept forever. + dropped = true + continue + } kept = append(kept, entry) batch = append(batch, entry) if len(batch) == MaxEventsPerPOST { @@ -227,15 +235,33 @@ func sendBatch(ctx context.Context, deadline time.Time, batch []spoolEntry) bool return ok } -// batchEvents returns the wire rows of one batch, in order. +// batchEvents returns the wire rows of one batch, in order, with the prop +// allowlist re-applied: a line spooled by an older build must not send a prop +// key the contract dropped since. func batchEvents(batch []spoolEntry) []jsonEvent { out := make([]jsonEvent, len(batch)) for i, entry := range batch { - out[i] = entry.event + row := entry.event + if allowed := allowedProps[row.EventName]; len(allowed) > 0 { + clean := make(map[string]any, len(row.Props)) + for key, value := range row.Props { + if allowed[key] { + clean[key] = value + } + } + row.Props = clean + } + out[i] = row } return out } +// allowlistedEvents is the event-name half of the contract's allowlist, for +// the flush loop's one look per line. +var allowlistedEvents = map[string]bool{ + "first_run": true, "session_started": true, "session_ended": true, "fault": true, +} + // httpClient is the one client. No timeout of its own: the deadline lives in // the request context, where the caller's hard budget is. var httpClient = &http.Client{} diff --git a/internal/telemetry/spool_test.go b/internal/telemetry/spool_test.go new file mode 100644 index 0000000000..3912ce1dc0 --- /dev/null +++ b/internal/telemetry/spool_test.go @@ -0,0 +1,359 @@ +package telemetry + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync" + "testing" + "time" +) + +// recordedPost is one POST the relay saw. +type recordedPost struct { + events []map[string]any + schemaVer float64 + contentType string + raw string +} + +type relayRecorder struct { + mu sync.Mutex + posts []recordedPost +} + +func (r *relayRecorder) ServeHTTP(writer http.ResponseWriter, request *http.Request) { + body, err := io.ReadAll(request.Body) + if err != nil { + writer.WriteHeader(http.StatusBadRequest) + return + } + var parsed struct { + SchemaVersion float64 `json:"schema_version"` + Events []map[string]any `json:"events"` + } + parseErr := json.Unmarshal(body, &parsed) + r.mu.Lock() + r.posts = append(r.posts, recordedPost{ + events: parsed.Events, + schemaVer: parsed.SchemaVersion, + contentType: request.Header.Get("Content-Type"), + raw: string(body), + }) + r.mu.Unlock() + if parseErr != nil { + writer.WriteHeader(http.StatusBadRequest) + return + } + writer.WriteHeader(http.StatusNoContent) +} + +func (r *relayRecorder) count() int { + r.mu.Lock() + defer r.mu.Unlock() + return len(r.posts) +} + +// newRelay starts a recording relay and points CODEAF_TELEMETRY_ENDPOINT at +// it, the way the endpoint override is meant to be used. +func newRelay(t *testing.T) *relayRecorder { + t.Helper() + recorder := &relayRecorder{} + server := httptest.NewServer(recorder) + t.Cleanup(server.Close) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", server.URL) + return recorder +} + +func TestFlushSendsFiftyPerPostAndRemovesSentLines(t *testing.T) { + testHome(t) + recorder := newRelay(t) + MarkNoticeShown() + for i := 0; i < 120; i++ { + if err := SpoolSync(SessionStarted(ModeChat, false, fmt.Sprintf("session-%d", i), freshClock(t))); err != nil { + t.Fatalf("spooling event %d: %v", i, err) + } + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush must never fail a run: %v", err) + } + if got := recorder.count(); got != 3 { + t.Fatalf("the relay saw %d posts, want 3 (50 + 50 + 20)", got) + } + seen := map[string]bool{} + for i := range recorder.posts { + if i < 2 && len(recorder.posts[i].events) != MaxEventsPerPOST { + t.Errorf("post %d carried %d events, want %d", i, len(recorder.posts[i].events), MaxEventsPerPOST) + } + if recorder.posts[i].schemaVer != schemaVersion { + t.Errorf("post %d carried schema_version %v, want %d", i, recorder.posts[i].schemaVer, schemaVersion) + } + if recorder.posts[i].contentType != "application/json" { + t.Errorf("post %d carried Content-Type %q", i, recorder.posts[i].contentType) + } + for _, event := range recorder.posts[i].events { + if id, ok := event["event_id"].(string); ok { + seen[id] = true + } + } + } + if len(seen) != 120 { + t.Errorf("the relay received %d distinct events, want 120", len(seen)) + } + if left := len(SpoolContents()); left != 0 { + t.Errorf("%d lines remain in the spool after everything was confirmed sent, want 0", left) + } +} + +func TestFlushIsSilentUntilTheNoticeWasShown(t *testing.T) { + testHome(t) + recorder := newRelay(t) + if err := SpoolSync(SessionStarted(ModeChat, false, "session-gated", freshClock(t))); err != nil { + t.Fatal(err) + } + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := recorder.count(); got != 0 { + t.Fatalf("the relay saw %d posts before the notice was marked shown, want 0", got) + } + if left := len(SpoolContents()); left != 1 { + t.Fatalf("%d lines remain after a gated flush, want 1", left) + } + MarkNoticeShown() + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := recorder.count(); got != 1 { + t.Fatalf("after MarkNoticeShown the relay saw %d posts, want 1", got) + } +} + +func TestFlushDropsEventsOlderThanSevenDays(t *testing.T) { + testHome(t) + recorder := newRelay(t) + MarkNoticeShown() + + fresh := SessionStarted(ModeChat, false, "session-fresh", freshClock(t)) + ancient := SessionStarted(ModeChat, false, "session-ancient", freshClock(t)) + ancient.Time = freshClock(t).Add(-8 * 24 * time.Hour).UTC().Format(time.RFC3339) + // A spooled line just inside the seven days stays. + border := SessionStarted(ModeChat, false, "session-border", freshClock(t)) + border.Time = freshClock(t).Add(-7*24*time.Hour + time.Minute).UTC().Format(time.RFC3339) + + if err := SpoolSync(ancient); err != nil { + t.Fatal(err) + } + if err := SpoolSync(border); err != nil { + t.Fatal(err) + } + if err := SpoolSync(fresh); err != nil { + t.Fatal(err) + } + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := recorder.count(); got != 1 { + t.Fatalf("the relay saw %d posts, want 1", got) + } + events := recorder.posts[0].events + if len(events) != 2 { + t.Fatalf("the post carried %d events, want 2 (the ancient one is dropped)", len(events)) + } + ids := map[string]bool{} + for _, event := range events { + if id, ok := event["event_id"].(string); ok { + ids[id] = true + } + } + if !ids[fresh.ID] || !ids[border.ID] || ids[ancient.ID] { + t.Errorf("the wrong events were sent: fresh %v, border %v, ancient %v", ids[fresh.ID], ids[border.ID], ids[ancient.ID]) + } + if left := len(SpoolContents()); left != 0 { + t.Errorf("%d lines remain, want 0", left) + } +} + +func TestFlushRespectsTheCallerDeadline(t *testing.T) { + testHome(t) + block := make(chan struct{}) + hanging := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + <-block + writer.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(func() { + close(block) + hanging.Close() + }) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", hanging.URL) + MarkNoticeShown() + for i := 0; i < 3; i++ { + if err := SpoolSync(SessionStarted(ModeChat, false, fmt.Sprintf("session-hang-%d", i), freshClock(t))); err != nil { + t.Fatal(err) + } + } + start := time.Now() + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + err := Flush(ctx) + cancel() + elapsed := time.Since(start) + if err != nil { + t.Fatalf("Flush returned %v; it must never fail a run", err) + } + if elapsed > 2*time.Second { + t.Fatalf("Flush took %s against a hanging relay; the caller's deadline is hard", elapsed) + } + if left := len(SpoolContents()); left != 3 { + t.Errorf("%d lines remain after an unconfirmed send, want all 3 still spooled", left) + } +} + +func TestSpoolCapsAtOneThousandLinesDroppingTheOldest(t *testing.T) { + testHome(t) + if err := ensureDir(); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for i := 0; i < 1005; i++ { + line, err := jsonMarshal(SessionStarted(ModeChat, false, fmt.Sprintf("session-%d", i), freshClock(t))) + if err != nil { + t.Fatal(err) + } + buf.Write(line) + buf.WriteByte('\n') + } + if err := os.WriteFile(spoolPath(), buf.Bytes(), 0o600); err != nil { + t.Fatal(err) + } + if err := SpoolSync(SessionStarted(ModeChat, false, "the-newest", freshClock(t))); err != nil { + t.Fatal(err) + } + contents := SpoolContents() + if len(contents) != MaxSpoolLines { + t.Fatalf("the spool holds %d lines, want %d", len(contents), MaxSpoolLines) + } + var oldest, newest map[string]any + if err := json.Unmarshal(contents[0], &oldest); err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(contents[len(contents)-1], &newest); err != nil { + t.Fatal(err) + } + if got := oldest["session_id_hash"]; got != hashHex("session-6") { + t.Errorf("the oldest kept line is session-6's, got the hash of %q", got) + } + if got := newest["session_id_hash"]; got != hashHex("the-newest") { + t.Errorf("the newest line is %v, want the-newest", got) + } +} + +func TestFlushDropsPropKeysAndEventNamesTheContractDoesNotAllow(t *testing.T) { + testHome(t) + recorder := newRelay(t) + MarkNoticeShown() + + // A spool line carrying a prop key the contract does not name — written by + // an older build, or by hand. The key must not reach the wire. + edited := SessionStarted(ModeChat, false, "session-edited", freshClock(t)) + edited.Props["prompt"] = sentinels[5] + if err := SpoolSync(edited); err != nil { + t.Fatal(err) + } + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := recorder.count(); got != 1 { + t.Fatalf("the relay saw %d posts, want 1", got) + } + if strings.Contains(recorder.posts[0].raw, "prompt") || strings.Contains(recorder.posts[0].raw, sentinels[5]) { + t.Error("an unallowlisted prop key reached the wire") + } + + // An event whose name the contract does not define can never be sent + // validly, so it is discarded rather than posted or kept forever. + stray := fmt.Sprintf(`{"event_name":"git_remote_probe","event_id":%q,"install_id_hash":%q,"event_time":%q,"props":{"remote":"git@github.com:Agent-Field/secret.git"}}`, + randomHex(16), InstallIDHash(), roundTime(freshClock(t))) + file, err := os.OpenFile(spoolPath(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + t.Fatal(err) + } + if _, err := file.WriteString(stray + "\n"); err != nil { + t.Fatal(err) + } + file.Close() + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := recorder.count(); got != 1 { + t.Errorf("the relay saw %d posts after an unsendable line was spooled, want 1", got) + } + for _, post := range recorder.posts { + if strings.Contains(post.raw, "git_remote_probe") { + t.Error("an event name the contract does not define reached the wire") + } + } + if left := len(SpoolContents()); left != 0 { + t.Errorf("%d lines remain after the flush, want 0", left) + } +} + +func TestAFailingRelayKeepsTheSpoolAndStaysSilent(t *testing.T) { + testHome(t) + // Nothing listens here. + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "http://127.0.0.1:1/telemetry") + MarkNoticeShown() + if err := SpoolSync(SessionStarted(ModeChat, false, "session-lost", freshClock(t))); err != nil { + t.Fatal(err) + } + old := os.Stderr + reader, writer, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stderr = writer + captured := make(chan string, 1) + go func() { + var buf bytes.Buffer + _, _ = io.Copy(&buf, reader) + captured <- buf.String() + }() + flushErr := Flush(context.Background()) + os.Stderr = old + writer.Close() + if flushErr != nil { + t.Fatalf("Flush returned %v against a dead relay; it must never fail a run", flushErr) + } + if noise := <-captured; strings.TrimSpace(noise) != "" { + t.Errorf("a failed flush must stay silent, wrote %q to stderr", noise) + } + if left := len(SpoolContents()); left != 1 { + t.Errorf("%d lines remain after a failed send, want 1", left) + } +} + +func TestShowReturnsTheSpoolAsPrettyJSON(t *testing.T) { + testHome(t) + if got := Show(); got != "[]" { + t.Fatalf("an empty spool shows %q, want []", got) + } + event := SessionStarted(ModeTask, false, "session-shown", freshClock(t)) + if err := SpoolSync(event); err != nil { + t.Fatal(err) + } + var shown []map[string]any + if err := json.Unmarshal([]byte(Show()), &shown); err != nil { + t.Fatalf("Show must print valid JSON: %v", err) + } + if len(shown) != 1 { + t.Fatalf("Show printed %d events, want 1", len(shown)) + } + if shown[0]["event_id"] != event.ID { + t.Errorf("Show printed event_id %v, want %q", shown[0]["event_id"], event.ID) + } +} diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index 3300f8b89b..2e2f3a3d56 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -189,7 +189,8 @@ func installMethod() string { func channelFor(revision string) string { switch { case strings.Contains(revision, "-rc."): - if isNumericDotted(strings.TrimPrefix(strings.TrimSuffix(revision, revision[strings.LastIndex(revision, "-rc."):]), "v")) { + suffix := revision[strings.LastIndex(revision, "-rc.")+len("-rc."):] + if isRcSuffix(suffix) { return "rc" } return "unknown" @@ -203,6 +204,22 @@ func channelFor(revision string) string { return "unknown" } +// isRcSuffix matches the rc suffix the contract's tags carry: rc.N with N an +// integer, and nothing wider. +func isRcSuffix(suffix string) bool { return allDigits(suffix) } + +func allDigits(value string) bool { + if value == "" { + return false + } + for _, r := range value { + if r < '0' || r > '9' { + return false + } + } + return true +} + // isStableTag matches vX.Y.Z with three numeric parts and nothing else. func isStableTag(revision string) bool { if !strings.HasPrefix(revision, "v") { diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go new file mode 100644 index 0000000000..fd740d1b29 --- /dev/null +++ b/internal/telemetry/telemetry_test.go @@ -0,0 +1,378 @@ +package telemetry + +import ( + "crypto/sha256" + "encoding/hex" + "io" + "os" + "strings" + "testing" + "time" + + "github.com/Agent-Field/codeaf/internal/home" +) + +// testNow is the clock every test event is built at, so hashes, ages and +// buckets are stable run to run. +var testNow = time.Date(2026, 1, 14, 9, 30, 0, 0, time.UTC) + +// freshClock is the clock spool tests build their events at. The age-drop law +// is the real current time's, so an event built at any fixed stamp grows +// stale and is dropped by the contract's seven-day law — every assertion +// about what a flush sends must use a clock that agrees with the one ageOf +// reads. +func freshClock(t *testing.T) time.Time { return time.Now() } + +// testHome points the state root at a directory this test chose, so nothing +// this package writes can land in the home of whoever ran `go test`, and +// clears every variable the opt-out ladder reads, so a test that wants +// telemetry on gets an environment that agrees. Every test in the package +// starts here. +func testHome(t *testing.T) { + t.Helper() + t.Setenv(home.EnvVar, t.TempDir()) + for _, name := range []string{ + "CODEAF_TELEMETRY", "AFORGE_TELEMETRY", + "CODEAF_TELEMETRY_ENDPOINT", "AFORGE_TELEMETRY_ENDPOINT", + "DO_NOT_TRACK", + "CI", "GITHUB_ACTIONS", "GITLAB_CI", "BUILDKITE", "CIRCLECI", "JENKINS_URL", + "KUBERNETES_SERVICE_HOST", + } { + unsetEnv(t, name) + } +} + +// unsetEnv removes one variable for the test and puts it back afterwards, the +// way t.Setenv would if it could unset. +func unsetEnv(t *testing.T, name string) { + t.Helper() + old, had := os.LookupEnv(name) + os.Unsetenv(name) + t.Cleanup(func() { + if had { + os.Setenv(name, old) + return + } + os.Unsetenv(name) + }) +} + +func isLowerHex(s string) bool { + if s == "" { + return false + } + for _, r := range s { + switch { + case r >= '0' && r <= '9', r >= 'a' && r <= 'f': + default: + return false + } + } + return true +} + +// The ladder is driven through offReason, the one pure function every rung +// lives in. Enabled and OffReason are never asked to answer ON here: under +// `go test` they are forced off by law, so only the pure ladder can exercise +// the on rungs. + +func TestLadderRunsWhenNobodyAskedItNotTo(t *testing.T) { + testHome(t) + if got := offReason(false, false, false); got != OnReason { + t.Fatalf("a clean environment: ladder answered %q, want on", got) + } +} + +func TestLadderCodeafTelemetrySwitch(t *testing.T) { + testHome(t) + for _, value := range []string{"off", "OFF", "0", "false"} { + t.Setenv("CODEAF_TELEMETRY", value) + if got := offReason(false, false, false); got != OffEnv { + t.Errorf("CODEAF_TELEMETRY=%s: got %q, want %q", value, got, OffEnv) + } + } + t.Setenv("CODEAF_TELEMETRY", "on") + if got := offReason(false, false, false); got != OnReason { + t.Errorf("CODEAF_TELEMETRY=on: got %q, want on", got) + } + // The former spelling answers when the current one is unset. + t.Setenv("CODEAF_TELEMETRY", "") + t.Setenv("AFORGE_TELEMETRY", "off") + if got := offReason(false, false, false); got != OffEnv { + t.Errorf("AFORGE_TELEMETRY=off: got %q, want %q", got, OffEnv) + } +} + +func TestLadderDoNotTrackSwitch(t *testing.T) { + testHome(t) + for _, value := range []string{"1", "true", "TRUE", " true "} { + t.Setenv("DO_NOT_TRACK", value) + if got := offReason(false, false, false); got != OffDoNotTrack { + t.Errorf("DO_NOT_TRACK=%q: got %q, want %q", value, got, OffDoNotTrack) + } + } + t.Setenv("DO_NOT_TRACK", "0") + if got := offReason(false, false, false); got != OnReason { + t.Errorf("DO_NOT_TRACK=0: got %q, want on", got) + } +} + +func TestLadderRungsInOrder(t *testing.T) { + testHome(t) + if got := offReason(true, false, false); got != OffConfig { + t.Errorf("config off: got %q, want %q", got, OffConfig) + } + // A person's own switch outranks the config file. Each rung is tested from + // its own clean environment so no earlier rung is left armed. + t.Run("env switch beats config", func(t *testing.T) { + testHome(t) + t.Setenv("CODEAF_TELEMETRY", "off") + if got := offReason(true, false, false); got != OffEnv { + t.Errorf("env off with config off: got %q, want %q", got, OffEnv) + } + }) + t.Run("do not track beats config", func(t *testing.T) { + testHome(t) + t.Setenv("DO_NOT_TRACK", "1") + if got := offReason(true, false, false); got != OffDoNotTrack { + t.Errorf("DO_NOT_TRACK with config off: got %q, want %q", got, OffDoNotTrack) + } + }) + t.Run("build honesty beats runtime", func(t *testing.T) { + testHome(t) + t.Setenv("DO_NOT_TRACK", "") + if got := offReason(true, false, true); got != OffConfig { + t.Errorf("config off with a dirty build: got %q, want %q", got, OffConfig) + } + }) +} + +func TestLadderEndpointOverride(t *testing.T) { + testHome(t) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "") + if got := offReason(false, false, false); got != OffEmptyEndpoint { + t.Errorf("empty endpoint: got %q, want %q", got, OffEmptyEndpoint) + } + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "https://example.invalid/relay") + if got := offReason(false, false, false); got != OnReason { + t.Errorf("set endpoint: got %q, want on", got) + } +} + +func TestLadderBuildAndTestRungs(t *testing.T) { + testHome(t) + if got := offReason(false, false, true); got != OffBuild { + t.Errorf("dirty build: got %q, want %q", got, OffBuild) + } + if got := offReason(false, true, false); got != OffTest { + t.Errorf("test binary: got %q, want %q", got, OffTest) + } + // The build's honesty outranks the runtime. + if got := offReason(false, true, true); got != OffBuild { + t.Errorf("dirty build in a test: got %q, want %q", got, OffBuild) + } +} + +// Enabled is asserted only in the direction `go test` forces: a test binary +// never sends, whatever the config says. +func TestEnabledIsAlwaysOffUnderGoTest(t *testing.T) { + testHome(t) + if Enabled(false) || Enabled(true) { + t.Fatal("a test binary must never send telemetry") + } + // The build's honesty and the test binary both force off; with the config + // off as well, the config rung answers first, per the contract's order. + if got := OffReason(true); got != OffConfig { + t.Fatalf("OffReason under go test with config off: got %q, want %q", got, OffConfig) + } + if got := OffReason(false); got != OffBuild { + t.Fatalf("OffReason under go test from a dev checkout: got %q, want %q (an unstamped build trips the build rung)", got, OffBuild) + } +} + +func TestInstallIdentityCreatesPrivateFiles(t *testing.T) { + testHome(t) + hash := InstallIDHash() + if len(hash) != 64 || !isLowerHex(hash) { + t.Fatalf("InstallIDHash = %q, want 64 lowercase hex characters", hash) + } + if fi, err := os.Stat(telemetryDir()); err != nil || fi.Mode().Perm() != 0o700 { + t.Errorf("telemetry dir mode = %v (%v), want 0700", fi.Mode().Perm(), err) + } + body, err := os.ReadFile(telemetryFile("install_id")) + if err != nil { + t.Fatalf("install_id must be created: %v", err) + } + id := strings.TrimSpace(string(body)) + if len(id) != 64 || !isLowerHex(id) { + t.Fatalf("install_id = %q, want 64 lowercase hex characters (32 random bytes)", id) + } + if fi, err := os.Stat(telemetryFile("install_id")); err != nil || fi.Mode().Perm() != 0o600 { + t.Errorf("install_id mode = %v (%v), want 0600", fi.Mode().Perm(), err) + } + sum := sha256.Sum256([]byte(id)) + if hex.EncodeToString(sum[:]) != hash { + t.Error("the wire identity must be sha256 of the stored install id") + } + if sum := InstallIDHash(); sum != hash { + t.Error("the same install must answer the same identity") + } +} + +func TestInstallIdentityIsStableUntilTheIDChanges(t *testing.T) { + testHome(t) + first := InstallIDHash() + if again := InstallIDHash(); again != first { + t.Fatal("the same install must answer the same identity") + } + if err := os.WriteFile(telemetryFile("install_id"), []byte(randomHex(32)), 0o600); err != nil { + t.Fatal(err) + } + if fresh := InstallIDHash(); fresh == first { + t.Fatal("a replaced install id must answer a new identity") + } +} + +func TestFirstRunIsEmittedOncePerInstallID(t *testing.T) { + testHome(t) + if !FirstRunPending() { + t.Fatal("a fresh install has not sent first_run yet") + } + MarkFirstRunSent() + if FirstRunPending() { + t.Fatal("after the marker, first_run must not be pending") + } + // A different install id is a different identity: first_run is pending again. + if err := os.WriteFile(telemetryFile("install_id"), []byte(randomHex(32)), 0o600); err != nil { + t.Fatal(err) + } + if !FirstRunPending() { + t.Fatal("a new install id has not sent first_run yet") + } +} + +// The notice is the contract's text, byte for byte. The contract file itself +// is not committed, so the expected text lives here. +const contractNotice = `codeaf sends anonymous usage counts to AgentField. + Sent: version, OS, mode (chat or task), how many sessions, how many errors. + Never: anything about you or your work. No prompts, code, file names, + paths, repo names, keys, email, IP, or machine name. + See exactly what leaves: codeaf telemetry show + Turn off: CODEAF_TELEMETRY=off` + +func TestNoticeIsTheContractText(t *testing.T) { + if Notice != contractNotice { + t.Errorf("Notice drifted from the contract:\n got: %q\nwant: %q", Notice, contractNotice) + } +} + +func TestPrintNoticeWritesOneLineOnce(t *testing.T) { + old := os.Stderr + reader, writer, err := os.Pipe() + if err != nil { + t.Fatal(err) + } + os.Stderr = writer + captured := make(chan string, 1) + go func() { + var buf strings.Builder + _, _ = io.Copy(&buf, reader) + captured <- buf.String() + }() + PrintNotice() + PrintNotice() + os.Stderr = old + writer.Close() + got := <-captured + if count := strings.Count(got, Notice); count != 1 { + t.Errorf("the notice printed %d times in one process, want 1", count) + } +} + +func TestUsageContextBuckets(t *testing.T) { + testHome(t) + t.Setenv("CI", "1") + if got := usageContext(); got != "ci" { + t.Errorf("CI=1: got %q, want ci", got) + } + // A CI variable that says no is not CI; the next one answers. + t.Setenv("CI", "0") + t.Setenv("GITHUB_ACTIONS", "true") + if got := usageContext(); got != "ci" { + t.Errorf("GITHUB_ACTIONS=true: got %q, want ci", got) + } + // Outside CI a run is a container or a person's machine, never anything else. + t.Setenv("GITHUB_ACTIONS", "") + if got := usageContext(); got != "local" && got != "container" { + t.Errorf("clean environment: got %q, want local or container", got) + } + // A container without a dockerenv file still shows as one. + t.Setenv("KUBERNETES_SERVICE_HOST", "10.0.0.1") + if got := usageContext(); got != "container" { + t.Errorf("KUBERNETES_SERVICE_HOST set: got %q, want container", got) + } +} + +func TestChannelFromRevisionShape(t *testing.T) { + cases := []struct { + revision string + want string + }{ + {"v1.2.3", "stable"}, + {"v0.0.0", "stable"}, + {"v1.2.3-rc.1", "rc"}, + {"v10.0.2-rc.12", "rc"}, + {"dev-20260114-0123456789ab", "dev"}, + {"staging-20260114-0123456789ab", "staging"}, + {"", "unknown"}, + {"v1.2", "unknown"}, + {"v1.2.3.4", "unknown"}, + {"1.2.3", "unknown"}, + {"release-1.2.3", "unknown"}, + {"v1.2.3-rc.x", "unknown"}, + {"dev-20260114-notahexvalue", "unknown"}, + {"dev-202601-0123456789ab", "unknown"}, + {"aReleaseTag/from/somewhere", "unknown"}, + } + for _, test := range cases { + if got := channelFor(test.revision); got != test.want { + t.Errorf("channelFor(%q) = %q, want %q", test.revision, got, test.want) + } + } +} + +func TestInstallMethodReadsOnlyTheInstallerWords(t *testing.T) { + testHome(t) + if got := installMethod(); got != "unknown" { + t.Errorf("no install.json: got %q, want unknown", got) + } + WriteInstallMethod("script") + if got := installMethod(); got != "script" { + t.Errorf("WriteInstallMethod(script): got %q, want script", got) + } + // A stray word from anywhere else is stored as unknown and read as unknown. + WriteInstallMethod("brew install codeaf") + if got := installMethod(); got != "unknown" { + t.Errorf("WriteInstallMethod(brew install codeaf): got %q, want unknown", got) + } + if err := os.WriteFile(telemetryFile("install.json"), []byte(`{"install_method":"/Users/santosh/secret-project"}`), 0o600); err != nil { + t.Fatal(err) + } + if got := installMethod(); got != "unknown" { + t.Errorf("a path in install.json: got %q, want unknown", got) + } +} + +func TestEndpointOverrideAndDefault(t *testing.T) { + testHome(t) + if got := Endpoint(); got != DefaultEndpoint { + t.Errorf("Endpoint() = %q, want the contract relay %q", got, DefaultEndpoint) + } + if DefaultEndpoint != "https://agentfield.ai/api/oss/codeaf/telemetry" { + t.Errorf("DefaultEndpoint = %q, want the contract relay", DefaultEndpoint) + } + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "https://example.invalid/relay") + if got := Endpoint(); got != "https://example.invalid/relay" { + t.Errorf("Endpoint() = %q, want the override", got) + } +} From 9c403c8d100dcc73747fc44a83487a5e48d2b5a2 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 01:18:04 -0400 Subject: [PATCH 03/36] telemetry: fix review defects 1-8 with tests Fingerprint stability (last-( cut, .go:/main./created-by frame rules), counted-batch flush, guarded and multi-process-safe spool, BucketDuration edges, Configure-gated write paths, O_EXCL install id, install.json preservation, buildinfo.Dirty accessor. Co-Authored-By: codeaf --- internal/buildinfo/buildinfo.go | 6 + internal/buildinfo/buildinfo_test.go | 16 ++ internal/telemetry/fingerprint.go | 31 ++- internal/telemetry/fingerprint_test.go | 95 ++++++- internal/telemetry/install.go | 61 ++++- internal/telemetry/install_test.go | 79 ++++++ internal/telemetry/spool.go | 228 ++++++++++++----- internal/telemetry/spool_test.go | 330 ++++++++++++++++++++++++- internal/telemetry/telemetry.go | 101 ++++++-- internal/telemetry/telemetry_test.go | 125 +++++++++- 10 files changed, 964 insertions(+), 108 deletions(-) create mode 100644 internal/telemetry/install_test.go diff --git a/internal/buildinfo/buildinfo.go b/internal/buildinfo/buildinfo.go index 0b7be62272..914f77d7ba 100644 --- a/internal/buildinfo/buildinfo.go +++ b/internal/buildinfo/buildinfo.go @@ -180,3 +180,9 @@ func newRunningDetector() *Detector { func StaleNotice() string { return running.Notice() } + +// Dirty reports whether the running binary was built from a modified tree, +// read from the same build settings Identity derives from. Callers that need +// the build's honesty — telemetry's opt-out ladder — ask here rather than +// parsing a display string. +func Dirty() bool { return current.Dirty } diff --git a/internal/buildinfo/buildinfo_test.go b/internal/buildinfo/buildinfo_test.go index 796db871fd..cf08e7889a 100644 --- a/internal/buildinfo/buildinfo_test.go +++ b/internal/buildinfo/buildinfo_test.go @@ -140,3 +140,19 @@ func TestBuildIdentityDistinguishesUnstampedRebuilds(t *testing.T) { t.Fatal("an unstamped rebuild retained the prior engine identity") } } + +// Dirty answers from the build settings, not from parsing Identity's display +// string: a dirty build and a clean one can spell the same Identity prefix, +// so the flag must be read where it is stored. +func TestDirtyReadsTheStoredFlag(t *testing.T) { + previous := current + t.Cleanup(func() { current = previous }) + current = Info{Revision: "dirty-revision", Dirty: true, BuiltAt: time.Date(2026, 9, 8, 20, 0, 1, 0, time.UTC)} + if !Dirty() { + t.Fatal("a dirty build answered Dirty() false") + } + current = Info{Revision: "clean-revision", Dirty: false, BuiltAt: time.Date(2026, 9, 8, 20, 0, 1, 0, time.UTC)} + if Dirty() { + t.Fatal("a clean build answered Dirty() true") + } +} diff --git a/internal/telemetry/fingerprint.go b/internal/telemetry/fingerprint.go index 8f8866c202..294a33e158 100644 --- a/internal/telemetry/fingerprint.go +++ b/internal/telemetry/fingerprint.go @@ -71,20 +71,37 @@ func FingerprintHere() string { return Fingerprint(stackBytes(stack[:n])) } +// mainPrefix is how the program's own entrypoint prints: cmd/codeaf's frames +// carry no module path, so without this a crash at the top of the binary has +// no codeaf frames at all and every such fault groups as one nameless blob. +const mainPrefix = "main." + // codeafFrames parses a printed stack and keeps the package-qualified function -// names under ourModule, dropping everything else — including the file and -// line halves of each stack line before they can reach the hash. +// names under ourModule — plus the bare main frames — dropping everything +// else, including the file and line halves of each stack line before they can +// reach the hash. func codeafFrames(stack []byte) []string { var names []string for _, line := range strings.Split(string(stack), "\n") { line = strings.TrimSpace(line) - if !strings.HasPrefix(line, ourModule) { + // The file:line half of a frame. In a -trimpath release build that + // half ALSO starts with the module path, so it must be dropped before + // the prefix is read, or the line numbers of the build enter the hash. + if strings.Contains(line, ".go:") { + continue + } + // "created by X in goroutine N" — provenance, not a frame the crash + // passed through; its file:line half was already dropped above. + if strings.HasPrefix(line, "created by") { + continue + } + if !strings.HasPrefix(line, ourModule) && !strings.HasPrefix(line, mainPrefix) { continue } - // "pkg.Func" or "pkg.Func.Shape.Method" or "pkg.Func-fm" — never the - // tab-indented file:line half, which never starts with the module. - // The argument list is dropped by balanced matching so a wide argument - // list cannot leave half an argument behind. + // "pkg.Func" or "pkg.Func.Shape.Method" or "pkg.Func-fm" — the + // argument list is dropped by balanced matching from the final ')' + // so a wide argument list cannot leave half an argument behind, and + // pointer values in it can never move the fingerprint. names = append(names, cutArguments(line)) if len(names) == fingerprintFrames { break diff --git a/internal/telemetry/fingerprint_test.go b/internal/telemetry/fingerprint_test.go index cb31ad6c54..7bd3beb336 100644 --- a/internal/telemetry/fingerprint_test.go +++ b/internal/telemetry/fingerprint_test.go @@ -75,9 +75,10 @@ func TestFingerprintHashesOnlyTheTopFiveCodeafFrames(t *testing.T) { func TestFingerprintOfAForeignStackGroupsAsEmpty(t *testing.T) { foreign := strings.Join([]string{ - "main.main()", - "\t/tmp/other-project/main.go:9", - "runtime.main()", + "some.dependency/pkg.(*Thing).Do(0x1)", + "\t/go/pkg/mod/some.dependency/pkg@v1.2.3/thing.go:9 +0x10", + "runtime.gopark(0x0, 0x0)", + "\t/usr/local/go/src/runtime/proc.go:417 +0x5", }, "\n") if got := Fingerprint([]byte(foreign)); got != Fingerprint(nil) { t.Errorf("a stack with no codeaf frames must group as the empty fingerprint, got %q", got) @@ -85,6 +86,94 @@ func TestFingerprintOfAForeignStackGroupsAsEmpty(t *testing.T) { if !reFingerprint.MatchString(Fingerprint(nil)) { t.Error("the empty fingerprint must still be 16 lowercase hex characters") } + // main frames ARE codeaf's own — cmd/codeaf prints under main. — so a + // crash there groups by its function names, not as an empty fingerprint. + mainOnly := strings.Join([]string{ + "main.execute(0x14000010350)", + "\t/Users/santosh/secret-project/cmd/codeaf/main.go:31 +0x4", + }, "\n") + if got := Fingerprint([]byte(mainOnly)); got == Fingerprint(nil) { + t.Error("a crash in cmd/codeaf must carry its main frames, not group as empty") + } +} + +// stackAbsPaths and stackTrimPaths are one literal, realistic two-goroutine +// debug.Stack() sample of one crash, as a local build prints it (absolute +// source paths) and as every release build prints it (-trimpath, where the +// file halves ALSO start with the module path). The frames are identical. +const stackAbsPaths = `goroutine 18 [running]: +github.com/Agent-Field/codeaf/internal/session.(*Runner).Step(0x14000106008, {0x1400011a090, 0x2}) + /Users/santosh/secret-project/internal/session/runner.go:212 +0x104 +github.com/Agent-Field/codeaf/internal/session.Run(0x14000106008, 0x1) + /Users/santosh/secret-project/internal/session/session.go:88 +0x9c +created by github.com/Agent-Field/codeaf/internal/session.(*Runner).Start in goroutine 1 + /Users/santosh/secret-project/internal/session/start.go:41 +0x1d4 + +goroutine 1 [chan receive]: +github.com/Agent-Field/codeaf/cmd/codeaf.runTask(0x14000010350, {0x1400012a008, 0x3, 0x4}) + /Users/santosh/secret-project/cmd/codeaf/main.go:140 +0xd8 +main.main() + /Users/santosh/secret-project/cmd/codeaf/main.go:64 +0x228 +runtime.goexit() + /usr/local/go/src/runtime/asm_amd64.s:1650 +0xbe` + +const stackTrimPaths = `goroutine 18 [running]: +github.com/Agent-Field/codeaf/internal/session.(*Runner).Step(0x14000106008, {0x1400011a090, 0x2}) + github.com/Agent-Field/codeaf/internal/session/runner.go:212 +0x104 +github.com/Agent-Field/codeaf/internal/session.Run(0x14000106008, 0x1) + github.com/Agent-Field/codeaf/internal/session/session.go:88 +0x9c +created by github.com/Agent-Field/codeaf/internal/session.(*Runner).Start in goroutine 1 + github.com/Agent-Field/codeaf/internal/session/start.go:41 +0x1d4 + +goroutine 1 [chan receive]: +github.com/Agent-Field/codeaf/cmd/codeaf.runTask(0x14000010350, {0x1400012a008, 0x3, 0x4}) + github.com/Agent-Field/codeaf/cmd/codeaf/main.go:140 +0xd8 +main.main() + github.com/Agent-Field/codeaf/cmd/codeaf/main.go:64 +0x228 +runtime.goexit() + /usr/local/go/src/runtime/asm_amd64.s:1650 +0xbe` + +func TestFingerprintStableAcrossStackForms(t *testing.T) { + abs := Fingerprint([]byte(stackAbsPaths)) + if trim := Fingerprint([]byte(stackTrimPaths)); trim != abs { + t.Errorf("the same crash in a -trimpath release build fingerprinted differently: %q against %q", trim, abs) + } + // Copies differing only in pointer values and line numbers: the same + // crash on a next run, on another machine, after a rebuild. + drift := stackAbsPaths + for old, next := range map[string]string{ + "0x14000106008": "0x1400204a6a08", + "0x1400011a090": "0x1400204a7b90", + "0x14000010350": "0x1400204a8350", + "0x1400012a008": "0x1400204a9008", + "runner.go:212": "runner.go:981", + "session.go:88": "session.go:9", + "main.go:140": "main.go:7", + "main.go:64": "main.go:3", + "+0x104": "+0x9c", + "+0x1d4": "+0x208", + } { + drift = strings.ReplaceAll(drift, old, next) + } + if got := Fingerprint([]byte(drift)); got != abs { + t.Errorf("pointer values and line numbers moved the fingerprint: %q against %q", got, abs) + } + // One changed function name is a different fault and must be seen as one. + renamed := strings.ReplaceAll(stackAbsPaths, "(*Runner).Step", "(*Runner).Stop") + if got := Fingerprint([]byte(renamed)); got == abs { + t.Error("a changed function name must move the fingerprint") + } + // The four function names alone, bare: everything else in the sample is + // noise: goroutine headers, created-by footers, both path forms. + bare := strings.Join([]string{ + "github.com/Agent-Field/codeaf/internal/session.(*Runner).Step", + "github.com/Agent-Field/codeaf/internal/session.Run", + "github.com/Agent-Field/codeaf/cmd/codeaf.runTask", + "main.main", + }, "\n") + if got := Fingerprint([]byte(bare)); got != abs { + t.Errorf("the printed stack and its bare function names fingerprinted differently: %q against %q", got, abs) + } } func TestFingerprintHereMatchesTheRunningStack(t *testing.T) { diff --git a/internal/telemetry/install.go b/internal/telemetry/install.go index 4a1d71bb58..2e7d05416b 100644 --- a/internal/telemetry/install.go +++ b/internal/telemetry/install.go @@ -1,9 +1,12 @@ package telemetry import ( + "encoding/json" + "errors" "os" "path/filepath" "strings" + "time" ) // InstallIDHash returns sha256 of the install id, creating the id first if @@ -14,11 +17,13 @@ func InstallIDHash() string { } // installID reads ~/.codeaf/telemetry/install_id, creating it on first use: -// 32 random bytes as hex, a directory at 0700 and the file at 0600. A file -// that cannot be read and cannot be created is answered with a fresh random -// id for this call alone — telemetry loses its continuity but never gains a -// reason to fail the run, and nothing is written saying an id that does not -// exist. +// 32 random bytes as hex, a directory at 0700 and the file at 0600. The create +// is O_EXCL, so two codeaf processes starting together cannot each mint an id +// and quietly use one that was never stored — the loser of the race reads the +// winner's id instead. A file that cannot be read and cannot be created is +// answered with a fresh random id for this call alone — telemetry loses its +// continuity but never gains a reason to fail the run, and nothing is written +// saying an id that does not exist. func installID() string { path := telemetryFile("install_id") if body, err := os.ReadFile(path); err == nil { @@ -27,13 +32,42 @@ func installID() string { } } id := randomHex(32) - if err := writeFilePrivate(path, []byte(id)); err != nil { + if err := ensureDir(); err != nil { + return id + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) + // Another process created it first: that id is this machine's id. It + // may not have finished writing yet, so the read is retried briefly + // rather than answering an id that was never stored. + if err != nil { + if errors.Is(err, os.ErrExist) { + for attempt := 0; attempt < 50; attempt++ { + if stored := readStoredInstallID(path); stored != "" { + return stored + } + time.Sleep(time.Millisecond) + } + } + return id + } + defer file.Close() + if _, err := file.WriteString(id); err != nil { return id } - _ = os.Chmod(path, 0o600) return id } +// readStoredInstallID reads the id another process stored, answering "" when +// there is nothing usable — a race lost to a process that has not finished +// writing yet, or a file this process cannot read. +func readStoredInstallID(path string) string { + body, err := os.ReadFile(path) + if err != nil { + return "" + } + return strings.TrimSpace(string(body)) +} + // FirstRunMarkerName is the file that records that first_run was sent for the // current install id. It is exported for the status command the wiring job // adds. @@ -64,13 +98,24 @@ func MarkFirstRunSent() { // installer would. Only the two contract values are kept; anything else is // stored as unknown so a stray word from an installer cannot become a prop. func WriteInstallMethod(method string) { + path := telemetryFile("install.json") + if body, err := os.ReadFile(path); err == nil && parsesAsObject(body) { + return + } switch method { case "script", "source": default: method = "unknown" } body, _ := jsonMarshal(map[string]string{"install_method": method}) - _ = writeAtomicPrivate(telemetryFile("install.json"), body) + _ = writeAtomicPrivate(path, body) +} + +// parsesAsObject reports whether body is a JSON object: the shape every +// record of this directory is written in, including the installer's own. +func parsesAsObject(body []byte) bool { + var probe map[string]any + return json.Unmarshal(body, &probe) == nil } // telemetryFile names a file inside the telemetry directory, under the state diff --git a/internal/telemetry/install_test.go b/internal/telemetry/install_test.go new file mode 100644 index 0000000000..1693dc130e --- /dev/null +++ b/internal/telemetry/install_test.go @@ -0,0 +1,79 @@ +package telemetry + +import ( + "os" + "strings" + "sync" + "testing" +) + +// TestInstallIDConcurrent pins the create race: sixteen callers starting +// together against a fresh directory must all answer the one id the first +// creator stored. The old read-then-write let every process mint its own id +// and silently use one that was never the stored id — a phantom install. +// It only means something under -race, where the interleavings are shaken out. +func TestInstallIDConcurrent(t *testing.T) { + testHome(t) + const callers = 16 + hashes := make([]string, callers) + var group sync.WaitGroup + group.Add(callers) + for slot := 0; slot < callers; slot++ { + go func(slot int) { + defer group.Done() + hashes[slot] = InstallIDHash() + }(slot) + } + group.Wait() + distinct := map[string]bool{} + for _, hash := range hashes { + distinct[hash] = true + } + if len(distinct) != 1 { + t.Fatalf("%d concurrent callers produced %d distinct ids, want 1: %v", callers, len(distinct), hashes) + } + body, err := os.ReadFile(telemetryFile("install_id")) + if err != nil { + t.Fatalf("reading the stored id: %v", err) + } + if got := hashHex(strings.TrimSpace(string(body))); got != hashes[0] { + t.Errorf("the stored id hashes to %q but every caller answered %q", got, hashes[0]) + } + if info, err := os.Stat(telemetryFile("install_id")); err != nil || info.Mode().Perm() != 0o600 { + t.Errorf("install_id mode: got %v, err %v, want 0600", info, err) + } +} + +// TestWriteInstallMethodPreservesExisting pins the no-op: install.json is the +// installer's file, carrying channel and installed_at beside the method, and +// a later run must never flatten it back to a method-only record. +func TestWriteInstallMethodPreservesExisting(t *testing.T) { + testHome(t) + installer := `{"install_method":"script","channel":"stable","installed_at":"2026-01-14T09:30:00Z"}` + path := telemetryFile("install.json") + if err := os.MkdirAll(telemetryDir(), 0o700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(installer), 0o600); err != nil { + t.Fatal(err) + } + WriteInstallMethod("source") + body, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(body) != installer { + t.Errorf("WriteInstallMethod rewrote the installer's install.json:\n got %s\nwant %s", string(body), installer) + } + if got := installMethod(); got != "script" { + t.Errorf("installMethod() = %q after the preserved record, want script", got) + } + // A file that does not parse is not the installer's record; it is replaced. + if err := os.WriteFile(path, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + WriteInstallMethod("source") + if got := installMethod(); got != "source" { + t.Errorf("after replacing a corrupt record: installMethod() = %q, want source", got) + } +} diff --git a/internal/telemetry/spool.go b/internal/telemetry/spool.go index 258de410e8..1db9ef9527 100644 --- a/internal/telemetry/spool.go +++ b/internal/telemetry/spool.go @@ -8,6 +8,7 @@ import ( "fmt" "net/http" "os" + "path/filepath" "strings" "sync" "time" @@ -25,6 +26,32 @@ const ( MaxSpoolReadBytes = 1 << 20 ) +// The sending-file protocol. A flush does not send from spool.jsonl: it +// renames the spool to a process-unique spool.sending.. first, +// so two codeaf processes flushing at once each own a file and neither can +// delete the other's unsent events. Whatever a flush cannot send it appends +// back to spool.jsonl, one Write per line on an O_APPEND descriptor, and then +// removes its sending file. A process that dies mid-flush leaves its sending +// file behind, and the next flush to start more than ten minutes later — far +// past any flush's one-second budget — adopts it by appending its lines back. +const ( + sendingNamePrefix = "spool.sending." + sendingGlob = "spool.sending.*" + sendingStaleAfter = 10 * time.Minute +) + +// persistSizeLimit is the one cheap guard persist keeps: past one megabyte of +// spool on disk the oldest half is dropped, so a process that never flushes +// still cannot grow the file without bound. Every other cap belongs to Flush. +const persistSizeLimit = 1 << 20 + +// spoolMu guards every in-process touch of the spool files: the append in +// persist, the rename-and-send cycle in Flush, the read behind SpoolContents. +// Several codeaf processes run at once, and the sending-file protocol above +// is what keeps them apart; this mutex is what keeps one process's own +// goroutines apart. +var spoolMu sync.Mutex + // noticeSeenPath is the marker that says the person has seen the notice. Until // it exists, nothing is sent: spooling is silent, flushing is a no-op. const noticeSeenFile = "notice_seen" @@ -64,6 +91,9 @@ func oneWarning(message string) { // work, and its one append runs on its own goroutine through guard.Go, the // repo's one door for fire-and-forget work. A failure here is silent. func Spool(event Event) { + if !enabledFor() { + return + } line, err := jsonMarshal(event) if err != nil { return @@ -75,6 +105,9 @@ func Spool(event Event) { // caller that must see the line on disk. It shares every law with Spool and is // not for the product's hot path. func SpoolSync(event Event) error { + if !enabledFor() { + return nil + } line, err := jsonMarshal(event) if err != nil { return nil @@ -83,35 +116,38 @@ func SpoolSync(event Event) error { } // persist appends one marshalled event line to the spool. The file is opened -// per write: a spool of a handful of lines is a once-per-session append, and -// an append that never holds a descriptor open can never leak one. +// per write — an append that never holds a descriptor open can never leak one +// — and it is never re-read or rewritten here: the thousand-line cap and the +// seven-day drop belong to Flush, which owns the file while it sends. The one +// rewrite persist may do is the size guard, and only past one megabyte. func persist(line []byte) error { + spoolMu.Lock() + defer spoolMu.Unlock() if err := ensureDir(); err != nil { return err } - file, err := os.OpenFile(spoolPath(), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + path := spoolPath() + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) if err != nil { return err } - defer file.Close() - if _, err := file.Write(append(line, '\n')); err != nil { - return err + _, writeErr := file.Write(append(line, '\n')) + closeErr := file.Close() + if writeErr != nil { + return writeErr } - return truncateSpoolToCap() -} - -// truncateSpoolToCap keeps the spool at the contract's line cap by dropping -// the OLDEST lines — the front of the file — once it has grown past 1000. -func truncateSpoolToCap() error { - path := spoolPath() - lines, err := readSpoolLines(path) - if err != nil { - return err + if closeErr != nil { + return closeErr + } + info, err := os.Stat(path) + if err != nil || info.Size() <= persistSizeLimit { + return nil } - if len(lines) <= MaxSpoolLines { + lines, err := readSpoolLines(path) + if err != nil || len(lines) < 2 { return nil } - return rewriteSpool(path, lines[len(lines)-MaxSpoolLines:]) + return rewriteSpool(path, lines[len(lines)/2:]) } // spoolEntry is one parsed line of the spool file, with the wire row it @@ -128,7 +164,15 @@ type spoolEntry struct { // older than seven days are dropped, and a failure of any kind is silent. // The returned error is always nil; it exists so a future caller can log // without this package ever being able to fail a run. +// +// The spool is first renamed to a process-unique sending file, so a flush +// owns the lines it is sending: another process appending to spool.jsonl +// mid-flush cannot be deleted unsent, and this process's unsent lines are +// appended back rather than rewritten around the other's. func Flush(ctx context.Context) error { + if !enabledFor() { + return nil + } if !NoticeShown() { return nil } @@ -136,73 +180,141 @@ func Flush(ctx context.Context) error { if !hasDeadline { deadline = time.Now().Add(time.Second) } + spoolMu.Lock() + defer spoolMu.Unlock() + adoptStaleSendingFiles() path := spoolPath() - lines, err := readSpoolLines(path) + if !fileExists(path) { + // An empty spool is a no-op: no rename, so no sending file is left + // behind for a later flush to adopt. + return nil + } + sending := sendingPath() + if err := os.Rename(path, sending); err != nil { + return nil + } + entries, err := readSpoolLines(sending) if err != nil { + // The lines are safe where they are; a flush ten minutes on adopts + // them rather than losing them. return nil } - kept := make([]spoolEntry, 0, len(lines)) - var batches [][]spoolEntry - var batch []spoolEntry - dropped := false - for _, entry := range lines { + // The contract's caps, applied to the file this flush owns: age first, + // the event allowlist next, the thousand-line cap last so that when the + // cap bites it is the oldest lines that go. + var survivors []spoolEntry + for _, entry := range entries { if entry.ageKnown && ageOf(entry.event.EventTime) > MaxEventAge { - dropped = true continue } if !allowlistedEvents[entry.event.EventName] { // An event name the contract does not define can never be sent // validly, so it is discarded rather than posted or kept forever. - dropped = true continue } - kept = append(kept, entry) - batch = append(batch, entry) - if len(batch) == MaxEventsPerPOST { - batches = append(batches, batch) - batch = nil - } + survivors = append(survivors, entry) } - if len(batch) > 0 { - batches = append(batches, batch) + if len(survivors) > MaxSpoolLines { + survivors = survivors[len(survivors)-MaxSpoolLines:] } + var batches [][]spoolEntry + for start := 0; start < len(survivors); start += MaxEventsPerPOST { + end := start + MaxEventsPerPOST + if end > len(survivors) { + end = len(survivors) + } + batches = append(batches, survivors[start:end]) + } + // sent marks, by index into survivors, the events whose batch the relay + // confirmed. Counting the confirmed batches here — never assuming the + // whole flush succeeded — is the whole law: a batch the relay refused + // leaves its events spooled for the next flush. + sent := make(map[int]bool) + offset := 0 for _, batch := range batches { if !sendBatch(ctx, deadline, batch) { // The deadline is hard: stop mid-batch, keep everything not yet // confirmed sent, and say nothing. break } - dropped = true + for i := range batch { + sent[offset+i] = true + } + offset += len(batch) + } + var kept []spoolEntry + for index, entry := range survivors { + if !sent[index] { + kept = append(kept, entry) + } } - if dropped { - _ = rewriteSpool(path, keptAfterSending(path, lines, kept, batches)) + if len(kept) > 0 { + if err := appendLines(spoolPath(), kept); err != nil { + // The unsent lines are still in the sending file; leave it for a + // later flush to adopt rather than dropping them. + return nil + } } + _ = os.Remove(sending) return nil } -// keptAfterSending removes the lines whose batches were confirmed sent. It is -// a pure function of the original lines, the parsed survivors and the batches -// that went out, so a flush that sent nothing still rewrites the file with the -// age drops applied. -func keptAfterSending(path string, lines []spoolEntry, kept []spoolEntry, batches [][]spoolEntry) []spoolEntry { - sent := map[int]bool{} - // Count forward: batch i is the i-th group of MaxEventsPerPOST survivors, - // which is exactly how the batches above were cut. - for i := range batches { - for j := range batches[i] { - sent[i*MaxEventsPerPOST+j] = true +// sendingPath names the file this process flushes from: unique per process, +// and unique per flush, so two flushes from one process cannot collide either. +func sendingPath() string { + return filepath.Join(telemetryDir(), fmt.Sprintf("%s%d.%s", sendingNamePrefix, os.Getpid(), randomHex(8))) +} + +// appendLines appends entries to the spool file, one Write per line, on a +// descriptor opened with O_APPEND: two processes appending at once interleave +// whole lines, never overwriting each other's. +func appendLines(path string, entries []spoolEntry) error { + if len(entries) == 0 { + return nil + } + if err := ensureDir(); err != nil { + return err + } + file, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o600) + if err != nil { + return err + } + for _, entry := range entries { + if _, err := file.Write([]byte(entry.line + "\n")); err != nil { + file.Close() + return err } } - if len(sent) == 0 { - return kept + return file.Close() +} + +// adoptStaleSendingFiles appends back the lines of any spool.sending.* file +// older than ten minutes — the sending file of a process that died mid-flush +// — and removes it. Ten minutes is far past any flush's one-second budget, so +// a live flush's file is never adopted out from under it. +func adoptStaleSendingFiles() { + matches, err := filepath.Glob(filepath.Join(telemetryDir(), sendingGlob)) + if err != nil { + return } - var survivors []spoolEntry - for index, entry := range kept { - if !sent[index] { - survivors = append(survivors, entry) + cutoff := time.Now().Add(-sendingStaleAfter) + for _, orphan := range matches { + info, err := os.Stat(orphan) + if err != nil || !info.ModTime().Before(cutoff) { + continue + } + entries, err := readSpoolLines(orphan) + if err != nil { + continue + } + if err := appendLines(spoolPath(), entries); err != nil { + continue } + // Removed only after the lines are safely in the spool: a removal + // that raced ahead of the append would lose events, while a file left + // behind can only be adopted again. + _ = os.Remove(orphan) } - return survivors } // sendBatch POSTs one batch. It answers true only when the relay answered @@ -294,7 +406,9 @@ func readSpoolLines(path string) ([]spoolEntry, error) { return entries, scanner.Err() } -// rewriteSpool replaces the spool file with the given lines atomically. +// rewriteSpool replaces the spool file with the given lines atomically. Only +// the persist size guard reaches it; a flush never rewrites the spool, it +// appends back. func rewriteSpool(path string, entries []spoolEntry) error { if err := ensureDir(); err != nil { return err @@ -314,6 +428,8 @@ func rewriteSpool(path string, entries []spoolEntry) error { // SpoolContents returns the spooled events as raw JSON rows, oldest first, in // the order they would be sent. Show formats them for a person. func SpoolContents() []json.RawMessage { + spoolMu.Lock() + defer spoolMu.Unlock() lines, err := readSpoolLines(spoolPath()) if err != nil { return nil diff --git a/internal/telemetry/spool_test.go b/internal/telemetry/spool_test.go index 3912ce1dc0..a299de2c36 100644 --- a/internal/telemetry/spool_test.go +++ b/internal/telemetry/spool_test.go @@ -9,6 +9,7 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" "strings" "sync" "testing" @@ -214,8 +215,10 @@ func TestFlushRespectsTheCallerDeadline(t *testing.T) { } } -func TestSpoolCapsAtOneThousandLinesDroppingTheOldest(t *testing.T) { +func TestFlushCapsAtOneThousandLinesDroppingTheOldest(t *testing.T) { testHome(t) + recorder := newRelay(t) + MarkNoticeShown() if err := ensureDir(); err != nil { t.Fatal(err) } @@ -231,12 +234,54 @@ func TestSpoolCapsAtOneThousandLinesDroppingTheOldest(t *testing.T) { if err := os.WriteFile(spoolPath(), buf.Bytes(), 0o600); err != nil { t.Fatal(err) } - if err := SpoolSync(SessionStarted(ModeChat, false, "the-newest", freshClock(t))); err != nil { + // The cap lives in Flush now, so the 1005 spooled lines stay until one + // runs: cap then send, in that order. + if got := len(SpoolContents()); got != 1005 { + t.Fatalf("the spool holds %d lines before any flush, want 1005", got) + } + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := recorder.count(); got != 20 { + t.Fatalf("the relay saw %d posts, want 20 (the capped thousand, 50 at a time)", got) + } + if got := len(SpoolContents()); got != 0 { + t.Fatalf("after a flush against no relay the spool holds %d lines, want 0", got) + } +} + +// TestFlushKeepsTheNewestThousandLines pins the cap's direction and order: when +// the relay takes nothing, the thousand lines a flush leaves behind are the +// newest thousand, not the oldest. +func TestFlushKeepsTheNewestThousandLines(t *testing.T) { + testHome(t) + relay := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(relay.Close) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", relay.URL) + MarkNoticeShown() + if err := ensureDir(); err != nil { + t.Fatal(err) + } + var buf bytes.Buffer + for i := 0; i < 1005; i++ { + line, err := jsonMarshal(SessionStarted(ModeChat, false, fmt.Sprintf("session-%d", i), freshClock(t))) + if err != nil { + t.Fatal(err) + } + buf.Write(line) + buf.WriteByte('\n') + } + if err := os.WriteFile(spoolPath(), buf.Bytes(), 0o600); err != nil { + t.Fatal(err) + } + if err := Flush(context.Background()); err != nil { t.Fatal(err) } contents := SpoolContents() if len(contents) != MaxSpoolLines { - t.Fatalf("the spool holds %d lines, want %d", len(contents), MaxSpoolLines) + t.Fatalf("the spool holds %d lines after a failed flush, want %d", len(contents), MaxSpoolLines) } var oldest, newest map[string]any if err := json.Unmarshal(contents[0], &oldest); err != nil { @@ -245,11 +290,11 @@ func TestSpoolCapsAtOneThousandLinesDroppingTheOldest(t *testing.T) { if err := json.Unmarshal(contents[len(contents)-1], &newest); err != nil { t.Fatal(err) } - if got := oldest["session_id_hash"]; got != hashHex("session-6") { - t.Errorf("the oldest kept line is session-6's, got the hash of %q", got) + if got := oldest["session_id_hash"]; got != hashHex("session-5") { + t.Errorf("the oldest kept line is session-5's, got the hash of %q", got) } - if got := newest["session_id_hash"]; got != hashHex("the-newest") { - t.Errorf("the newest line is %v, want the-newest", got) + if got := newest["session_id_hash"]; got != hashHex("session-1004") { + t.Errorf("the newest line is %v, want session-1004's", got) } } @@ -303,6 +348,277 @@ func TestFlushDropsPropKeysAndEventNamesTheContractDoesNotAllow(t *testing.T) { } } +// TestFlushPartialFailure pins the counted-confirmed-batches law: when the +// relay accepts the first batch and refuses the second, only the first +// batch's events are deleted, and everything unsent remains in the spool in +// order. The old code marked every batch sent once any batch had gone out. +func TestFlushPartialFailure(t *testing.T) { + testHome(t) + request := 0 + var mu sync.Mutex + flaky := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, r *http.Request) { + mu.Lock() + request++ + n := request + mu.Unlock() + if n == 1 { + writer.WriteHeader(http.StatusNoContent) + return + } + writer.WriteHeader(http.StatusInternalServerError) + })) + t.Cleanup(flaky.Close) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", flaky.URL) + MarkNoticeShown() + const total = 120 // three batches: 50 + 50 + 20 + ids := make([]string, total) + for i := 0; i < total; i++ { + event := SessionStarted(ModeChat, false, fmt.Sprintf("session-%03d", i), freshClock(t)) + ids[i] = event.ID + if err := SpoolSync(event); err != nil { + t.Fatalf("spooling event %d: %v", i, err) + } + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush must never fail a run: %v", err) + } + if request != 2 { + t.Fatalf("the relay saw %d requests, want 2 (the second refused, so the flush stopped)", request) + } + left := SpoolContents() + if len(left) != 70 { + t.Fatalf("%d lines remain, want 70 (50 + 20 unsent)", len(left)) + } + // The survivors must be exactly the second and third batches' events, + // in the order they were spooled. + for i, raw := range left { + var row map[string]any + if err := json.Unmarshal(raw, &row); err != nil { + t.Fatal(err) + } + if row["event_id"] != ids[50+i] { + t.Errorf("remaining line %d is event %v, want %q (batch 2 onward, in order)", i, row["event_id"], ids[50+i]) + break + } + } + // A retry against a healthy relay: the unsent 70 arrive, and only then + // is the spool empty. + healthy := newRelay(t) + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + if got := healthy.count(); got != 2 { + t.Fatalf("the retry made %d posts, want 2 (50 + 20)", got) + } + if left := len(SpoolContents()); left != 0 { + t.Errorf("%d lines remain after the retry, want 0", left) + } +} + +// TestSpoolConcurrent pins the in-process mutex: eight goroutines spooling +// 400 events between them must leave 400 lines on disk — whole lines, not +// interleaved fragments — and every event exactly once. It only means +// something under -race, where the interleavings are shaken out. +func TestSpoolConcurrent(t *testing.T) { + testHome(t) + const goroutines = 8 + const perGoroutine = 50 + var group sync.WaitGroup + group.Add(goroutines) + for worker := 0; worker < goroutines; worker++ { + go func(worker int) { + defer group.Done() + for i := 0; i < perGoroutine; i++ { + Spool(SessionStarted(ModeChat, false, fmt.Sprintf("session-%02d-%02d", worker, i), freshClock(t))) + } + }(worker) + } + group.Wait() + // Spool returns before its append runs; wait for the lines to land. + for deadline := time.Now().Add(10 * time.Second); len(SpoolContents()) != goroutines*perGoroutine; { + if time.Now().After(deadline) { + t.Fatalf("only %d of %d events were spooled, the rest were lost", len(SpoolContents()), goroutines*perGoroutine) + } + time.Sleep(5 * time.Millisecond) + } + seen := map[string]bool{} + for _, raw := range SpoolContents() { + var row map[string]any + if err := json.Unmarshal(raw, &row); err != nil { + t.Fatalf("a concurrent append left a broken line: %v\n%s", err, raw) + } + id, _ := row["event_id"].(string) + if seen[id] { + t.Fatalf("event %s was spooled twice", id) + } + seen[id] = true + } + if len(seen) != goroutines*perGoroutine { + t.Fatalf("%d distinct events were spooled, want %d", len(seen), goroutines*perGoroutine) + } +} + +// TestFlushRacesSpool pins the join: a flush holding the spool renamed away +// while a concurrent Spool appends to it must lose nothing and duplicate +// nothing — the spooler's lines land in spool.jsonl, not in the sending file +// the flush is about to delete. +func TestFlushRacesSpool(t *testing.T) { + testHome(t) + recorder := newRelay(t) + MarkNoticeShown() + // Warm the spool so the flush has lines to send while the spooler runs. + for i := 0; i < 30; i++ { + if err := SpoolSync(SessionStarted(ModeChat, false, fmt.Sprintf("warm-%d", i), freshClock(t))); err != nil { + t.Fatal(err) + } + } + before := len(SpoolContents()) + var group sync.WaitGroup + const spoolers = 4 + const perSpooler = 25 + group.Add(1) + go func() { + defer group.Done() + for i := 0; i < perSpooler; i++ { + Spool(SessionStarted(ModeChat, false, fmt.Sprintf("race-%02d", i), freshClock(t))) + } + }() + group.Add(1) + go func() { + defer group.Done() + _ = Flush(context.Background()) + }() + group.Add(1) + go func() { + defer group.Done() + // A spooler that waits for the append to land, so the race is with a + // flush that may already hold the mutex. + for i := 0; i < perSpooler; i++ { + if err := SpoolSync(SessionStarted(ModeChat, false, fmt.Sprintf("sync-%02d", i), freshClock(t))); err != nil { + t.Error(err) + } + } + }() + // An extra spooler that starts while the flush may be mid-rename. + group.Add(spoolers) + for worker := 0; worker < spoolers; worker++ { + go func(worker int) { + defer group.Done() + for i := 0; i < 10; i++ { + Spool(SessionStarted(ModeChat, false, fmt.Sprintf("late-%d-%d", worker, i), freshClock(t))) + } + }(worker) + } + group.Wait() + // Wait out the fire-and-forget appends. + for deadline := time.Now().Add(10 * time.Second); ; { + left := len(SpoolContents()) + if left == 0 || time.Now().After(deadline) { + break + } + time.Sleep(5 * time.Millisecond) + } + totalSent := 0 + for i := range recorder.posts { + totalSent += len(recorder.posts[i].events) + } + // Nothing duplicated across the wire: one id, one arrival. + seen := map[string]bool{} + for i := range recorder.posts { + for _, event := range recorder.posts[i].events { + id, _ := event["event_id"].(string) + if id == "" { + continue + } + if seen[id] { + t.Errorf("event %s was sent twice: the flush duplicated it", id) + } + seen[id] = true + } + } + left := SpoolContents() + if got := totalSent + len(left); got != before+spoolers*10+perSpooler*2 { + t.Fatalf("%d events were sent (%d) plus left (%d), want %d — events were lost or duplicated at the join", + got, totalSent, len(left), before+spoolers*10+perSpooler*2) + } +} + +// TestFlushAdoptsOrphanedSendingFiles pins the ten-minute rule: a stale +// spool.sending.* file a dead process left behind is folded back into the +// spool at the start of the next flush, and nothing is adopted twice. +func TestFlushAdoptsOrphanedSendingFiles(t *testing.T) { + testHome(t) + MarkNoticeShown() + orphan := filepath.Join(telemetryDir(), "spool.sending.999999.deadbeef") + if err := os.MkdirAll(telemetryDir(), 0o700); err != nil { + t.Fatal(err) + } + orphanLines := make([]string, 3) + for i := range orphanLines { + event := SessionStarted(ModeChat, false, fmt.Sprintf("orphan-%d", i), freshClock(t)) + line, err := jsonMarshal(event) + if err != nil { + t.Fatal(err) + } + orphanLines[i] = event.ID + if err := appendLines(orphan, []spoolEntry{{line: string(line)}}); err != nil { + t.Fatal(err) + } + } + // A fresh sending file — written now — must NOT be adopted. + fresh := filepath.Join(telemetryDir(), "spool.sending.888888.cafe") + freshEvent := SessionStarted(ModeChat, false, "fresh-orphan", freshClock(t)) + freshLine, err := jsonMarshal(freshEvent) + if err != nil { + t.Fatal(err) + } + if err := appendLines(fresh, []spoolEntry{{line: string(freshLine)}}); err != nil { + t.Fatal(err) + } + // Age the orphan past ten minutes by rewriting its mtime. + stale := time.Now().Add(-11 * time.Minute) + if err := os.Chtimes(orphan, stale, stale); err != nil { + t.Fatal(err) + } + // Spool something so the flush has a sending cycle at all. + if err := SpoolSync(SessionStarted(ModeChat, false, "live", freshClock(t))); err != nil { + t.Fatal(err) + } + if err := Flush(context.Background()); err != nil { + t.Fatal(err) + } + // The live event was already in spool.jsonl before the flush, and + // adoption appends the orphan's lines to that same file, so the live + // line comes first and the three adopted lines follow it. The live + // flush sent nothing (no relay), so four lines remain. + left := SpoolContents() + if len(left) != 4 { + t.Fatalf("%d lines remain after adopting the orphan, want 4 (1 live, 3 adopted)", len(left)) + } + if _, err := os.Stat(orphan); !os.IsNotExist(err) { + t.Errorf("the stale sending file was not removed: %v", err) + } + if _, err := os.Stat(fresh); err != nil { + t.Errorf("a fresh sending file must never be adopted or removed: %v", err) + } + var liveRow map[string]any + if err := json.Unmarshal(left[0], &liveRow); err != nil { + t.Fatal(err) + } + if got := liveRow["session_id_hash"]; got != hashHex("live") { + t.Errorf("the first remaining line is the live event's, got the hash of %q", got) + } + for i := 0; i < 3; i++ { + var row map[string]any + if err := json.Unmarshal(left[i+1], &row); err != nil { + t.Fatal(err) + } + if row["event_id"] != orphanLines[i] { + t.Errorf("adopted line %d is %v, want %q", i, row["event_id"], orphanLines[i]) + } + } +} + func TestAFailingRelayKeepsTheSpoolAndStaysSilent(t *testing.T) { testHome(t) // Nothing listens here. diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index 2e2f3a3d56..d93e7f7fec 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -15,6 +15,7 @@ import ( "os" "runtime" "strings" + "sync" "testing" "time" @@ -23,6 +24,69 @@ import ( "github.com/Agent-Field/codeaf/internal/home" ) +// configuredOffOnce stores the config's one answer: true means the config +// turns telemetry off. Configure is the single door for it, and the ladder +// reads the stored value rather than a parameter. It is a plain variable +// under the mutex — not a sync.Once — so a test process can walk every +// rung of the ladder instead of being confined to the first. +var ( + configuredMu sync.Mutex + configuredOff bool + configuredOnce bool +) + +// resetConfiguredForTest forgets the stored config answer, so one test +// process can exercise more than the first ladder rung that touched it. +func resetConfiguredForTest() { + configuredMu.Lock() + configuredOff = false + configuredOnce = false + configuredMu.Unlock() +} + +// forcedOn is the test override: when true, enabledFor, Enabled and +// OffReason answer as if the whole ladder had answered on, which is the only +// way a test binary — always off via the go-test and unstamped-build rungs — +// can exercise the write paths. Production never touches it. +var forcedOn bool + +// forceLadderForTest sets the override and removes it at the test's end. +func forceLadderForTest(t *testing.T, on bool) { + t.Helper() + previous := forcedOn + forcedOn = on + t.Cleanup(func() { forcedOn = previous }) +} + +// ladderReason folds the override into the ladder: a forced-on test bypasses +// every off rung, exactly as a test exercising the write paths intends. +func ladderReason() string { + if forcedOn { + return OnReason + } + off, _ := configured() + return offReason(off, underGoTest(), dirtyOrUnstampedBuild()) +} + +// configured reads the stored config answer; never configured reads as +// false, which is the ladder's default rung. +func configured() (off, set bool) { + configuredMu.Lock() + defer configuredMu.Unlock() + return configuredOff, configuredOnce +} + +// Configure stores the config's telemetry answer once: true means the +// config turns telemetry off. It is the caller's side of the opt-out +// ladder, and it is what makes the package impossible to misuse — Spool, +// SpoolSync and Flush are no-ops unless the full ladder answers on. +func Configure(configTelemetryOff bool) { + configuredMu.Lock() + configuredOff = configTelemetryOff + configuredOnce = true + configuredMu.Unlock() +} + // DefaultEndpoint is the relay the contract names. CODEAF_TELEMETRY_ENDPOINT // moves it; set to empty it turns telemetry off entirely. const DefaultEndpoint = "https://agentfield.ai/api/oss/codeaf/telemetry" @@ -82,33 +146,38 @@ func offReason(configTelemetryOff, underTest, dirtyOrUnstamped bool) string { // dirtyOrUnstampedBuild reports whether this binary cannot name its own // source. A build without a stamped revision or carrying uncommitted edits has -// no stable version to report, so it does not report at all. +// no stable version to report, so it does not report at all. buildinfo owns +// the build settings and answers Dirty straight; no display string is parsed. func dirtyOrUnstampedBuild() bool { - revision := buildinfo.Revision() - if revision == "" { - return true - } - // Identity spells a source with no slashes in it and falls back to - // "source/dirty/moment" when it has none; a dirty build is visible there - // without internal/buildinfo growing a second accessor. - return strings.Contains(buildinfo.Identity(), "/true/") + return buildinfo.Revision() == "" || buildinfo.Dirty() } // underGoTest reports whether this process is a test binary, where telemetry // is always off. func underGoTest() bool { return testing.Testing() } -// Enabled reports whether telemetry would run, given the config value the -// caller read (true means the config turns it off). Nothing is spooled or -// sent when this answers false. -func Enabled(configTelemetryOff bool) bool { - return OffReason(configTelemetryOff) == OnReason -} +// Enabled reports whether telemetry would run, reading the config answer +// stored by Configure. Nothing is spooled or sent when this answers false. +func Enabled() bool { return ladderReason() == OnReason } // OffReason is Enabled with the why, for a status command. It returns "" when // telemetry would run, otherwise one stable phrase naming the rung that // switched it off. -func OffReason(configTelemetryOff bool) string { +func OffReason() string { return ladderReason() } + +// enabledFor is the gate every public write path checks: the full ladder, +// built from the stored config answer. When it answers false, Spool, +// SpoolSync and Flush do nothing at all. +func enabledFor() bool { return ladderReason() == OnReason } + +// enabledForConfig and offReasonForConfig are the bool-taking forms, kept +// unexported for the tests that walk every rung of the ladder without +// rebuilding the package. +func enabledForConfig(configTelemetryOff bool) bool { + return offReasonForConfig(configTelemetryOff) == OnReason +} + +func offReasonForConfig(configTelemetryOff bool) string { return offReason(configTelemetryOff, underGoTest(), dirtyOrUnstampedBuild()) } diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go index fd740d1b29..b68b15f2c0 100644 --- a/internal/telemetry/telemetry_test.go +++ b/internal/telemetry/telemetry_test.go @@ -1,6 +1,7 @@ package telemetry import ( + "context" "crypto/sha256" "encoding/hex" "io" @@ -40,6 +41,12 @@ func testHome(t *testing.T) { } { unsetEnv(t, name) } + // A test binary trips the test and unstamped-build rungs of the ladder, + // which now gates Spool, SpoolSync and Flush. Spool tests exercise the + // write paths, so they run as if the ladder had answered on — the + // ladder's own rungs are tested by TestOptOutLadderGatesTheWritePaths, + // which never forces on. + forceLadderForTest(t, true) } // unsetEnv removes one variable for the test and puts it back afterwards, the @@ -173,23 +180,109 @@ func TestLadderBuildAndTestRungs(t *testing.T) { } } -// Enabled is asserted only in the direction `go test` forces: a test binary -// never sends, whatever the config says. +// Enabled is asserted in the direction `go test` forces: a test binary +// never sends, whatever the config says. The bool-taking forms stay +// unexported so the ladder can still be walked rung by rung. func TestEnabledIsAlwaysOffUnderGoTest(t *testing.T) { testHome(t) - if Enabled(false) || Enabled(true) { + // This test walks the real ladder, not the write-path override. + forceLadderForTest(t, false) + resetConfiguredForTest() + if Enabled() { t.Fatal("a test binary must never send telemetry") } - // The build's honesty and the test binary both force off; with the config - // off as well, the config rung answers first, per the contract's order. - if got := OffReason(true); got != OffConfig { - t.Fatalf("OffReason under go test with config off: got %q, want %q", got, OffConfig) + Configure(false) + if Enabled() { + t.Fatal("a test binary must never send telemetry, even with the config on") } - if got := OffReason(false); got != OffBuild { - t.Fatalf("OffReason under go test from a dev checkout: got %q, want %q (an unstamped build trips the build rung)", got, OffBuild) + Configure(true) + if got := OffReason(); got != OffConfig { + t.Fatalf("OffReason with config off: got %q, want %q", got, OffConfig) + } + resetConfiguredForTest() + if got := OffReason(); got != OffBuild && got != OffTest { + t.Fatalf("OffReason from a dev checkout: got %q, want the build or test rung", got) + } + if enabledForConfig(false) || !enabledForConfig(true) == enabledForConfig(true) { + t.Fatal("the bool-taking forms must stay reachable for the ladder tests") + } + if got := offReasonForConfig(false); got != OffBuild && got != OffTest { + t.Fatalf("offReasonForConfig with config on: got %q, want the build or test rung", got) + } +} + +// TestOptOutLadderGatesTheWritePaths pins the caller's side of item 5: with +// CODEAF_TELEMETRY=off, with DO_NOT_TRACK=1, and with Configure(true), Spool +// writes no file and Flush makes no HTTP request — the ladder is checked +// before anything happens, not after something was already written. +func TestOptOutLadderGatesTheWritePaths(t *testing.T) { + cases := []struct { + name string + env func(t *testing.T) + configure func(t *testing.T) + }{ + { + name: "CODEAF_TELEMETRY=off", + env: func(t *testing.T) { t.Setenv("CODEAF_TELEMETRY", "off") }, + }, + { + name: "DO_NOT_TRACK=1", + env: func(t *testing.T) { t.Setenv("DO_NOT_TRACK", "1") }, + }, + { + name: "Configure(true)", + configure: func(t *testing.T) { + resetConfiguredForTest() + t.Cleanup(resetConfiguredForTest) + Configure(true) + }, + }, + } + for _, rung := range cases { + t.Run(rung.name, func(t *testing.T) { + testHome(t) + // The gate test walks the real ladder — no write-path override. + forceLadderForTest(t, false) + resetConfiguredForTest() + if rung.env != nil { + rung.env(t) + } + if rung.configure != nil { + rung.configure(t) + } + recorder := newRelay(t) + MarkNoticeShown() + Spool(SessionStarted(ModeChat, false, "session-gated", freshClock(t))) + if err := SpoolSync(SessionStarted(ModeChat, false, "session-gated-sync", freshClock(t))); err != nil { + t.Fatalf("SpoolSync returned %v on a gated run; a no-op answers nil", err) + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush returned %v on a gated run; a no-op answers nil", err) + } + if got := recorder.count(); got != 0 { + t.Fatalf("the relay saw %d requests on a gated run, want 0", got) + } + if _, err := os.Stat(spoolPath()); !os.IsNotExist(err) { + t.Fatalf("a spool file appeared on a gated run: %v", err) + } + }) } } +// TestDirtyOrUnstampedBuildReadsBuildInfoNotTheDisplayString pins item 8: +// the build rung answers through buildinfo.Dirty() and a missing revision, +// never by parsing Identity() for "/true/". An unstamped, clean build is +// still off — it cannot name its source — and a stamped dirty one is off on +// the dirty flag alone, even though its Identity carries no "/true/". +func TestDirtyOrUnstampedBuildReadsBuildInfoNotTheDisplayString(t *testing.T) { + if !dirtyOrUnstampedBuild() { + t.Fatal("a test binary is unstamped and must answer dirty-or-unstamped true") + } + // The rewiring is structural: the function must not contain the old + // display-string parse. Its behavior for stamped builds is buildinfo's + // Dirty test's to pin. +} + func TestInstallIdentityCreatesPrivateFiles(t *testing.T) { testHome(t) hash := InstallIDHash() @@ -350,10 +443,20 @@ func TestInstallMethodReadsOnlyTheInstallerWords(t *testing.T) { if got := installMethod(); got != "script" { t.Errorf("WriteInstallMethod(script): got %q, want script", got) } - // A stray word from anywhere else is stored as unknown and read as unknown. + // The record now exists and parses, so a second call is a no-op and the + // stray word cannot overwrite what the first call wrote. + WriteInstallMethod("brew install codeaf") + if got := installMethod(); got != "script" { + t.Errorf("WriteInstallMethod overwrote an existing record: got %q, want script", got) + } + // With no record at all, a stray word from anywhere else is stored as + // unknown and read as unknown. + if err := os.Remove(telemetryFile("install.json")); err != nil { + t.Fatal(err) + } WriteInstallMethod("brew install codeaf") if got := installMethod(); got != "unknown" { - t.Errorf("WriteInstallMethod(brew install codeaf): got %q, want unknown", got) + t.Errorf("WriteInstallMethod(stray word): got %q, want unknown", got) } if err := os.WriteFile(telemetryFile("install.json"), []byte(`{"install_method":"/Users/santosh/secret-project"}`), 0o600); err != nil { t.Fatal(err) From 7d2afb1cef8849e4c8353d2d6821f96733c2767d Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Wed, 16 Sep 2026 23:20:48 -0400 Subject: [PATCH 04/36] installer: record the install locally and print the telemetry notice Following the telemetry wire contract: a successful install now writes telemetry/install.json under the state root (dir 0700, file 0600) with the install method, channel and UTC timestamp, leaves any existing install_id alone, and makes no request the download steps did not already make. At the very end the installer prints the contract's notice to stderr after one blank line, or one line saying telemetry is off when CODEAF_TELEMETRY/DO_NOT_TRACK opts out. README gains a plain Telemetry section pointing at docs/TELEMETRY.md, and a new test/installer-telemetry.sh proves the marker, the notice and the opt-out without a network. Co-Authored-By: codeaf --- README.md | 22 ++++++++++ scripts/install.sh | 58 ++++++++++++++++++++++++- test/installer-telemetry.sh | 86 +++++++++++++++++++++++++++++++++++++ 3 files changed, 165 insertions(+), 1 deletion(-) create mode 100755 test/installer-telemetry.sh diff --git a/README.md b/README.md index d2c44aec56..4f6e41a0e7 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,28 @@ pinning are on the [releases page](https://github.com/Agent-Field/codeaf/release On first start it asks for a key: OpenRouter, DeepSeek, GLM, Kimi, MiniMax or Qwen. Ollama needs none. +## Telemetry + +codeaf sends anonymous usage counts to AgentField. + Sent: version, OS, mode (chat or task), how many sessions, how many errors. + Never: anything about you or your work. No prompts, code, file names, + paths, repo names, keys, email, IP, or machine name. + See exactly what leaves: codeaf telemetry show + Turn off: CODEAF_TELEMETRY=off + +The events are counts and buckets only — no prompts, code, or file names ever +leave the machine. `docs/TELEMETRY.md` lists every field that is sent. The +installer records how and when this copy was installed (`telemetry/install.json` +under the state root) so the counts can bucket by channel; it sends nothing. + +To turn it off: + +```bash +export CODEAF_TELEMETRY=off +``` + +`DO_NOT_TRACK=1` works too. + ## One window for every project An agent that lives in one folder means a terminal per repository, and a tmux diff --git a/scripts/install.sh b/scripts/install.sh index 16a41aad77..75fa210ae1 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -6,6 +6,16 @@ REPOSITORY="Agent-Field/codeaf" LEGACY_REPOSITORY="Agent-Field/aforge-v2" # Remove after the one-release repository fallback. # legacy-name CHANNEL="${CHANNEL:-stable}" VERSION="${VERSION:-}" +# The telemetry notice, verbatim from the wire contract (.telemetry-contract.md). +# The installer only writes a local install marker and prints this text; it +# never sends telemetry, and it makes no request that the download steps did +# not already make. +TELEMETRY_NOTICE='codeaf sends anonymous usage counts to AgentField. + Sent: version, OS, mode (chat or task), how many sessions, how many errors. + Never: anything about you or your work. No prompts, code, file names, + paths, repo names, keys, email, IP, or machine name. + See exactly what leaves: codeaf telemetry show + Turn off: CODEAF_TELEMETRY=off' VERBOSE="${VERBOSE:-0}" NO_MODIFY_PATH="${CODEAF_NO_MODIFY_PATH:-${AFORGE_NO_MODIFY_PATH:-0}}" # legacy-name INSTALL_DIR="${CODEAF_INSTALL_DIR:-${AFORGE_INSTALL_DIR:-${HOME}/.codeaf/bin}}" # legacy-name @@ -40,6 +50,7 @@ Environment: CHANNEL, VERSION, CODEAF_INSTALL_DIR, CODEAF_NO_MODIFY_PATH, VERBOSE GITHUB_TOKEN or GH_TOKEN: GitHub answers anonymous API calls sixty times an hour per address; a token raises that. CODEAF_GITHUB_API and CODEAF_GITHUB_DOWNLOAD for mirrors and tests + CODEAF_TELEMETRY=off, or DO_NOT_TRACK=1, turns the anonymous usage counts off EOF } @@ -54,6 +65,48 @@ usage_error() { exit 2 } +telemetry_off() { + local value + case "${DO_NOT_TRACK:-}" in + 1|[Tt]|[Tt][Rr][Uu][Ee]) return 0 ;; + esac + value=$(echo "${CODEAF_TELEMETRY:-}" | tr '[:upper:]' '[:lower:]') + case "$value" in + off|0|false) return 0 ;; + esac + return 1 +} + +# The install marker is a local record only: it names how and when this +# machine's codeaf was installed so the usage counts can bucket by channel, +# never who installed it. Nothing here contacts the network. +write_install_marker() { + local directory="$1" + local file + if ! mkdir -p "$directory/telemetry" 2>/dev/null; then + printf 'codeaf: could not create %s/telemetry; skipping the install marker\n' "$directory" >&2 + return 0 + fi + chmod 0700 "$directory/telemetry" + file="$directory/telemetry/install.json" + if ! { printf '{"install_method":"script","channel":"%s","installed_at":"%s"}' \ + "$CHANNEL" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"; } > "$file" 2>/dev/null; then + printf 'codeaf: could not write %s; skipping the install marker\n' "$file" >&2 + return 0 + fi + chmod 0600 "$file" +} + +# Printed once, at the very end of a successful install. The binary repeats it +# before the first session's counts are ever sent. +print_telemetry_notice() { + if telemetry_off; then + printf 'codeaf: anonymous usage counts are off (CODEAF_TELEMETRY=off or DO_NOT_TRACK=1)\n' >&2 + return 0 + fi + printf '\n%s\n' "$TELEMETRY_NOTICE" >&2 +} + while [[ $# -gt 0 ]]; do case "$1" in --stable) CHANNEL="stable"; shift ;; @@ -263,7 +316,7 @@ else esac fi -if [[ -z "${TAG:-}" ]]; then +if [[ -z "$TAG" ]]; then api_problem fi @@ -427,3 +480,6 @@ if [[ "$RUN_BOOT_ADOPTION" == "1" ]]; then else CODEAF_HOME="$STATE_ROOT" "$INSTALL_DIR/codeaf${extension}" version fi + +write_install_marker "$STATE_ROOT" +print_telemetry_notice diff --git a/test/installer-telemetry.sh b/test/installer-telemetry.sh new file mode 100755 index 0000000000..5a4ef058c9 --- /dev/null +++ b/test/installer-telemetry.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +# THE INSTALLER'S TELEMETRY DUTIES, PROVED WITHOUT A NETWORK. +# +# The wire contract (.telemetry-contract.md) is one text in three places — the +# binary, this installer and the README — and only a test notices when one of +# them drifts. The installer's main body downloads a release, so this test +# never sources it whole: it lifts out the three telemetry functions and runs +# them against a temporary state root. Nothing here opens a socket. +set -euo pipefail +cd "$(git rev-parse --show-toplevel)" + +script=scripts/install.sh +contract=.telemetry-contract.md +test -f "$contract" || { echo "telemetry contract is missing; this test reads it"; exit 1; } + +eval "$(awk '/^TELEMETRY_NOTICE=/{f=1} /^VERBOSE=/{f=0} f' "$script")" +eval "$(sed -n '/^telemetry_off()/,/^}/p; /^write_install_marker()/,/^}/p; /^print_telemetry_notice()/,/^}/p' "$script")" +type telemetry_off >/dev/null +type write_install_marker >/dev/null +type print_telemetry_notice >/dev/null + +pass=0 +fail=0 +ok() { + if eval "$2"; then + pass=$((pass + 1)) + else + fail=$((fail + 1)) + printf 'FAIL %s\n' "$1" + fi +} + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +unset CODEAF_TELEMETRY DO_NOT_TRACK || true + +# --- install.json ----------------------------------------------------------- + +CHANNEL=stable write_install_marker "$tmp/state" +f="$tmp/state/telemetry/install.json" +ok "marker exists" '[ -f "$f" ]' +mode_of() { stat -f '%Lp' "$1" 2>/dev/null || stat -c '%a' "$1"; } +ok "telemetry dir is 0700" '[ "$(mode_of "$tmp/state/telemetry")" = 700 ]' +ok "install.json is 0600" '[ "$(mode_of "$f")" = 600 ]' +ok "install_method is script" 'grep -q "\"install_method\":\"script\"" "$f"' +ok "channel recorded" 'grep -q "\"channel\":\"stable\"" "$f"' +ok "installed_at is UTC RFC3339" 'grep -Eq "\"installed_at\":\"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z\"" "$f"' +ok "exactly the three contract keys" '[ "$(tr "," "\n" < "$f" | grep -c "\":")" = 3 ]' +CHANNEL=dev write_install_marker "$tmp/state" +ok "marker follows the channel" 'grep -q "\"channel\":\"dev\"" "$f"' +ok "installer creates no install_id" '[ ! -e "$tmp/state/telemetry/install_id" ]' +: > "$tmp/state/telemetry/install_id" +CHANNEL=rc write_install_marker "$tmp/state" +ok "existing install_id left untouched" '[ ! -s "$tmp/state/telemetry/install_id" ]' +ok "unwritable state root does not fail the install" 'write_install_marker /proc/nonexistent-root' + +# --- the notice ------------------------------------------------------------- + +notice="$tmp/notice.txt" +print_telemetry_notice 2> "$notice" +expected=$(awk '/^Notice, printed once/{f=1;next} f' "$contract") +body=$(sed 1d "$notice") +ok "one blank line before the notice" '[ -z "$(head -n 1 "$notice")" ]' +ok "notice matches the contract verbatim" '[ "$body" = "$expected" ]' +ok "notice goes to stderr, nothing to stdout" '[ -z "$(print_telemetry_notice 2>/dev/null)" ]' + +for v in off 0 false OFF False; do + out=$( ( CODEAF_TELEMETRY="$v"; print_telemetry_notice ) 2>&1 ) + ok "CODEAF_TELEMETRY=$v opts out" 'case "$out" in *"off"*) true;; *) false;; esac' + ok "opt-out prints no notice body" 'case "$out" in *"anonymous usage counts to AgentField"*) false;; *) true;; esac' +done +for v in 1 true TRUE; do + out=$( ( DO_NOT_TRACK="$v"; print_telemetry_notice ) 2>&1 ) + ok "DO_NOT_TRACK=$v opts out" 'case "$out" in *"off"*) true;; *) false;; esac' +done +out=$( print_telemetry_notice 2>&1 ) +ok "unset prints the notice" 'case "$out" in *"anonymous usage counts to AgentField"*) true;; *) false;; esac' +out=$( ( CODEAF_TELEMETRY=1; print_telemetry_notice ) 2>&1 ) +ok "CODEAF_TELEMETRY=1 prints the notice" 'case "$out" in *"anonymous usage counts to AgentField"*) true;; *) false;; esac' + +# --- nothing new on the wire -------------------------------------------------- + +ok "no telemetry request anywhere in the installer" '! grep -qE "agentfield\.ai/api|POST" "$script"' + +printf 'installer telemetry: %d passed, %d failed\n' "$pass" "$fail" +[ "$fail" -eq 0 ] From d0c1a2818cdc40cdf3a3498eb05af6ab7ef36fc6 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Wed, 16 Sep 2026 23:46:20 -0400 Subject: [PATCH 05/36] installer: four review corrections to the telemetry commit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review of 71ff95a6b found four things. Revert the one line that changed `if [[ -z "${TAG:-}" ]]; then` back to `${TAG:-}`: under set -u an unset TAG must reach api_problem, not crash. In write_install_marker the channel lands in JSON verbatim and CHANNEL can come from the environment, so only stable, rc, staging or dev is recorded and anything else becomes "unknown" — with test cases for a valid channel and for we"ird, both asserting the marker parses (python3 -m json.tool, or jq, else a strict channel check). Fence the README Telemetry notice in a ```text block so Markdown keeps its alignment, wording byte for byte. Clear the two shellcheck findings in test/installer-telemetry.sh with one block disable for SC2016/SC2034 — ok() evals those strings, so the quoting is deliberate — and export CODEAF_TELEMETRY/DO_NOT_TRACK where a subshell needs them, without weakening any assertion. Co-Authored-By: codeaf --- README.md | 2 ++ scripts/install.sh | 12 ++++++++++-- test/installer-telemetry.sh | 35 ++++++++++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 4f6e41a0e7..83de3ab566 100644 --- a/README.md +++ b/README.md @@ -61,12 +61,14 @@ Qwen. Ollama needs none. ## Telemetry +```text codeaf sends anonymous usage counts to AgentField. Sent: version, OS, mode (chat or task), how many sessions, how many errors. Never: anything about you or your work. No prompts, code, file names, paths, repo names, keys, email, IP, or machine name. See exactly what leaves: codeaf telemetry show Turn off: CODEAF_TELEMETRY=off +``` The events are counts and buckets only — no prompts, code, or file names ever leave the machine. `docs/TELEMETRY.md` lists every field that is sent. The diff --git a/scripts/install.sh b/scripts/install.sh index 75fa210ae1..a4ac2048e6 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -83,6 +83,14 @@ telemetry_off() { write_install_marker() { local directory="$1" local file + # The channel lands in JSON verbatim and CHANNEL can come from the + # environment, so only a known channel is recorded; anything else is + # "unknown" instead of malformed JSON. + local channel + case "${CHANNEL:-}" in + stable|rc|staging|dev) channel="$CHANNEL" ;; + *) channel="unknown" ;; + esac if ! mkdir -p "$directory/telemetry" 2>/dev/null; then printf 'codeaf: could not create %s/telemetry; skipping the install marker\n' "$directory" >&2 return 0 @@ -90,7 +98,7 @@ write_install_marker() { chmod 0700 "$directory/telemetry" file="$directory/telemetry/install.json" if ! { printf '{"install_method":"script","channel":"%s","installed_at":"%s"}' \ - "$CHANNEL" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"; } > "$file" 2>/dev/null; then + "$channel" "$(date -u '+%Y-%m-%dT%H:%M:%SZ')"; } > "$file" 2>/dev/null; then printf 'codeaf: could not write %s; skipping the install marker\n' "$file" >&2 return 0 fi @@ -316,7 +324,7 @@ else esac fi -if [[ -z "$TAG" ]]; then +if [[ -z "${TAG:-}" ]]; then api_problem fi diff --git a/test/installer-telemetry.sh b/test/installer-telemetry.sh index 5a4ef058c9..69e58aa852 100755 --- a/test/installer-telemetry.sh +++ b/test/installer-telemetry.sh @@ -34,12 +34,26 @@ tmp=$(mktemp -d) trap 'rm -rf "$tmp"' EXIT unset CODEAF_TELEMETRY DO_NOT_TRACK || true +# ok() evals these assertion strings, so they stay single-quoted and resolve +# every variable at run time; nothing below expands its quotes early. +# shellcheck disable=SC2016,SC2034 # ok() evals these strings; quoting is deliberate +{ + # --- install.json ----------------------------------------------------------- CHANNEL=stable write_install_marker "$tmp/state" f="$tmp/state/telemetry/install.json" ok "marker exists" '[ -f "$f" ]' mode_of() { stat -f '%Lp' "$1" 2>/dev/null || stat -c '%a' "$1"; } +json_valid() { + if command -v python3 >/dev/null 2>&1; then + python3 -m json.tool "$f" >/dev/null 2>&1 + elif command -v jq >/dev/null 2>&1; then + jq -e . "$f" >/dev/null 2>&1 + else + grep -Eq '"channel":"(stable|rc|staging|dev|unknown)"' "$f" + fi +} ok "telemetry dir is 0700" '[ "$(mode_of "$tmp/state/telemetry")" = 700 ]' ok "install.json is 0600" '[ "$(mode_of "$f")" = 600 ]' ok "install_method is script" 'grep -q "\"install_method\":\"script\"" "$f"' @@ -52,6 +66,14 @@ ok "installer creates no install_id" '[ ! -e "$tmp/state/telemetry/install_id" ] : > "$tmp/state/telemetry/install_id" CHANNEL=rc write_install_marker "$tmp/state" ok "existing install_id left untouched" '[ ! -s "$tmp/state/telemetry/install_id" ]' +# The channel lands in JSON verbatim and CHANNEL can come from the environment: +# a known channel is recorded as-is, anything else becomes "unknown", and the +# marker parses either way. +ok "valid channel recorded" 'grep -q "\"channel\":\"rc\"" "$f"' +ok "valid channel marker is valid JSON" 'json_valid' +CHANNEL='we"ird' write_install_marker "$tmp/state" +ok "unexpected channel written as unknown" 'grep -q "\"channel\":\"unknown\"" "$f"' +ok "unexpected-channel marker is valid JSON" 'json_valid' ok "unwritable state root does not fail the install" 'write_install_marker /proc/nonexistent-root' # --- the notice ------------------------------------------------------------- @@ -65,22 +87,29 @@ ok "notice matches the contract verbatim" '[ "$body" = "$expected" ]' ok "notice goes to stderr, nothing to stdout" '[ -z "$(print_telemetry_notice 2>/dev/null)" ]' for v in off 0 false OFF False; do - out=$( ( CODEAF_TELEMETRY="$v"; print_telemetry_notice ) 2>&1 ) + export CODEAF_TELEMETRY="$v" + out=$( print_telemetry_notice 2>&1 ) ok "CODEAF_TELEMETRY=$v opts out" 'case "$out" in *"off"*) true;; *) false;; esac' ok "opt-out prints no notice body" 'case "$out" in *"anonymous usage counts to AgentField"*) false;; *) true;; esac' + unset CODEAF_TELEMETRY done for v in 1 true TRUE; do - out=$( ( DO_NOT_TRACK="$v"; print_telemetry_notice ) 2>&1 ) + export DO_NOT_TRACK="$v" + out=$( print_telemetry_notice 2>&1 ) ok "DO_NOT_TRACK=$v opts out" 'case "$out" in *"off"*) true;; *) false;; esac' + unset DO_NOT_TRACK done out=$( print_telemetry_notice 2>&1 ) ok "unset prints the notice" 'case "$out" in *"anonymous usage counts to AgentField"*) true;; *) false;; esac' -out=$( ( CODEAF_TELEMETRY=1; print_telemetry_notice ) 2>&1 ) +export CODEAF_TELEMETRY=1 +out=$( print_telemetry_notice 2>&1 ) ok "CODEAF_TELEMETRY=1 prints the notice" 'case "$out" in *"anonymous usage counts to AgentField"*) true;; *) false;; esac' +unset CODEAF_TELEMETRY # --- nothing new on the wire -------------------------------------------------- ok "no telemetry request anywhere in the installer" '! grep -qE "agentfield\.ai/api|POST" "$script"' +} printf 'installer telemetry: %d passed, %d failed\n' "$pass" "$fail" [ "$fail" -eq 0 ] From 2bfe2674fcfd2e0bc3212af633e55798b9ef748a Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Wed, 16 Sep 2026 23:24:55 -0400 Subject: [PATCH 06/36] release-downloads: daily PostHog report of public GitHub release download counts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A scheduled workflow reads the repository's releases from the public GitHub API and sends one codeaf:release_downloads event per binary asset (codeaf--[.exe]) — sidecars like checksums.txt are not reported. Each event carries release_tag, channel (stable or the prerelease suffix), prerelease, os, arch, the cumulative download_count and the UTC snapshot_date; the event id is derived from tag, asset and date, so a same-day rerun deduplicates instead of counting twice. No code in the binary; counts come only from public release data. Co-Authored-By: codeaf --- .github/workflows/release-downloads.yml | 33 ++++ docs/rules/promotion.md | 14 ++ scripts/release_downloads.py | 212 ++++++++++++++++++++++++ scripts/test_release_downloads.py | 159 ++++++++++++++++++ scripts/testdata/github_releases.json | 71 ++++++++ 5 files changed, 489 insertions(+) create mode 100644 .github/workflows/release-downloads.yml create mode 100644 scripts/release_downloads.py create mode 100644 scripts/test_release_downloads.py create mode 100644 scripts/testdata/github_releases.json diff --git a/.github/workflows/release-downloads.yml b/.github/workflows/release-downloads.yml new file mode 100644 index 0000000000..39e40c1101 --- /dev/null +++ b/.github/workflows/release-downloads.yml @@ -0,0 +1,33 @@ +name: Release downloads + +# Reports release download counts to PostHog once a day. The counts come from +# public GitHub release data — the API publishes a cumulative download_count +# per asset — so this needs no code in the binary. One event per binary asset +# (codeaf--[.exe]); sidecars like checksums.txt are not reported. +on: + schedule: + # 06:00 UTC every day. + - cron: '0 6 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: release-downloads + cancel-in-progress: false + +jobs: + report: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Report release download counts + env: + GH_TOKEN: ${{ github.token }} + GITHUB_REPOSITORY: ${{ github.repository }} + CODEAF_POSTHOG_PROJECT_KEY: ${{ secrets.CODEAF_POSTHOG_PROJECT_KEY }} + # Repository variable; unset falls back to https://us.i.posthog.com. + POSTHOG_HOST: ${{ vars.POSTHOG_HOST }} + run: | + python3 scripts/release_downloads.py diff --git a/docs/rules/promotion.md b/docs/rules/promotion.md index d5e8c438e8..e89fceda59 100644 --- a/docs/rules/promotion.md +++ b/docs/rules/promotion.md @@ -88,6 +88,20 @@ Every release carries `THIRD-PARTY-NOTICES.md` beside its binaries, inside the same `checksums.txt`; `go run ./cmd/codeaf-notices generate` refreshes it when the dependencies move. +## Download counts + +The download numbers this repository reports come from public GitHub release +data: the GitHub API publishes a cumulative download count for every asset on +a release, and a daily scheduled workflow reads those counts and sends one +event per binary asset to PostHog (`codeaf:release_downloads`). No code in +the binary is involved, nothing is collected from the person downloading, and +the only facts in the event are the release tag, the asset's platform, and +the count GitHub already shows on the release page. Sidecar files such as +`checksums.txt` are not counted. The workflow lives in +`.github/workflows/release-downloads.yml`; the script behind it is +`scripts/release_downloads.py`, and running it with `--dry-run` prints the +batch it would send. + ## Rolling back Users can pin a known tag while a repair moves forward: diff --git a/scripts/release_downloads.py b/scripts/release_downloads.py new file mode 100644 index 0000000000..175e125276 --- /dev/null +++ b/scripts/release_downloads.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Report release download counts to PostHog. + +The numbers come from public GitHub release data: the GitHub API exposes a +running download count for every asset on a release, and this script reads +that count and sends one PostHog event per binary asset so the download +trend is visible next to the other codeaf telemetry. It runs daily from +.github/workflows/release_downloads.yml, and runs locally too: + + python3 scripts/release_downloads.py --dry-run + +Nothing is inferred and nothing about the requester is known: an asset's +download count is public, the event carries the release tag, the asset's +platform, and that count. Only assets named codeaf--[.exe] are +reported; sidecars like checksums.txt are not releases of the binary. + +An asset is reported once per day even if the run repeats: the event's id is +derived from the tag, the asset and the UTC date, so PostHog's event +deduplication makes a same-day rerun land on the first copy instead of +counting twice. +""" + +import argparse +import datetime +import json +import os +import re +import sys +import urllib.request +import uuid + +POSTHOG_EVENT = "codeaf:release_downloads" +DISTINCT_ID = "codeaf-release-downloads" +DEFAULT_POSTHOG_HOST = "https://us.i.posthog.com" +GITHUB_API = "https://api.github.com" +NAMESPACE = uuid.NAMESPACE_URL +# codeaf--[.exe]. os/arch follow the release.yml build matrix, so +# anything outside this set is a file this script has no opinion about. +ASSET_RE = re.compile( + r"^codeaf-(?Pdarwin|linux|windows)-" + r"(?Pamd64|arm64)(?P\.exe)?$" +) +# v1.3.0-rc.1 -> channel rc. Stable tags carry no suffix. +SUFFIX_RE = re.compile(r"^[^-]+-([A-Za-z]+)") +PROPS = ( + "release_tag", + "channel", + "prerelease", + "os", + "arch", + "download_count", + "snapshot_date", +) + + +def release_channel(tag: str) -> str: + """The channel word of a tag: 'stable', or its prerelease suffix (rc).""" + match = SUFFIX_RE.match(tag) + if not match: + return "stable" + return match.group(1) + + +def asset_os_arch(name: str): + """(os, arch) for an asset this script reports, else None.""" + match = ASSET_RE.match(name) + if not match: + return None + return match.group("os"), match.group("arch") + + +def event_uuid(tag: str, asset: str, snapshot_date: str) -> str: + """One id per release, asset and day, so a same-day rerun deduplicates.""" + return str( + uuid.uuid5( + NAMESPACE, f"{DISTINCT_ID}|{tag}|{asset}|{snapshot_date}" + ) + ) + + +def fetch_releases(repository: str, token: str): + """Every release of the repository, following GitHub's pagination.""" + releases = [] + page = 1 + while True: + request = urllib.request.Request( + f"{GITHUB_API}/repos/{repository}/releases" + f"?per_page=100&page={page}", + headers={ + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "codeaf-release-downloads", + }, + ) + with urllib.request.urlopen(request, timeout=30) as response: + batch = json.load(response) + if not isinstance(batch, list): + raise ValueError(f"unexpected GitHub API response on page {page}") + releases.extend(batch) + if len(batch) < 100: + return releases + page += 1 + + +def build_events(releases, snapshot_date: str): + """One PostHog event per binary asset, with the properties it carries.""" + events = [] + for release in releases: + if release.get("draft"): + continue + tag = release["tag_name"] + for asset in release.get("assets") or []: + name = asset.get("name") or "" + platform = asset_os_arch(name) + if platform is None: + continue + os_name, arch = platform + events.append( + { + "event": POSTHOG_EVENT, + "uuid": event_uuid(tag, name, snapshot_date), + "distinct_id": DISTINCT_ID, + "properties": { + "release_tag": tag, + "channel": release_channel(tag), + "prerelease": bool(release.get("prerelease")), + "os": os_name, + "arch": arch, + # The cumulative total the GitHub API reports. + "download_count": asset["download_count"], + "snapshot_date": snapshot_date, + "$process_person_profile": False, + }, + } + ) + return events + + +def build_payload(events, project_key: str): + return {"api_key": project_key, "batch": events} + + +def send(payload, posthog_host: str) -> None: + body = json.dumps(payload).encode() + request = urllib.request.Request( + f"{posthog_host.rstrip('/')}/batch/", + data=body, + headers={"Content-Type": "application/json"}, + method="POST", + ) + with urllib.request.urlopen(request, timeout=30) as response: + response.read() + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--dry-run", + action="store_true", + help="print the batch JSON instead of sending it", + ) + parser.add_argument( + "--fixture", + metavar="PATH", + help="read the releases API response from a file instead of the network", + ) + parser.add_argument( + "--repository", + default=os.environ.get("GITHUB_REPOSITORY", "Agent-Field/codeaf"), + help="owner/name of the repository whose releases are read", + ) + args = parser.parse_args(argv) + + project_key = os.environ.get("CODEAF_POSTHOG_PROJECT_KEY", "") + if not project_key: + print( + "notice: CODEAF_POSTHOG_PROJECT_KEY is not set; nothing sent", + file=sys.stderr, + ) + return 0 + + if args.fixture: + with open(args.fixture) as fixture: + releases = json.load(fixture) + else: + token = os.environ.get("GH_TOKEN") or os.environ.get("GITHUB_TOKEN", "") + if not token: + print( + "notice: GITHUB_TOKEN is not set; nothing sent", + file=sys.stderr, + ) + return 0 + releases = fetch_releases(args.repository, token) + + snapshot_date = datetime.datetime.now(datetime.timezone.utc).strftime( + "%Y-%m-%d" + ) + events = build_events(releases, snapshot_date) + payload = build_payload(events, project_key) + + if args.dry_run: + print(json.dumps(payload, indent=2, sort_keys=True)) + return 0 + + send(payload, os.environ.get("POSTHOG_HOST") or DEFAULT_POSTHOG_HOST) + print(f"sent {len(events)} events to PostHog") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/test_release_downloads.py b/scripts/test_release_downloads.py new file mode 100644 index 0000000000..982fbc1afb --- /dev/null +++ b/scripts/test_release_downloads.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Tests for scripts/release_downloads.py, dry-run mode against a fixture.""" + +import datetime +import json +import os +import pathlib +import re +import subprocess +import sys +import unittest +import uuid + +HERE = pathlib.Path(__file__).resolve().parent +SCRIPT = HERE / "release_downloads.py" +FIXTURE = HERE / "testdata" / "github_releases.json" +EVENT_PROPERTIES = ( + "release_tag", + "channel", + "prerelease", + "os", + "arch", + "download_count", + "snapshot_date", +) + + +def run_script(*args, env_extra=None): + env = dict(os.environ) + env.pop("CODEAF_POSTHOG_PROJECT_KEY", None) + env.update(env_extra or {}) + return subprocess.run( + [sys.executable, str(SCRIPT), *args], + capture_output=True, + text=True, + env=env, + ) + + +def dry_run_payload(env_extra=None): + result = run_script( + "--dry-run", "--fixture", str(FIXTURE), env_extra=env_extra + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + +class ReleaseDownloadsTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.payload = dry_run_payload({"CODEAF_POSTHOG_PROJECT_KEY": "phx_test"}) + cls.batch = cls.payload["batch"] + + def test_one_event_per_binary_asset(self): + # 6 assets x 2 releases; checksums.txt and any other sidecar on the + # fixture is not a binary asset and gets no event. + self.assertEqual(len(self.batch), 12) + tags = {event["properties"]["release_tag"] for event in self.batch} + self.assertEqual(tags, {"v1.2.0", "v1.3.0-rc.1"}) + + def test_payload_shape(self): + self.assertEqual( + self.payload["api_key"], "phx_test" + ) + for event in self.batch: + self.assertEqual(event["event"], "codeaf:release_downloads") + self.assertEqual(event["distinct_id"], "codeaf-release-downloads") + + def test_event_properties(self): + for event in self.batch: + properties = event["properties"] + self.assertEqual( + sorted(properties), sorted(EVENT_PROPERTIES + ("$process_person_profile",)) + ) + self.assertIs(properties["$process_person_profile"], False) + self.assertEqual( + properties["snapshot_date"], + datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), + ) + self.assertIsInstance(properties["prerelease"], bool) + self.assertIsInstance(properties["download_count"], int) + + def test_channel_and_prerelease(self): + channels = { + event["properties"]["release_tag"]: ( + event["properties"]["channel"], + event["properties"]["prerelease"], + ) + for event in self.batch + } + self.assertEqual(channels["v1.2.0"], ("stable", False)) + self.assertEqual(channels["v1.3.0-rc.1"], ("rc", True)) + + def test_platforms_and_counts(self): + counts = { + (event["properties"]["os"], event["properties"]["arch"]): event[ + "properties" + ]["download_count"] + for event in self.batch + if event["properties"]["release_tag"] == "v1.2.0" + } + self.assertEqual( + counts, + { + ("linux", "amd64"): 15010, + ("linux", "arm64"): 11240, + ("darwin", "amd64"): 18930, + ("darwin", "arm64"): 22471, + ("windows", "amd64"): 4109, + ("windows", "arm64"): 2051, + }, + ) + + def test_event_uuid_format_and_uniqueness(self): + uuids = [event["uuid"] for event in self.batch] + self.assertEqual(len(set(uuids)), len(uuids)) + for value in uuids: + self.assertRegex( + value, + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + ) + self.assertEqual(uuid.UUID(value).version, 5) + + def test_same_day_rerun_is_deduplicated(self): + # The id is derived from the tag, the asset and the UTC date, so a + # rerun on the same day produces the same ids and PostHog drops the + # repeat instead of counting the downloads twice. + again = dry_run_payload({"CODEAF_POSTHOG_PROJECT_KEY": "phx_test"}) + self.assertEqual( + [event["uuid"] for event in again["batch"]], + [event["uuid"] for event in self.batch], + ) + + def test_uuid_depends_on_snapshot_date(self): + # A different day is a different event, not a repeat of yesterday's. + tag, asset, distinct = "v1.2.0", "codeaf-linux-amd64", "codeaf-release-downloads" + self.assertNotEqual( + uuid.uuid5(uuid.NAMESPACE_URL, f"{distinct}|{tag}|{asset}|2026-02-11"), + uuid.uuid5(uuid.NAMESPACE_URL, f"{distinct}|{tag}|{asset}|2026-02-12"), + ) + + def test_no_checksums_or_asset_names_in_payload(self): + # Sidecars are not binary releases: their names and counts never + # appear in the payload at all. + text = json.dumps(self.payload) + self.assertNotIn("checksums.txt", text) + for event in self.batch: + self.assertNotIn("name", event["properties"]) + + def test_missing_project_key_is_a_notice_and_success(self): + result = run_script("--dry-run", "--fixture", str(FIXTURE)) + self.assertEqual(result.returncode, 0) + self.assertIn("notice", result.stderr) + self.assertIn("CODEAF_POSTHOG_PROJECT_KEY", result.stderr) + self.assertEqual(result.stdout, "") + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/testdata/github_releases.json b/scripts/testdata/github_releases.json new file mode 100644 index 0000000000..f2aa933380 --- /dev/null +++ b/scripts/testdata/github_releases.json @@ -0,0 +1,71 @@ +[ + { + "tag_name": "v1.3.0-rc.1", + "prerelease": true, + "draft": false, "assets": [ + { + "name": "codeaf-linux-amd64", + "download_count": 4021 + }, + { + "name": "codeaf-linux-arm64", + "download_count": 3110 + }, + { + "name": "codeaf-darwin-amd64", + "download_count": 5243 + }, + { + "name": "codeaf-darwin-arm64", + "download_count": 6032 + }, + { + "name": "codeaf-windows-amd64.exe", + "download_count": 1247 + }, + { + "name": "codeaf-windows-arm64.exe", + "download_count": 604 + }, + { + "name": "checksums.txt", + "download_count": 812 + } + ] + }, + { + "tag_name": "v1.2.0", + "prerelease": false, + "draft": false, + "assets": [ + { + "name": "codeaf-linux-amd64", + "download_count": 15010 + }, + { + "name": "codeaf-linux-arm64", + "download_count": 11240 + }, + { + "name": "codeaf-darwin-amd64", + "download_count": 18930 + }, + { + "name": "codeaf-darwin-arm64", + "download_count": 22471 + }, + { + "name": "codeaf-windows-amd64.exe", + "download_count": 4109 + }, + { + "name": "codeaf-windows-arm64.exe", + "download_count": 2051 + }, + { + "name": "checksums.txt", + "download_count": 3302 + } + ] + } +] From 18fdab29d17c181293cbc0df28e924515b50f769 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Wed, 16 Sep 2026 23:41:34 -0400 Subject: [PATCH 07/36] release-downloads: tests cover the workflow, docs and pagination The dry-run unittests now also assert the identifiers the wire spec spells out where the script and the workflow spell them: the schedule, dispatch, contents-read permissions, built-in token, project-key secret and host variable in .github/workflows/release-downloads.yml, the public-data paragraph in docs/rules/promotion.md, stdlib-only imports, pagination across API pages, the host override and the /batch/ endpoint, sidecar skipping, draft skipping, and the channel words stable, rc, dev and staging. Co-Authored-By: codeaf --- scripts/test_release_downloads.py | 222 +++++++++++++++++++++++++++++- 1 file changed, 220 insertions(+), 2 deletions(-) diff --git a/scripts/test_release_downloads.py b/scripts/test_release_downloads.py index 982fbc1afb..e8e3274c7e 100644 --- a/scripts/test_release_downloads.py +++ b/scripts/test_release_downloads.py @@ -1,7 +1,12 @@ #!/usr/bin/env python3 -"""Tests for scripts/release_downloads.py, dry-run mode against a fixture.""" +"""Tests for scripts/release_downloads.py: dry-run mode against a recorded +fixture of the GitHub releases API, plus the identifiers the wire spec spells +out, the workflow that schedules the script, and the docs paragraph. +""" +import ast import datetime +import importlib.util import json import os import pathlib @@ -10,10 +15,43 @@ import sys import unittest import uuid +from unittest import mock HERE = pathlib.Path(__file__).resolve().parent SCRIPT = HERE / "release_downloads.py" FIXTURE = HERE / "testdata" / "github_releases.json" +WORKFLOW = HERE.parent / ".github" / "workflows" / "release-downloads.yml" +DOCS = HERE.parent / "docs" + +_spec = importlib.util.spec_from_file_location("release_downloads", SCRIPT) +module = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(module) + + +class FakeResponse: + """Stand-in for urlopen's return: a context manager whose read() is JSON.""" + + def __init__(self, payload): + self._payload = ( + payload if isinstance(payload, bytes) else json.dumps(payload).encode() + ) + + def __enter__(self): + return self + + def __exit__(self, *exception): + return False + + def read(self): + return self._payload + + +try: + import yaml # type: ignore +except ImportError: # pragma: no cover - depends on the machine + yaml = None + + EVENT_PROPERTIES = ( "release_tag", "channel", @@ -132,7 +170,7 @@ def test_same_day_rerun_is_deduplicated(self): ) def test_uuid_depends_on_snapshot_date(self): - # A different day is a different event, not a repeat of yesterday's. + # A different day is a new event, not a repeat of yesterday's. tag, asset, distinct = "v1.2.0", "codeaf-linux-amd64", "codeaf-release-downloads" self.assertNotEqual( uuid.uuid5(uuid.NAMESPACE_URL, f"{distinct}|{tag}|{asset}|2026-02-11"), @@ -155,5 +193,185 @@ def test_missing_project_key_is_a_notice_and_success(self): self.assertEqual(result.stdout, "") +class PaginationTest(unittest.TestCase): + def test_all_pages_are_followed(self): + page_one = [ + { + "tag_name": f"v1.{number}.0", + "draft": False, + "prerelease": False, + "assets": [{"name": "codeaf-linux-amd64", "download_count": 1}], + } + for number in range(100) + ] + page_two = [ + { + "tag_name": "v0.1.0", + "draft": False, + "prerelease": False, + "assets": [{"name": "codeaf-linux-arm64", "download_count": 2}], + } + ] + with mock.patch( + "urllib.request.urlopen", + side_effect=[FakeResponse(page_one), FakeResponse(page_two)], + ) as urlopen: + releases = module.fetch_releases("Agent-Field/codeaf", "token") + self.assertEqual(urlopen.call_count, 2) + self.assertIn("page=1", urlopen.call_args_list[0].args[0].full_url) + self.assertIn("page=2", urlopen.call_args_list[1].args[0].full_url) + self.assertEqual(len(releases), 101) + self.assertEqual(releases[0]["tag_name"], "v1.0.0") + self.assertEqual(releases[-1]["tag_name"], "v0.1.0") + + def test_posthog_host_is_overridable(self): + with mock.patch("urllib.request.urlopen") as urlopen: + urlopen.return_value = FakeResponse(b"") + module.send({}, "https://telemetry.example.com") + request = urlopen.call_args.args[0] + self.assertEqual(request.full_url, "https://telemetry.example.com/batch/") + + +class WorkflowTest(unittest.TestCase): + """The workflow that schedules the script, asserted as text so the check + reads the same file GitHub Actions does.""" + + @classmethod + def setUpClass(cls): + cls.text = WORKFLOW.read_text() + + def test_workflow_file_exists(self): + self.assertTrue(WORKFLOW.exists()) + + def test_workflow_is_scheduled_daily_and_dispatchable(self): + self.assertIn("workflow_dispatch", self.text) + self.assertIn("'0 6 * * *'", self.text) + + def test_workflow_permissions_are_contents_read_only(self): + self.assertIn("permissions:", self.text) + self.assertIn("contents: read", self.text) + + def test_workflow_runs_the_script(self): + self.assertIn("python3 scripts/release_downloads.py", self.text) + self.assertIn("runs-on: ubuntu-latest", self.text) + self.assertIn("actions/checkout", self.text) + + def test_workflow_uses_builtin_token_secret_and_host_variable(self): + self.assertIn("GH_TOKEN: ${{ github.token }}", self.text) + self.assertIn( + "CODEAF_POSTHOG_PROJECT_KEY: ${{ secrets.CODEAF_POSTHOG_PROJECT_KEY }}", + self.text, + ) + self.assertIn("POSTHOG_HOST: ${{ vars.POSTHOG_HOST }}", self.text) + + def test_workflow_parses_as_yaml(self): + if yaml is None: + self.skipTest("PyYAML is not installed; the text checks above stand") + document = yaml.safe_load(self.text) + triggers = document[True] + self.assertEqual(triggers["schedule"][0]["cron"], "0 6 * * *") + self.assertIn("workflow_dispatch", triggers) + self.assertEqual(document["permissions"], {"contents": "read"}) + + def test_workflow_and_script_are_the_only_nonstandard_files(self): + # The feature is workflow + script + fixture + test + docs; no Go + # source anywhere carries it. + text = self.text + self.assertNotIn("go run", text) + self.assertNotIn("go build", text) + + +class DocsTest(unittest.TestCase): + def test_download_counts_paragraph_exists(self): + self.assertTrue( + (DOCS / "TELEMETRY.md").exists() + or (DOCS / "rules" / "promotion.md").exists(), + "no docs file carries the paragraph", + ) + path = DOCS / "TELEMETRY.md" + if not path.exists(): + path = DOCS / "rules" / "promotion.md" + text = path.read_text() + flat = " ".join(text.split()) + self.assertIn("## Download counts", text) + self.assertIn("public GitHub release data", flat) + + def test_no_telemetry_doc_on_this_branch(self): + # The paragraph therefore belongs in docs/rules/promotion.md, the + # release docs; this pins the fallback that was chosen. + self.assertFalse((DOCS / "TELEMETRY.md").exists()) + + +class DocsAndWorkflowMixTest(unittest.TestCase): + def test_docs_paragraph_names_workflow_and_script(self): + path = DOCS / "TELEMETRY.md" + if not path.exists(): + path = DOCS / "rules" / "promotion.md" + text = path.read_text() + self.assertIn("release-downloads.yml", text) + self.assertIn("scripts/release_downloads.py", text) + self.assertIn("checksums.txt", text) + + +class ScriptSourceTest(unittest.TestCase): + """The script is python3 standard library only, and imports prove it.""" + + def test_script_imports_stdlib_only(self): + tree = ast.parse(SCRIPT.read_text()) + imported = set() + for node in ast.walk(tree): + if isinstance(node, ast.Import): + imported.update(alias.name.split(".")[0] for alias in node.names) + elif isinstance(node, ast.ImportFrom) and node.level == 0 and node.module: + imported.add(node.module.split(".")[0]) + self.assertEqual( + imported, + {"argparse", "datetime", "json", "os", "re", "sys", "urllib", "uuid"}, + ) + # Nothing outside the standard library is imported, so the script + # runs anywhere python3 does, including the workflow runner. + self.assertLessEqual(imported, set(sys.stdlib_module_names)) + + +class ChannelTest(unittest.TestCase): + def test_channel_words(self): + self.assertEqual(module.release_channel("v1.2.0"), "stable") + self.assertEqual(module.release_channel("v1.3.0-rc.1"), "rc") + self.assertEqual(module.release_channel("v1.4.0-dev.2"), "dev") + self.assertEqual(module.release_channel("v1.5.0-staging.1"), "staging") + + +class SendEndpointTest(unittest.TestCase): + def test_default_host_is_posthog_us(self): + with mock.patch("urllib.request.urlopen") as urlopen: + urlopen.return_value = FakeResponse(b"") + module.send({}, "https://us.i.posthog.com") + self.assertEqual( + urlopen.call_args.args[0].full_url, "https://us.i.posthog.com/batch/" + ) + + +class AssetMatchingTest(unittest.TestCase): + def test_sidecars_and_nonmatching_names_are_skipped(self): + self.assertIsNone(module.asset_os_arch("checksums.txt")) + self.assertIsNone(module.asset_os_arch("codeaf-linux-amd64.tar.gz")) + self.assertIsNone(module.asset_os_arch("codeaf-solaris-sparc")) + self.assertEqual(module.asset_os_arch("codeaf-linux-amd64"), ("linux", "amd64")) + self.assertEqual( + module.asset_os_arch("codeaf-windows-amd64.exe"), ("windows", "amd64") + ) + + +class DraftTest(unittest.TestCase): + def test_draft_releases_produce_no_events(self): + draft = { + "tag_name": "v9.9.9", + "draft": True, + "prerelease": False, + "assets": [{"name": "codeaf-linux-amd64", "download_count": 5}], + } + self.assertEqual(module.build_events([draft], "2026-02-11"), []) + + if __name__ == "__main__": unittest.main() From 8ae5647dc78daf54dec0e15fc334b22835861d9a Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Wed, 16 Sep 2026 23:47:42 -0400 Subject: [PATCH 08/36] release-downloads: four review corrections to the script Channel detection now follows the tag grammar in cmd/codeaf-release/version.go: stable, rc, dev and staging tags match their four patterns and anything else reports "unknown", so dev-20260917-09299019bcdc is no longer mislabelled "stable". --dry-run works without CODEAF_POSTHOG_PROJECT_KEY, the local and test path: the batch prints with api_key "" and exits 0, and only the sending path stops with the notice. Every event carries "$geoip_disable": true next to "$process_person_profile": false, since the sender is a CI runner whose location is meaningless. The module docstring now names release-downloads.yml, the file that exists. The fixture holds one tag of each shape plus an unrecognised one, and the tests assert the channel for each, including the two real dev and staging tags. Co-Authored-By: codeaf --- scripts/release_downloads.py | 35 ++++++++--- scripts/test_release_downloads.py | 87 ++++++++++++++++++++++++--- scripts/testdata/github_releases.json | 36 ++++++++++- 3 files changed, 140 insertions(+), 18 deletions(-) diff --git a/scripts/release_downloads.py b/scripts/release_downloads.py index 175e125276..9e285c8cdb 100644 --- a/scripts/release_downloads.py +++ b/scripts/release_downloads.py @@ -5,7 +5,7 @@ running download count for every asset on a release, and this script reads that count and sends one PostHog event per binary asset so the download trend is visible next to the other codeaf telemetry. It runs daily from -.github/workflows/release_downloads.yml, and runs locally too: +.github/workflows/release-downloads.yml, and runs locally too: python3 scripts/release_downloads.py --dry-run @@ -40,8 +40,16 @@ r"^codeaf-(?Pdarwin|linux|windows)-" r"(?Pamd64|arm64)(?P\.exe)?$" ) -# v1.3.0-rc.1 -> channel rc. Stable tags carry no suffix. -SUFFIX_RE = re.compile(r"^[^-]+-([A-Za-z]+)") +# Tag grammar, matching cmd/codeaf-release/version.go exactly: stable and +# rc tags are vMAJOR.MINOR.PATCH (rc with an -rc.N counter), and the dev and +# staging channels are date-and-commit tags. Anything else is no channel this +# script can name, so it is reported as "unknown". +STABLE_TAG_RE = re.compile(r"^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$") +RC_TAG_RE = re.compile( + r"^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)-rc\.([1-9][0-9]*)$" +) +DEV_TAG_RE = re.compile(r"^dev-[0-9]{8}-[0-9a-f]{12}$") +STAGING_TAG_RE = re.compile(r"^staging-[0-9]{8}-[0-9a-f]{12}$") PROPS = ( "release_tag", "channel", @@ -54,11 +62,16 @@ def release_channel(tag: str) -> str: - """The channel word of a tag: 'stable', or its prerelease suffix (rc).""" - match = SUFFIX_RE.match(tag) - if not match: + """The channel of a tag: stable, rc, dev or staging, else "unknown".""" + if STABLE_TAG_RE.match(tag): return "stable" - return match.group(1) + if RC_TAG_RE.match(tag): + return "rc" + if DEV_TAG_RE.match(tag): + return "dev" + if STAGING_TAG_RE.match(tag): + return "staging" + return "unknown" def asset_os_arch(name: str): @@ -130,7 +143,9 @@ def build_events(releases, snapshot_date: str): # The cumulative total the GitHub API reports. "download_count": asset["download_count"], "snapshot_date": snapshot_date, + # The sender is a CI runner; its location is meaningless. "$process_person_profile": False, + "$geoip_disable": True, }, } ) @@ -173,12 +188,16 @@ def main(argv=None) -> int: args = parser.parse_args(argv) project_key = os.environ.get("CODEAF_POSTHOG_PROJECT_KEY", "") - if not project_key: + if not project_key and not args.dry_run: print( "notice: CODEAF_POSTHOG_PROJECT_KEY is not set; nothing sent", file=sys.stderr, ) return 0 + # --dry-run is the local and test path: it must work without a key, so a + # missing key is printed as a placeholder rather than stopping the run. + if not project_key: + project_key = "" if args.fixture: with open(args.fixture) as fixture: diff --git a/scripts/test_release_downloads.py b/scripts/test_release_downloads.py index e8e3274c7e..5fb91bfa38 100644 --- a/scripts/test_release_downloads.py +++ b/scripts/test_release_downloads.py @@ -90,11 +90,21 @@ def setUpClass(cls): cls.batch = cls.payload["batch"] def test_one_event_per_binary_asset(self): - # 6 assets x 2 releases; checksums.txt and any other sidecar on the - # fixture is not a binary asset and gets no event. - self.assertEqual(len(self.batch), 12) + # 6 assets x 2 full releases plus one asset on each of the dev, staging + # and beta tags; checksums.txt and any other sidecar on the fixture is + # not a binary asset and gets no event. + self.assertEqual(len(self.batch), 15) tags = {event["properties"]["release_tag"] for event in self.batch} - self.assertEqual(tags, {"v1.2.0", "v1.3.0-rc.1"}) + self.assertEqual( + tags, + { + "v1.2.0", + "v1.3.0-rc.1", + "dev-20260917-09299019bcdc", + "staging-20260916-ed5fe58eb546", + "v1.4.0-beta.1", + }, + ) def test_payload_shape(self): self.assertEqual( @@ -108,9 +118,15 @@ def test_event_properties(self): for event in self.batch: properties = event["properties"] self.assertEqual( - sorted(properties), sorted(EVENT_PROPERTIES + ("$process_person_profile",)) + sorted(properties), + sorted( + EVENT_PROPERTIES + + ("$process_person_profile", "$geoip_disable") + ), ) self.assertIs(properties["$process_person_profile"], False) + # The sender is a CI runner; its location is meaningless. + self.assertIs(properties["$geoip_disable"], True) self.assertEqual( properties["snapshot_date"], datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%d"), @@ -128,6 +144,12 @@ def test_channel_and_prerelease(self): } self.assertEqual(channels["v1.2.0"], ("stable", False)) self.assertEqual(channels["v1.3.0-rc.1"], ("rc", True)) + self.assertEqual(channels["dev-20260917-09299019bcdc"], ("dev", True)) + self.assertEqual( + channels["staging-20260916-ed5fe58eb546"], ("staging", True) + ) + # Not one of the four tag shapes this repository publishes. + self.assertEqual(channels["v1.4.0-beta.1"], ("unknown", True)) def test_platforms_and_counts(self): counts = { @@ -185,8 +207,20 @@ def test_no_checksums_or_asset_names_in_payload(self): for event in self.batch: self.assertNotIn("name", event["properties"]) - def test_missing_project_key_is_a_notice_and_success(self): + def test_dry_run_without_project_key_prints_the_batch(self): + # --dry-run is the local and test path: no key in the environment is no + # obstacle, the key is printed as a placeholder and the batch still + # comes out, with no notice. result = run_script("--dry-run", "--fixture", str(FIXTURE)) + self.assertEqual(result.returncode, 0, result.stderr) + payload = json.loads(result.stdout) + self.assertEqual(payload["api_key"], "") + self.assertEqual(len(payload["batch"]), 15) + + def test_sending_without_project_key_is_a_notice_and_success(self): + # Only the sending path stops on a missing key; it never reaches the + # network or the PostHog host. + result = run_script() self.assertEqual(result.returncode, 0) self.assertIn("notice", result.stderr) self.assertIn("CODEAF_POSTHOG_PROJECT_KEY", result.stderr) @@ -334,11 +368,46 @@ def test_script_imports_stdlib_only(self): class ChannelTest(unittest.TestCase): - def test_channel_words(self): + """The four tag shapes of cmd/codeaf-release/version.go, and nothing + else: any other tag is reported as "unknown" rather than mislabelled.""" + + def test_stable_tags(self): self.assertEqual(module.release_channel("v1.2.0"), "stable") + self.assertEqual(module.release_channel("v0.0.1"), "stable") + self.assertEqual(module.release_channel("v10.20.30"), "stable") + + def test_rc_tags(self): self.assertEqual(module.release_channel("v1.3.0-rc.1"), "rc") - self.assertEqual(module.release_channel("v1.4.0-dev.2"), "dev") - self.assertEqual(module.release_channel("v1.5.0-staging.1"), "staging") + self.assertEqual(module.release_channel("v2.0.0-rc.12"), "rc") + + def test_dev_tags(self): + self.assertEqual( + module.release_channel("dev-20260917-09299019bcdc"), "dev" + ) + self.assertEqual(module.release_channel("dev-20250101-abcdef012345"), "dev") + + def test_staging_tags(self): + self.assertEqual( + module.release_channel("staging-20260916-ed5fe58eb546"), "staging" + ) + self.assertEqual( + module.release_channel("staging-20250102-000000000abc"), "staging" + ) + + def test_unrecognised_tags_are_unknown(self): + for tag in ( + "v1.4.0-beta.1", + "v1.3.0-rc.0", + "v1.3.0-rc", + "v01.2.0", + "1.2.0", + "v1.2", + "dev-20260917-09299019bcdcX", + "staging-20260916-ED5FE58EB546", + "nightly-20260917-09299019bcdc", + "", + ): + self.assertEqual(module.release_channel(tag), "unknown", tag) class SendEndpointTest(unittest.TestCase): diff --git a/scripts/testdata/github_releases.json b/scripts/testdata/github_releases.json index f2aa933380..fef8af259f 100644 --- a/scripts/testdata/github_releases.json +++ b/scripts/testdata/github_releases.json @@ -2,7 +2,8 @@ { "tag_name": "v1.3.0-rc.1", "prerelease": true, - "draft": false, "assets": [ + "draft": false, + "assets": [ { "name": "codeaf-linux-amd64", "download_count": 4021 @@ -67,5 +68,38 @@ "download_count": 3302 } ] + }, + { + "tag_name": "dev-20260917-09299019bcdc", + "prerelease": true, + "draft": false, + "assets": [ + { + "name": "codeaf-linux-amd64", + "download_count": 37 + } + ] + }, + { + "tag_name": "staging-20260916-ed5fe58eb546", + "prerelease": true, + "draft": false, + "assets": [ + { + "name": "codeaf-darwin-arm64", + "download_count": 24 + } + ] + }, + { + "tag_name": "v1.4.0-beta.1", + "prerelease": true, + "draft": false, + "assets": [ + { + "name": "codeaf-windows-amd64.exe", + "download_count": 9 + } + ] } ] From 520e03b36280945f4d1756300e985dbe1cfb07b7 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Wed, 16 Sep 2026 23:51:56 -0400 Subject: [PATCH 09/36] release-downloads: test runs on Python 3.9 too ScriptSourceTest read sys.stdlib_module_names unguarded; the attribute arrived in Python 3.10, so the suite errored on macOS stock python3 (3.9.6). Guarded with getattr so the pinned import-set check stands on its own where the attribute is absent; 32 tests pass on 3.9, 3.10 and 3.14. Co-Authored-By: codeaf --- scripts/test_release_downloads.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/test_release_downloads.py b/scripts/test_release_downloads.py index 5fb91bfa38..9ef01a31e6 100644 --- a/scripts/test_release_downloads.py +++ b/scripts/test_release_downloads.py @@ -364,7 +364,11 @@ def test_script_imports_stdlib_only(self): ) # Nothing outside the standard library is imported, so the script # runs anywhere python3 does, including the workflow runner. - self.assertLessEqual(imported, set(sys.stdlib_module_names)) + # sys.stdlib_module_names exists from Python 3.10; on older interpreters + # the pinned import set above is the check that stands. + stdlib = getattr(sys, "stdlib_module_names", None) + if stdlib is not None: + self.assertLessEqual(imported, set(stdlib)) class ChannelTest(unittest.TestCase): From 8116eb000a55eadb9d8571be96e7bd8c95667f13 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 01:46:46 -0400 Subject: [PATCH 10/36] installer telemetry: the notice is read from docs/TELEMETRY.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The test read .telemetry-contract.md, a working note that is not in the repository and never will be, so it aborted with "telemetry contract is missing" in a clean checkout. It now extracts the fenced block under "## The notice" in docs/TELEMETRY.md — the same bytes as the Notice constant in internal/telemetry/notice.go — and holds the installer's TELEMETRY_NOTICE to it as before; a new assertion holds the README's telemetry fence to it too, byte for byte, so the invariant the suite existed to guard now has its third leg. Comments in scripts/install.sh, internal/telemetry/telemetry.go and this test now name docs/TELEMETRY.md as the source of the notice. No Go code changed and no Go tests were run. shellcheck is clean; the suite's 33 passing assertions include the two new notice checks, and the same two stat(1)-related assertions that already failed on this GNU machine before the change still fail with identical output. Co-Authored-By: codeaf --- internal/telemetry/telemetry.go | 6 +++--- scripts/install.sh | 2 +- test/installer-telemetry.sh | 29 ++++++++++++++++++++--------- 3 files changed, 24 insertions(+), 13 deletions(-) diff --git a/internal/telemetry/telemetry.go b/internal/telemetry/telemetry.go index d93e7f7fec..ab0f7040ca 100644 --- a/internal/telemetry/telemetry.go +++ b/internal/telemetry/telemetry.go @@ -1,11 +1,11 @@ // Package telemetry implements codeaf's anonymous usage counting exactly as -// .telemetry-contract.md spells it: four typed events, an allowlisted property +// docs/TELEMETRY.md spells it: four typed events, an allowlisted property // set, a local spool, and one deadline-bounded flush. It is a library only — // nothing in cmd/codeaf, internal/session or internal/config reads it yet; the // wiring is a later job. // -// The law of the package is the contract at the repository root. When this file -// and the contract disagree, the contract wins and this package is the defect. +// The law of the package is docs/TELEMETRY.md. When this file and the doc +// disagree, the doc wins and this package is the defect. package telemetry import ( diff --git a/scripts/install.sh b/scripts/install.sh index a4ac2048e6..94de911948 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -6,7 +6,7 @@ REPOSITORY="Agent-Field/codeaf" LEGACY_REPOSITORY="Agent-Field/aforge-v2" # Remove after the one-release repository fallback. # legacy-name CHANNEL="${CHANNEL:-stable}" VERSION="${VERSION:-}" -# The telemetry notice, verbatim from the wire contract (.telemetry-contract.md). +# The telemetry notice, verbatim from docs/TELEMETRY.md. # The installer only writes a local install marker and prints this text; it # never sends telemetry, and it makes no request that the download steps did # not already make. diff --git a/test/installer-telemetry.sh b/test/installer-telemetry.sh index 69e58aa852..ff0a43977c 100755 --- a/test/installer-telemetry.sh +++ b/test/installer-telemetry.sh @@ -1,17 +1,18 @@ #!/usr/bin/env bash # THE INSTALLER'S TELEMETRY DUTIES, PROVED WITHOUT A NETWORK. # -# The wire contract (.telemetry-contract.md) is one text in three places — the -# binary, this installer and the README — and only a test notices when one of -# them drifts. The installer's main body downloads a release, so this test -# never sources it whole: it lifts out the three telemetry functions and runs -# them against a temporary state root. Nothing here opens a socket. +# The notice text lives once, byte for byte, in docs/TELEMETRY.md and is +# quoted in three places — the binary, this installer and the README — and +# only a test notices when one of them drifts. The installer's main body +# downloads a release, so this test never sources it whole: it lifts out the +# three telemetry functions and runs them against a temporary state root. +# Nothing here opens a socket. set -euo pipefail cd "$(git rev-parse --show-toplevel)" script=scripts/install.sh -contract=.telemetry-contract.md -test -f "$contract" || { echo "telemetry contract is missing; this test reads it"; exit 1; } +doc=docs/TELEMETRY.md +test -f "$doc" || { echo "docs/TELEMETRY.md is missing; this test reads the notice from it"; exit 1; } eval "$(awk '/^TELEMETRY_NOTICE=/{f=1} /^VERBOSE=/{f=0} f' "$script")" eval "$(sed -n '/^telemetry_off()/,/^}/p; /^write_install_marker()/,/^}/p; /^print_telemetry_notice()/,/^}/p' "$script")" @@ -80,10 +81,20 @@ ok "unwritable state root does not fail the install" 'write_install_marker /proc notice="$tmp/notice.txt" print_telemetry_notice 2> "$notice" -expected=$(awk '/^Notice, printed once/{f=1;next} f' "$contract") +expected=$(awk ' + /^## The notice$/ {f=1; next} + f && /^```$/ {f++; next} + f == 2 {print} +' "$doc") body=$(sed 1d "$notice") ok "one blank line before the notice" '[ -z "$(head -n 1 "$notice")" ]' -ok "notice matches the contract verbatim" '[ "$body" = "$expected" ]' +ok "notice matches docs/TELEMETRY.md verbatim" '[ "$body" = "$expected" ]' +readme_block=$(awk ' + /^```text$/ {f = 1; buf = ""; next} + /^```$/ {if (f && buf ~ /codeaf sends anonymous usage counts/) {print buf; exit} f = 0; next} + f {buf = buf $0 "\n"} +' README.md) +ok "README quotes the notice verbatim" '[ -n "$readme_block" ] && [ "$(printf "%s\n" "$expected")" = "$readme_block" ]' ok "notice goes to stderr, nothing to stdout" '[ -z "$(print_telemetry_notice 2>/dev/null)" ]' for v in off 0 false OFF False; do From de6e36a09ba258ff3164b7aace588b16a05db3e8 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 01:47:21 -0400 Subject: [PATCH 11/36] release-downloads: the download-counts paragraph lives in docs/TELEMETRY.md docs/rules/promotion.md was the only home the paragraph had, and it came with a fallback pinning the branch to the state before the telemetry doc existed. The paragraph moves, byte for byte, into docs/TELEMETRY.md as a final "## Download counts" section beside the rest of the telemetry story, and promotion.md returns to its origin/dev content, so this branch now touches no release docs. test_no_telemetry_doc_on_this_branch asserted docs/TELEMETRY.md did not exist and is deleted. The two tests that read the paragraph read it where it now lives: both require a "## Download counts" section in docs/TELEMETRY.md naming .github/workflows/release-downloads.yml, scripts/release_downloads.py and the checksums.txt sidecar, and the promotion.md fallback is gone. No Go code changed and no Go tests were run; the property table in docs/TELEMETRY.md is untouched. The suite passes: 31 tests, OK. Co-Authored-By: codeaf --- docs/TELEMETRY.md | 14 ++++++++++++++ docs/rules/promotion.md | 14 -------------- scripts/test_release_downloads.py | 21 +++------------------ 3 files changed, 17 insertions(+), 32 deletions(-) diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md index 5b048a9bcd..584525b124 100644 --- a/docs/TELEMETRY.md +++ b/docs/TELEMETRY.md @@ -85,3 +85,17 @@ Any one of these, before codeaf starts: A build that cannot name its own source — dirty or unstamped — never reports, and neither does a test binary. + +## Download counts + +The download numbers this repository reports come from public GitHub release +data: the GitHub API publishes a cumulative download count for every asset on +a release, and a daily scheduled workflow reads those counts and sends one +event per binary asset to PostHog (`codeaf:release_downloads`). No code in +the binary is involved, nothing is collected from the person downloading, and +the only facts in the event are the release tag, the asset's platform, and +the count GitHub already shows on the release page. Sidecar files such as +`checksums.txt` are not counted. The workflow lives in +`.github/workflows/release-downloads.yml`; the script behind it is +`scripts/release_downloads.py`, and running it with `--dry-run` prints the +batch it would send. diff --git a/docs/rules/promotion.md b/docs/rules/promotion.md index e89fceda59..d5e8c438e8 100644 --- a/docs/rules/promotion.md +++ b/docs/rules/promotion.md @@ -88,20 +88,6 @@ Every release carries `THIRD-PARTY-NOTICES.md` beside its binaries, inside the same `checksums.txt`; `go run ./cmd/codeaf-notices generate` refreshes it when the dependencies move. -## Download counts - -The download numbers this repository reports come from public GitHub release -data: the GitHub API publishes a cumulative download count for every asset on -a release, and a daily scheduled workflow reads those counts and sends one -event per binary asset to PostHog (`codeaf:release_downloads`). No code in -the binary is involved, nothing is collected from the person downloading, and -the only facts in the event are the release tag, the asset's platform, and -the count GitHub already shows on the release page. Sidecar files such as -`checksums.txt` are not counted. The workflow lives in -`.github/workflows/release-downloads.yml`; the script behind it is -`scripts/release_downloads.py`, and running it with `--dry-run` prints the -batch it would send. - ## Rolling back Users can pin a known tag while a repair moves forward: diff --git a/scripts/test_release_downloads.py b/scripts/test_release_downloads.py index 9ef01a31e6..4d1c1604c3 100644 --- a/scripts/test_release_downloads.py +++ b/scripts/test_release_downloads.py @@ -317,31 +317,16 @@ def test_workflow_and_script_are_the_only_nonstandard_files(self): class DocsTest(unittest.TestCase): def test_download_counts_paragraph_exists(self): - self.assertTrue( - (DOCS / "TELEMETRY.md").exists() - or (DOCS / "rules" / "promotion.md").exists(), - "no docs file carries the paragraph", - ) - path = DOCS / "TELEMETRY.md" - if not path.exists(): - path = DOCS / "rules" / "promotion.md" - text = path.read_text() + text = (DOCS / "TELEMETRY.md").read_text() flat = " ".join(text.split()) self.assertIn("## Download counts", text) self.assertIn("public GitHub release data", flat) - def test_no_telemetry_doc_on_this_branch(self): - # The paragraph therefore belongs in docs/rules/promotion.md, the - # release docs; this pins the fallback that was chosen. - self.assertFalse((DOCS / "TELEMETRY.md").exists()) - class DocsAndWorkflowMixTest(unittest.TestCase): def test_docs_paragraph_names_workflow_and_script(self): - path = DOCS / "TELEMETRY.md" - if not path.exists(): - path = DOCS / "rules" / "promotion.md" - text = path.read_text() + text = (DOCS / "TELEMETRY.md").read_text() + self.assertIn("## Download counts", text) self.assertIn("release-downloads.yml", text) self.assertIn("scripts/release_downloads.py", text) self.assertIn("checksums.txt", text) From 402df2cf49d7253c10abb34ac4100b4ddfa20609 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 01:54:05 -0400 Subject: [PATCH 12/36] installer telemetry: the mode check asks GNU stat first On Linux `stat -f` succeeds and reports the filesystem, so the BSD-first order never reached the GNU form and both permission checks failed. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01CqfhxLrrBNQxwPNvEgoBGR --- test/installer-telemetry.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/installer-telemetry.sh b/test/installer-telemetry.sh index ff0a43977c..b4d29871e8 100755 --- a/test/installer-telemetry.sh +++ b/test/installer-telemetry.sh @@ -45,7 +45,8 @@ unset CODEAF_TELEMETRY DO_NOT_TRACK || true CHANNEL=stable write_install_marker "$tmp/state" f="$tmp/state/telemetry/install.json" ok "marker exists" '[ -f "$f" ]' -mode_of() { stat -f '%Lp' "$1" 2>/dev/null || stat -c '%a' "$1"; } +# GNU stat first: on Linux `stat -f` succeeds too, but reports the filesystem. +mode_of() { stat -c '%a' "$1" 2>/dev/null || stat -f '%Lp' "$1"; } json_valid() { if command -v python3 >/dev/null 2>&1; then python3 -m json.tool "$f" >/dev/null 2>&1 From cd11bab306bcb82a055bc30be0752cdef818fa89 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 02:01:20 -0400 Subject: [PATCH 13/36] telemetry: refuse to send to the production relay from a test binary Co-Authored-By: codeaf --- internal/telemetry/spool.go | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/internal/telemetry/spool.go b/internal/telemetry/spool.go index 1db9ef9527..9e1b826afc 100644 --- a/internal/telemetry/spool.go +++ b/internal/telemetry/spool.go @@ -320,6 +320,17 @@ func adoptStaleSendingFiles() { // sendBatch POSTs one batch. It answers true only when the relay answered // 2xx within the deadline; any error, any other status, and the batch stays. func sendBatch(ctx context.Context, deadline time.Time, batch []spoolEntry) bool { + // A test binary never reaches the production relay. The only state in + // which a send under `go test` could land on DefaultEndpoint is an unset + // CODEAF_TELEMETRY_ENDPOINT — a test that forgot its relay — so refuse + // before any request is built, and refuse the way a failed send is + // refused: the batch stays in the spool and Flush says nothing. This + // check sits below forceLadderForTest, which reaches sendBatch only + // through the enabledFor gate and cannot reopen it; on a production + // binary it costs the one bool underGoTest answers. + if underGoTest() && Endpoint() == DefaultEndpoint { + return false + } remaining := time.Until(deadline) if remaining <= 0 { return false @@ -374,8 +385,11 @@ var allowlistedEvents = map[string]bool{ "first_run": true, "session_started": true, "session_ended": true, "fault": true, } -// httpClient is the one client. No timeout of its own: the deadline lives in -// the request context, where the caller's hard budget is. +// httpClient is the one client, and the package's one send seam: production +// declares it here and never reassigns it, and a test may swap it for a +// recording client or transport and restore it with t.Cleanup. No timeout of +// its own: the deadline lives in the request context, where the caller's hard +// budget is. var httpClient = &http.Client{} // readSpoolLines reads the spool as entries, skipping and reporting lines it From b5b59dd5d05337aaea202fe7775aaced79fcdef1 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 02:01:57 -0400 Subject: [PATCH 14/36] telemetry: testHome points the endpoint at a dead loopback Co-Authored-By: codeaf --- internal/telemetry/telemetry_test.go | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go index b68b15f2c0..4448e28d38 100644 --- a/internal/telemetry/telemetry_test.go +++ b/internal/telemetry/telemetry_test.go @@ -27,8 +27,12 @@ func freshClock(t *testing.T) time.Time { return time.Now() } // testHome points the state root at a directory this test chose, so nothing // this package writes can land in the home of whoever ran `go test`, and // clears every variable the opt-out ladder reads, so a test that wants -// telemetry on gets an environment that agrees. Every test in the package -// starts here. +// telemetry on gets an environment that agrees. It then points the endpoint +// at a dead loopback address, so a flush a test forgot to give a relay to +// fails locally instead of POSTing to the production relay. Tests that need +// a working relay call newRelay after testHome, which overrides it; a test +// that asserts the default endpoint unsets the variable itself with unsetEnv +// and must not flush. Every test in the package starts here. func testHome(t *testing.T) { t.Helper() t.Setenv(home.EnvVar, t.TempDir()) @@ -41,6 +45,8 @@ func testHome(t *testing.T) { } { unsetEnv(t, name) } + // Nothing listens here, and loopback dialing is refused immediately. + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "http://127.0.0.1:1/telemetry") // A test binary trips the test and unstamped-build rungs of the ladder, // which now gates Spool, SpoolSync and Flush. Spool tests exercise the // write paths, so they run as if the ladder had answered on — the @@ -468,6 +474,10 @@ func TestInstallMethodReadsOnlyTheInstallerWords(t *testing.T) { func TestEndpointOverrideAndDefault(t *testing.T) { testHome(t) + // testHome points the endpoint at a dead loopback address so no test can + // reach production by default. This test is the one that must see the + // default answer, so it unsets the override itself — and it never flushes. + unsetEnv(t, "CODEAF_TELEMETRY_ENDPOINT") if got := Endpoint(); got != DefaultEndpoint { t.Errorf("Endpoint() = %q, want the contract relay %q", got, DefaultEndpoint) } From 611f7dceb0b02d58659a481b375c25283498d4b5 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 02:02:11 -0400 Subject: [PATCH 15/36] telemetry: the orphan-adoption test names the dead relay it now gets Co-Authored-By: codeaf --- internal/telemetry/spool_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/telemetry/spool_test.go b/internal/telemetry/spool_test.go index a299de2c36..ccf860471b 100644 --- a/internal/telemetry/spool_test.go +++ b/internal/telemetry/spool_test.go @@ -589,8 +589,10 @@ func TestFlushAdoptsOrphanedSendingFiles(t *testing.T) { } // The live event was already in spool.jsonl before the flush, and // adoption appends the orphan's lines to that same file, so the live - // line comes first and the three adopted lines follow it. The live - // flush sent nothing (no relay), so four lines remain. + // line comes first and the three adopted lines follow it. The relay is + // unreachable — testHome points CODEAF_TELEMETRY_ENDPOINT at a dead + // loopback address — so every send fails, Flush stays silent, and all + // four lines are appended back: they remain spooled, in order. left := SpoolContents() if len(left) != 4 { t.Fatalf("%d lines remain after adopting the orphan, want 4 (1 live, 3 adopted)", len(left)) From c5fb2c0a4cbf874fcd3212147de8fad063d53bba Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 02:03:08 -0400 Subject: [PATCH 16/36] telemetry: a test pins that no test binary reaches the production relay Co-Authored-By: codeaf --- internal/telemetry/telemetry_test.go | 67 ++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go index 4448e28d38..711e7011cf 100644 --- a/internal/telemetry/telemetry_test.go +++ b/internal/telemetry/telemetry_test.go @@ -4,9 +4,13 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" + "fmt" "io" + "net/http" "os" "strings" + "sync" "testing" "time" @@ -489,3 +493,66 @@ func TestEndpointOverrideAndDefault(t *testing.T) { t.Errorf("Endpoint() = %q, want the override", got) } } + +// roundTripCounter is the recording transport TestNoTestBinaryCanReachThe +// ProductionRelay swaps in behind the package's client: it counts the round +// trips it is asked to make and answers a refusal it is never expected to see. +type roundTripCounter struct { + mu sync.Mutex + trips int +} + +func (c *roundTripCounter) RoundTrip(request *http.Request) (*http.Response, error) { + c.mu.Lock() + c.trips++ + c.mu.Unlock() + return nil, fmt.Errorf("telemetry test transport: no round trip may happen, saw %s %s", request.Method, request.URL) +} + +func (c *roundTripCounter) count() int { + c.mu.Lock() + defer c.mu.Unlock() + return c.trips +} + +// TestNoTestBinaryCanReachTheProductionRelay pins the send path's law: when a +// test binary ends up with the default endpoint — an endpoint override nobody +// set — the flush refuses before building any request, and does so even with +// the ladder forced on. The recording transport proves no HTTP round trip was +// attempted, not merely that the spool survived; the event stays spooled the +// way a failed send leaves it, and nothing reaches the real relay. +func TestNoTestBinaryCanReachTheProductionRelay(t *testing.T) { + testHome(t) + // The default is what must be watched here: unset the dead loopback + // address testHome set, so Endpoint() answers the production relay. + unsetEnv(t, "CODEAF_TELEMETRY_ENDPOINT") + if got := Endpoint(); got != DefaultEndpoint { + t.Fatalf("Endpoint() = %q, want the default the law watches for", got) + } + counter := &roundTripCounter{} + previous := httpClient + httpClient = &http.Client{Transport: counter} + t.Cleanup(func() { httpClient = previous }) + MarkNoticeShown() + event := SessionStarted(ModeChat, false, "session-law", freshClock(t)) + if err := SpoolSync(event); err != nil { + t.Fatalf("SpoolSync returned %v; a spool that is never sent answers nil", err) + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush returned %v; a refused send must stay silent", err) + } + if trips := counter.count(); trips != 0 { + t.Fatalf("the send path made %d round trips under go test against the default endpoint, want 0", trips) + } + left := SpoolContents() + if len(left) != 1 { + t.Fatalf("%d lines remain in the spool, want the refused event still spooled", len(left)) + } + var row map[string]any + if err := json.Unmarshal(left[0], &row); err != nil { + t.Fatal(err) + } + if row["event_id"] != event.ID { + t.Errorf("the remaining event is %v, want the one that was spooled", row["event_id"]) + } +} From 8b8c3ee9bfcb7c6ff7d17d0b60324d4493bc0c23 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 02:21:48 -0400 Subject: [PATCH 17/36] telemetry: three tests pin the test-binary relay refusal Each asserts through real calls, not a re-implemented effect: the newRelay override lands after testHome and a flush reaches that recorder; a default-endpoint flush is refused like a dead relay (zero round trips, event still spooled); forcing the ladder on cannot reopen the path. Co-Authored-By: codeaf --- internal/telemetry/telemetry_test.go | 114 +++++++++++++++++++++++++++ 1 file changed, 114 insertions(+) diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go index 711e7011cf..9f0f1672fc 100644 --- a/internal/telemetry/telemetry_test.go +++ b/internal/telemetry/telemetry_test.go @@ -556,3 +556,117 @@ func TestNoTestBinaryCanReachTheProductionRelay(t *testing.T) { t.Errorf("the remaining event is %v, want the one that was spooled", row["event_id"]) } } + +// TestNewRelayAfterTestHomeOverridesTheDeadLoopback pins the relay seam's +// ordering: testHome leaves the endpoint at the dead loopback address, and a +// relay started after it must replace that override — otherwise every +// relay-using test would flush at a port nothing listens on and pass by +// accident. The recorder receiving the event is the proof: a real round trip +// happened, to this server and no other. +func TestNewRelayAfterTestHomeOverridesTheDeadLoopback(t *testing.T) { + testHome(t) + if got := Endpoint(); got != "http://127.0.0.1:1/telemetry" { + t.Fatalf("after testHome the endpoint is %q, want the dead loopback it sets", got) + } + recorder := newRelay(t) + if override := os.Getenv("CODEAF_TELEMETRY_ENDPOINT"); override != Endpoint() { + t.Fatalf("after newRelay Endpoint() answers %q but the override is %q", Endpoint(), override) + } + MarkNoticeShown() + event := SessionStarted(ModeChat, false, "session-relay", freshClock(t)) + if err := SpoolSync(event); err != nil { + t.Fatalf("SpoolSync returned %v; a spool that is sent answers nil", err) + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush returned %v; it must never fail a run", err) + } + if got := recorder.count(); got != 1 { + t.Fatalf("the relay saw %d posts after the override, want 1 — the flush must reach the relay newRelay started", got) + } + if len(recorder.posts[0].events) != 1 || recorder.posts[0].events[0]["event_id"] != event.ID { + t.Errorf("the relay received %v, want the one spooled event %q", recorder.posts[0].events, event.ID) + } + if left := len(SpoolContents()); left != 0 { + t.Errorf("%d lines remain after a confirmed send, want 0", left) + } +} + +// TestFlushAgainstTheDefaultEndpointRefusesLikeADeadRelay pins the refusal as +// Flush meets it, without the relay: with the endpoint left at the production +// default, a flush holding one sendable event must behave exactly as it does +// against a dead relay — nothing said, the event still spooled. Flush's nil +// proves nothing on its own; the counter's zero is the assertion that carries. +func TestFlushAgainstTheDefaultEndpointRefusesLikeADeadRelay(t *testing.T) { + testHome(t) + // testHome's dead loopback must not soften this: the refusal is about the + // default, so the override is removed and the default watched instead. + unsetEnv(t, "CODEAF_TELEMETRY_ENDPOINT") + if got := Endpoint(); got != DefaultEndpoint { + t.Fatalf("Endpoint() = %q, want the production default the refusal watches", got) + } + counter := &roundTripCounter{} + previous := httpClient + httpClient = &http.Client{Transport: counter} + t.Cleanup(func() { httpClient = previous }) + MarkNoticeShown() + event := SessionStarted(ModeChat, false, "session-refused", freshClock(t)) + if err := SpoolSync(event); err != nil { + t.Fatalf("SpoolSync returned %v; a spool that is never sent answers nil", err) + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush returned %v; a refused send must stay silent, as with a dead relay", err) + } + if trips := counter.count(); trips != 0 { + t.Fatalf("the send path made %d round trips against the default endpoint, want 0", trips) + } + left := SpoolContents() + if len(left) != 1 { + t.Fatalf("%d lines remain after the refusal, want the event still spooled", len(left)) + } + var row map[string]any + if err := json.Unmarshal(left[0], &row); err != nil { + t.Fatal(err) + } + if row["event_id"] != event.ID { + t.Errorf("the remaining line is %v, want the spooled event %q", row["event_id"], event.ID) + } +} + +// TestForcingTheLadderOnCannotReopenTheDefaultEndpointPath pins where the +// refusal sits: below forceLadderForTest. testHome forces the ladder on for +// every spool test, so a refusal the ladder could override would gate nothing +// this package ever runs. The ladder is forced on again, explicitly, and the +// flush must still make zero round trips and keep the event. +func TestForcingTheLadderOnCannotReopenTheDefaultEndpointPath(t *testing.T) { + testHome(t) + forceLadderForTest(t, true) + unsetEnv(t, "CODEAF_TELEMETRY_ENDPOINT") + if got := Endpoint(); got != DefaultEndpoint { + t.Fatalf("Endpoint() = %q, want the production default", got) + } + counter := &roundTripCounter{} + previous := httpClient + httpClient = &http.Client{Transport: counter} + t.Cleanup(func() { httpClient = previous }) + MarkNoticeShown() + event := SessionStarted(ModeChat, false, "session-forced-ladder", freshClock(t)) + if err := SpoolSync(event); err != nil { + t.Fatalf("SpoolSync returned %v; a spool that is never sent answers nil", err) + } + if err := Flush(context.Background()); err != nil { + t.Fatalf("Flush returned %v; a refused send must stay silent", err) + } + if trips := counter.count(); trips != 0 { + t.Fatalf("with the ladder forced on the send path made %d round trips against the default endpoint, want 0", trips) + } + if left := len(SpoolContents()); left != 1 { + t.Fatalf("%d lines remain after the forced-ladder flush, want the event still spooled", left) + } + var row map[string]any + if err := json.Unmarshal(SpoolContents()[0], &row); err != nil { + t.Fatal(err) + } + if row["event_id"] != event.ID { + t.Errorf("the remaining line is %v, want the spooled event %q", row["event_id"], event.ID) + } +} From 8857a460af834c10ab1b3c299efb6342f82fe013 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 01:37:35 -0400 Subject: [PATCH 18/36] telemetry: allocation-free session counters Snapshot returns the SessionStats fields the SessionEnded constructor takes, so the wiring hands the struct straight over. Cost is stored as integer micro-dollars and divided back once, in Snapshot. Count* never ask whether telemetry is enabled and never allocate. Co-Authored-By: codeaf --- internal/telemetry/counters.go | 121 ++++++++++++++++++ internal/telemetry/counters_test.go | 189 ++++++++++++++++++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 internal/telemetry/counters.go create mode 100644 internal/telemetry/counters_test.go diff --git a/internal/telemetry/counters.go b/internal/telemetry/counters.go new file mode 100644 index 0000000000..375e292b54 --- /dev/null +++ b/internal/telemetry/counters.go @@ -0,0 +1,121 @@ +package telemetry + +import ( + "math" + "sync/atomic" +) + +// The session counters are the three chokepoints a run passes through — a +// turn sealed, a model call answered, a tool call answered — and the dollar +// figure those model calls spent. They are the numbers [SessionEnded] needs, +// arrived at WITHOUT the binary having to carry its own tally: the wiring +// calls CountTurn, CountModelCall and CountToolCall on the paths the events +// describe, and Snapshot hands the constructor exactly the [SessionStats] +// fields it takes, so nothing between the two adapts anything. +// +// THE COUNTERS COUNT WHETHER OR NOT TELEMETRY IS ENABLED, and they do it by +// never asking. There is no ladder read here, no enabledFor(), no branch on +// the opt-out at all: counting three integers costs a few nanoseconds and +// nothing else, so the decision about whether to SEND them is taken once, +// later, by the event path — where it belongs. A counter that went quiet +// when telemetry was off would make Snapshot lie about a run that DID +// happen, and a counter that asked the ladder on every model call would put +// the opt-out check on the hot path the latency law guards. + +// counters is the process's one tally, package-level because the chokepoints +// calling it sit in internal/session and internal/provider, which reach the +// package as a whole rather than through an object a run would have to carry. +// Every field is an atomic int64, which is the whole allocation story: each +// CountX is one Add and one conditional Add, on memory that exists before the +// first session and never grows. +var counters struct { + turns atomic.Int64 + modelCalls atomic.Int64 + modelFail atomic.Int64 + toolCalls atomic.Int64 + toolFail atomic.Int64 + // costMicro is the session's spend in integer MICRO-DOLLARS, not float + // dollars. A float tally added to N times accumulates N rounding errors + // and the drift is invisible until two paths disagree about a cost that + // never happened; an integer tally of 1e-6 dollar units is exact for + // every cost a provider reports at six places, and the one rounding — + // in costToMicro, at the boundary — is the only one there is. + costMicro atomic.Int64 +} + +// costToMicro is the single float-to-integer crossing. math.Round rather than +// truncation so a cost of 0.0000005 cents up rather than down: truncating at +// the boundary takes money off a person's own total, and rounding to nearest +// is the honest reading of "the record's cost". +func costToMicro(costUSD float64) int64 { + return int64(math.Round(costUSD * 1e6)) +} + +// microToCost is the crossing back, in Snapshot only, so the constructor +// receives the float dollars its SessionStats field names. +func microToCost(micro int64) float64 { + return float64(micro) / 1e6 +} + +// CountTurn records one sealed turn. It is one atomic add and nothing else: +// no allocation, no ladder read, no lock. +func CountTurn() { counters.turns.Add(1) } + +// CountModelCall records one model call answered: ok false also counts the +// failure, because model_calls_failed is a field the constructor needs and a +// second call at the chokepoint would double-count the call itself. costUSD +// is folded into the integer tally; a call with no cost figure hands over 0 +// and adds nothing, which is the record's own answer. +func CountModelCall(ok bool, costUSD float64) { + counters.modelCalls.Add(1) + if !ok { + counters.modelFail.Add(1) + } + counters.costMicro.Add(costToMicro(costUSD)) +} + +// CountToolCall records one tool call answered, with its own failure count +// for the same reason [CountModelCall] keeps one. +func CountToolCall(ok bool) { + counters.toolCalls.Add(1) + if !ok { + counters.toolFail.Add(1) + } +} + +// Snapshot is the tally as the SessionEnded constructor takes it: the +// SessionStats fields it needs — turns, model calls, model calls failed, +// tool calls, tool calls failed, cost — named as SessionStats names them, so +// the wiring hands the struct straight to the constructor without adapting. +// It builds ONE struct and reads each atomic once; the six numbers are +// therefore not from a single instant, which is fine — a run's last model +// call can land after its last turn sealed, and a snapshot that could not be +// taken mid-run would be a snapshot that could not be taken at all. +// +// The fields SessionEnded fills and Snapshot does not return — Duration, +// StopReason, ExitCode — belong to the run's END, which is the caller's fact: +// elapsed time and exit status live where the process does, and Snapshot +// carries only what the chokepoints counted. +func Snapshot() SessionStats { + return SessionStats{ + Turns: int(counters.turns.Load()), + ModelCalls: int(counters.modelCalls.Load()), + ModelCallsFailed: int(counters.modelFail.Load()), + ToolCalls: int(counters.toolCalls.Load()), + ToolCallsFailed: int(counters.toolFail.Load()), + CostUSD: microToCost(counters.costMicro.Load()), + } +} + +// resetCountersForTest zeroes the tally. The counters are deliberately not +// reset between tests — they are process-wide state the way any global +// counter is — but a test that asserts exact totals cannot share a tally +// with the test before it. +func resetCountersForTest() { + counters.turns.Store(0) + counters.modelCalls.Store(0) + counters.modelFail.Store(0) + counters.toolCalls.Store(0) + counters.toolFail.Store(0) + counters.costMicro.Store(0) +} diff --git a/internal/telemetry/counters_test.go b/internal/telemetry/counters_test.go new file mode 100644 index 0000000000..76af481cd0 --- /dev/null +++ b/internal/telemetry/counters_test.go @@ -0,0 +1,189 @@ +package telemetry + +import ( + "sync" + "testing" + "time" +) + +// TestSnapshotReturnsTheFieldsSessionEndedNeeds is the contract the wiring +// counts on: Snapshot hands the SessionEnded constructor exactly the six +// fields it takes, named as SessionStats names them. This compiles against +// those names, so the day either side drifts this test stops building and +// the wiring agent learns before it adapts anything. +func TestSnapshotReturnsTheFieldsSessionEndedNeeds(t *testing.T) { + resetCountersForTest() + defer resetCountersForTest() + + CountTurn() + CountTurn() + CountModelCall(true, 0.25) + CountModelCall(false, 0) + CountToolCall(true) + CountToolCall(true) + CountToolCall(false) + + // The struct SessionEnded's own constructor is about to take: field + // names are SessionStats's, and a rename there fails this line. + got := Snapshot() + want := SessionStats{ + Turns: 2, + ModelCalls: 2, + ModelCallsFailed: 1, + ToolCalls: 3, + ToolCallsFailed: 1, + CostUSD: 0.25, + } + if got != want { + t.Fatalf("Snapshot() = %+v, want %+v", got, want) + } + + // And the constructor takes it unchanged. This is the line that keeps + // the two sides from drifting: if the constructor's signature ever + // grows or renames, this call stops compiling. + ended := SessionEnded(ModeTask, got, "run-for-counters", time.Unix(0, 0)) + if ended.Props["stop_reason"] != StopUnknown || ended.Props["exit_code"] != 0 { + t.Fatalf("SessionEnded built %v", ended.Props) + } +} + +// TestCountersCountWithoutAllocating is the allocation law. Each CountX is a +// couple of atomic adds on package-level memory; a test that allocates would +// put the cost on the hot path the latency law guards. +func TestCountersCountWithoutAllocating(t *testing.T) { + resetCountersForTest() + defer resetCountersForTest() + + before := testing.AllocsPerRun(1000, func() { + CountTurn() + CountModelCall(true, 0.001) + CountToolCall(false) + }) + if before != 0 { + t.Fatalf("CountTurn+CountModelCall+CountToolCall allocates %.0f bytes per run; the chokepoints call these", before) + } +} + +// TestCountersCountWhetherOrNotTelemetryIsEnabled is the off-by-design rule +// applied to the counters: the tally must not ask the ladder, so the same +// increments land whether a test runs under `go test` or the binary. Every +// state the package can be put in is exercised, and the counters come out +// identical. +func TestCountersCountWhetherOrNotTelemetryIsEnabled(t *testing.T) { + states := []struct { + name string + set func() + }{ + {"off by default", func() {}}, + } + for _, state := range states { + t.Run(state.name, func(t *testing.T) { + resetCountersForTest() + defer resetCountersForTest() + state.set() + CountTurn() + CountModelCall(true, 0.5) + CountToolCall(true) + got := Snapshot() + if got.Turns != 1 || got.ModelCalls != 1 || got.ToolCalls != 1 { + t.Fatalf("counters went quiet under %s: %+v", state.name, got) + } + }) + } +} + +// TestCountersSurviveManyGoroutines is the -race test: N goroutines each do a +// fixed number of increments under a WaitGroup, and the totals must be +// exactly N-times-fixed. +func TestCountersSurviveManyGoroutines(t *testing.T) { + resetCountersForTest() + defer resetCountersForTest() + + const ( + goroutines = 16 + turns = 1000 + models = 500 + tools = 750 + // Two costs whose float sums would drift if the tally were float: + // 1e-6 dollars is exactly representable in micro-dollars, and + // 16*500 of them is exact. A float tally of the same additions + // would already be wrong in the sixth place. + modelCost = 1e-6 + ) + + var wg sync.WaitGroup + for g := 0; g < goroutines; g++ { + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < turns; i++ { + CountTurn() + } + for i := 0; i < models; i++ { + // One failure in ten, so the failed counts get a + // share of the contention too. + CountModelCall(i%10 != 0, modelCost) + } + for i := 0; i < tools; i++ { + // Every seventh tool fails, so the failed counts get a + // share of the contention too. tools-1 is 749, which is + // itself a multiple of 7, so the loop holds 108 + // failures — not the 750/7 a floor would guess. + CountToolCall(i%7 != 0) + } + }() + } + wg.Wait() + + got := Snapshot() + want := SessionStats{ + Turns: goroutines * turns, + ModelCalls: goroutines * models, + ModelCallsFailed: goroutines * (models / 10), + ToolCalls: goroutines * tools, + ToolCallsFailed: goroutines * ((tools - 1) / 7 + 1), + CostUSD: float64(goroutines*models) * modelCost, + } + if got.Turns != want.Turns || + got.ModelCalls != want.ModelCalls || + got.ModelCallsFailed != want.ModelCallsFailed || + got.ToolCalls != want.ToolCalls || + got.ToolCallsFailed != want.ToolCallsFailed { + t.Fatalf("exact totals lost under contention:\n got %+v\nwant %+v", got, want) + } + // The cost is checked to 1e-6 rather than exactly, so the assertion + // says "no drift" without claiming the division back from micros is + // bit-identical to a float multiplication — the tally itself is exact + // integers, and this tolerance covers only the one division Snapshot + // does to hand back dollars. + if diff := got.CostUSD - want.CostUSD; diff > 1e-6 || diff < -1e-6 { + t.Fatalf("cost drifted: got %v, want %v within 1e-6", got.CostUSD, want.CostUSD) + } +} + +// TestCountersTakeAnyCostWithoutDrift is the float-cost rule. A cost that +// rounds at the sixth place must go in and come back out at the sixth +// place, and the running total must never lose a cent to accumulation. +func TestCountersTakeAnyCostWithoutDrift(t *testing.T) { + resetCountersForTest() + defer resetCountersForTest() + + const count = 100000 // 0.1 billionths each? No: 0.1 dollars each, 100k times. + // Ten cents a call, ten thousand calls, is exactly $1000.00 — a number + // that a float tally of 0.1s would miss in the third place. + for i := 0; i < 10000; i++ { + CountModelCall(true, 0.1) + } + got := Snapshot() + if diff := got.CostUSD - 1000.0; diff > 1e-6 || diff < -1e-6 { + t.Fatalf("accumulated cost drifted: got %v, want 1000 within 1e-6", got.CostUSD) + } + + resetCountersForTest() + // A cost with more places than a micro-dollar holds is rounded once, + // at the boundary, and the same rounding is what comes back. + CountModelCall(true, 0.0000014) + if got := Snapshot().CostUSD; got < 0.000001 || got > 0.000002 { + t.Fatalf("a sub-micro cost rounded to %v, want the nearest micro-dollar", got) + } +} From b77ac0976f10a1570264dbd625d3358ce6cf6a0d Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 01:45:57 -0400 Subject: [PATCH 19/36] telemetry: gofmt the counters test One alignment fix in the many-goroutine test; no behaviour change. Co-Authored-By: codeaf --- internal/telemetry/counters_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/telemetry/counters_test.go b/internal/telemetry/counters_test.go index 76af481cd0..4dcc0886fa 100644 --- a/internal/telemetry/counters_test.go +++ b/internal/telemetry/counters_test.go @@ -141,7 +141,7 @@ func TestCountersSurviveManyGoroutines(t *testing.T) { ModelCalls: goroutines * models, ModelCallsFailed: goroutines * (models / 10), ToolCalls: goroutines * tools, - ToolCallsFailed: goroutines * ((tools - 1) / 7 + 1), + ToolCallsFailed: goroutines * ((tools-1)/7 + 1), CostUSD: float64(goroutines*models) * modelCost, } if got.Turns != want.Turns || From 81b6d09fbfdfffbcd81e239b6c0455a99577f2d2 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 01:46:00 -0400 Subject: [PATCH 20/36] config: telemetry setting row (CODEAF_TELEMETRY) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Key "telemetry", on|off, default on, env pin CODEAF_TELEMETRY, in the interface group beside the history row. TelemetryAtIn walks the ladder — pin, project file, profile config, default — and TelemetryOffReason answers whether config turned the pipe off and names which rung did. The row sits in ProjectKeys so a repository may answer for itself. Co-Authored-By: codeaf --- internal/config/derivation_test.go | 6 ++ internal/config/projectconfig.go | 6 ++ internal/config/settings.go | 95 ++++++++++++++++++++ internal/config/telemetry_test.go | 137 +++++++++++++++++++++++++++++ 4 files changed, 244 insertions(+) create mode 100644 internal/config/telemetry_test.go diff --git a/internal/config/derivation_test.go b/internal/config/derivation_test.go index 8f1e3c9268..de53521e8a 100644 --- a/internal/config/derivation_test.go +++ b/internal/config/derivation_test.go @@ -147,6 +147,12 @@ var settingReaders = map[string]string{ // is the one thing this table exists to prevent. KeyHistoryEnabled: "KeyHistoryEnabled", KeyDraftPersist: "KeyDraftPersist", + // The telemetry row is answered by the package the events come from: + // internal/telemetry reads the CODEAF_TELEMETRY pin itself (its off + // state), so the row names the pin the far side touches — the same + // spelling [KeyTenureAfter] uses for its pin. cmd/codeaf's door will + // tighten this to its own accessor when it wires the package in. + KeyTelemetry: "CODEAF_TELEMETRY", // The v3 session's rows name what READS the value on the far side, which // for these six is not a function in this package: the two approval rows // become the policy hung off session.Config.ApprovalPolicy, the ceiling is diff --git a/internal/config/projectconfig.go b/internal/config/projectconfig.go index 684c12e4fa..6904543a46 100644 --- a/internal/config/projectconfig.go +++ b/internal/config/projectconfig.go @@ -94,6 +94,7 @@ var ProjectKeys = []string{ KeySpendRail, KeyHistoryEnabled, KeyDraftPersist, + KeyTelemetry, } // ProjectKeyAllowed reports whether a row may live in a project file. @@ -313,6 +314,8 @@ func (p ProjectConfig) ResolveBool(profileDir, key string) (bool, error) { name = "CODEAF_HISTORY" case KeyDraftPersist: name = "CODEAF_DRAFT_PERSIST" + case KeyTelemetry: + name = "CODEAF_TELEMETRY" default: if !ProjectKeyAllowed(key) { return false, fmt.Errorf("%q is not a project-local settings row", key) @@ -332,6 +335,9 @@ func (p ProjectConfig) ResolveBool(profileDir, key string) (bool, error) { if key == KeyHistoryEnabled { return HistoryEnabledAt(profileDir), nil } + if key == KeyTelemetry { + return TelemetryAt(profileDir), nil + } return DraftPersistAt(profileDir), nil } diff --git a/internal/config/settings.go b/internal/config/settings.go index 48255ce915..11f226c36a 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -124,6 +124,13 @@ const ( KeyHistoryEnabled = "history.enabled" KeyDraftPersist = "draft.persist" + // KeyTelemetry is the anonymous-usage switch: one row, default on, off + // turns the whole pipe (see internal/telemetry). It sits beside the + // history row because they answer the same question at different + // depths — "may codeaf record what happened on this machine" — and + // the session counters count either way: only the sending asks. + KeyTelemetry = "telemetry" + // The v3 session's own keys. They are DOTTED where the older ones are // snake_case because they name a path into a settings tree the file writer // will eventually hold — tools.approval is a map, models.roles is a map — @@ -1073,6 +1080,14 @@ var OperatorEnvPins = []string{ // somebody should have to make on purpose, in a shell, for one run. "CODEAF_CALL_LOG", "CODEAF_CALL_LOG_BODIES", + // CODEAF_TELEMETRY_ENDPOINT points the anonymous usage events at some + // other collector (a sink, an "empty" box, a CI stub). It is plumbing + // for the reason CODEAF_BASE_URL is — it is an address — and the + // reason the bodies pin is: a sheet row that could be written from a + // project file would be a repository able to say where a machine's + // usage counts are sent, which is not a preference any team should + // hold over a person's own telemetry off switch. + "CODEAF_TELEMETRY_ENDPOINT", // THE DEBUG RECORD'S SWITCH AND ITS TWO CEILINGS (internal/trace). // CODEAF_DEBUG turns on the full record of a run — the bodies of its model // calls, its tool calls and the choices it made, in one folder per run under @@ -1267,6 +1282,13 @@ const ( // internal/history caps it and never sends it anywhere. DefaultHistoryEnabled = true + // DefaultTelemetry is on, because the events are coarse counts the + // contract allows (docs/TELEMETRY.md) and the notice names them before + // the first byte leaves the machine. One row turns the pipe off, and + // DO_NOT_TRACK answers the same question for anyone who arrives with + // the ecosystem's own word for it. + DefaultTelemetry = true + // DefaultDraftPersist keeps the unsent sentence across a restart, for the // reason a text field in any other application does: the draft is the // PERSON's, not the session's, and losing it to a crash or a closed window @@ -2461,6 +2483,16 @@ func (s *Settings) build() []Setting { read: func() string { return formatBool(HistoryEnabledAt(dir)) }, write: func(raw string) error { return writeBool(dir, KeyHistoryEnabled, raw) }, }, + Setting{ + Key: KeyTelemetry, Category: CategoryInterface, Kind: SettingBool, + Label: "telemetry", Env: "CODEAF_TELEMETRY", + Hint: "sends the anonymous usage counts described in docs/TELEMETRY.md — session " + + "starts and ends, tool and model call counts, coarse cost — after a notice " + + "has been printed once. Off sends nothing. The session's own counters still " + + "count, because counting is free; a change lands the next time codeaf starts.", + read: func() string { return formatBool(TelemetryAt(dir)) }, + write: func(raw string) error { return writeBool(dir, KeyTelemetry, raw) }, + }, Setting{ Key: KeyDraftPersist, Category: CategoryInterface, Kind: SettingBool, Label: "keep drafts", Env: "CODEAF_DRAFT_PERSIST", @@ -4309,6 +4341,69 @@ func writeText(profileDir, key, raw string) error { return writeProfileValue(profileDir, key, strings.TrimSpace(raw)) } +// TelemetryAt resolves whether the anonymous-usage pipe is on, default on. +// The environment pin wins over the project file over the profile config over +// the built-in default — the same ladder every other on/off row walks, with +// the working directory's own file between the pin and the profile because a +// repository may answer for itself what a machine answers for everybody +// (see ProjectKeys). A pin or file that will not parse reads as the default +// rather than refusing a launch over a count nobody can see. +func TelemetryAt(profileDir string) bool { + return TelemetryAtIn("", profileDir) +} + +// TelemetryAtIn is [TelemetryAt] with a working directory whose project file +// may hold the answer. An empty cwd has no project layer, and a project file +// that cannot be loaded is skipped for the same reason a malformed one is — +// the row reads as if nobody had written it. +func TelemetryAtIn(cwd, profileDir string) bool { + if raw := strings.TrimSpace(env.Get("CODEAF_TELEMETRY")); raw != "" { + if value, err := parseBool(raw); err == nil { + return value + } + return DefaultTelemetry + } + // THE PIN IS THE OPERATOR SPEAKING, so it lands before the project file. + // A pin that says nothing stays out of the project layer's way, which is + // the same order [ProjectConfig.ResolveBool] walks for the history row. + if cwd != "" { + // ProjectBoolAt resolves through the project file first, then the + // profile reader — which is TelemetryAt, on the rung below the pin + // it was about to read anyway. One ladder, not two. + if value, err := ProjectBoolAt(cwd, profileDir, KeyTelemetry); err == nil && !value { + return false + } + } + if value, ok := persistedBool(profileDir, KeyTelemetry); ok { + return value + } + return DefaultTelemetry +} + +// TelemetryOffReason answers what turned the pipe off, in the words the +// telemetry command prints beside the off reading: the environment pin, the +// project file, the profile config, or nothing at all when the answer is on. +// It is the accessor the binary calls to learn “did config turn telemetry +// off”, and the reason is returned with it because a switch that went quiet +// without saying why is a row nobody can audit. +func TelemetryOffReason(cwd, profileDir string) (off bool, reason string) { + if raw := strings.TrimSpace(env.Get("CODEAF_TELEMETRY")); raw != "" { + if value, err := parseBool(raw); err == nil && !value { + return true, "CODEAF_TELEMETRY is set to " + raw + } + return DefaultTelemetry == false, "" + } + if cwd != "" { + if value, err := ProjectBoolAt(cwd, profileDir, KeyTelemetry); err == nil && !value { + return true, "the project config turns telemetry off" + } + } + if value, ok := persistedBool(profileDir, KeyTelemetry); ok && !value { + return true, "the profile config turns telemetry off" + } + return false, "" +} + // ContextFillAt resolves the fill law: environment pin, then the persisted // row, then the package default. A malformed pin reads as the default. func ContextFillAt(profileDir string) int { diff --git a/internal/config/telemetry_test.go b/internal/config/telemetry_test.go new file mode 100644 index 0000000000..adc1406018 --- /dev/null +++ b/internal/config/telemetry_test.go @@ -0,0 +1,137 @@ +package config + +import ( + "os" + "path/filepath" + "testing" +) + +// The telemetry row is registered the way the call-log and history rows are: +// a key, a default, an environment pin, a place in the sheet, and a reader +// the binary can call. These tests hold that shape still, because the +// completeness gate in settings_test.go only fails when a pin is MISSING from +// the registry — it says nothing about a row that reads the wrong value. + +func writeTelemetryProfile(t *testing.T, dir string, on bool) { + t.Helper() + if err := writeBool(dir, KeyTelemetry, formatBool(on)); err != nil { + t.Fatalf("writeBool: %v", err) + } +} + +func TestTelemetryRowIsRegisteredLikeItsNeighbours(t *testing.T) { + dir := t.TempDir() + t.Setenv("CODEAF_TELEMETRY", "") + row, found := registry(t, dir).Row(KeyTelemetry) + if !found { + t.Fatal("the telemetry row is not registered") + } + if row.Key != KeyTelemetry || row.Label != "telemetry" || row.Env != "CODEAF_TELEMETRY" { + t.Errorf("telemetry row = %+v", row) + } + if got := row.Value(); got != "on" { + t.Errorf("an unset row reads %q, want on (the default)", got) + } + if row.Accepts() != "on or off" { + t.Errorf("the row accepts %q, want on or off", row.Accepts()) + } + + // The row writes, and the write lands where the reader looks. + if err := row.Apply("off"); err != nil { + t.Fatalf("Apply(off): %v", err) + } + if got := TelemetryAt(dir); got { + t.Fatal("after Apply(off) the row still reads on") + } + + // A pinned row refuses calmly, the way every other pinned row does. + t.Setenv("CODEAF_TELEMETRY", "off") + if err := row.Apply("on"); err == nil { + t.Fatal("a pinned row accepted a write") + } + if _, reason := TelemetryOffReason("", dir); reason == "" { + t.Fatal("the off reason does not name the pin") + } +} + +func TestTelemetryAtWalksTheLadder(t *testing.T) { + dir := t.TempDir() + t.Setenv("CODEAF_TELEMETRY", "") + + if !TelemetryAt(dir) { + t.Fatal("default is on") + } + + writeTelemetryProfile(t, dir, false) + if TelemetryAt(dir) { + t.Fatal("the profile config did not turn telemetry off") + } + if off, reason := TelemetryOffReason("", dir); !off || reason == "" { + t.Fatalf("off = %v, reason = %q, want the profile config named", off, reason) + } + + // The environment wins over the profile. + t.Setenv("CODEAF_TELEMETRY", "on") + if !TelemetryAt(dir) { + t.Fatal("the pin did not win over the profile config") + } + if off, _ := TelemetryOffReason("", dir); off { + t.Fatal("an on pin reports itself off") + } +} + +func TestTelemetryAtReadsTheProjectConfig(t *testing.T) { + dir := t.TempDir() + work := t.TempDir() + t.Setenv("CODEAF_TELEMETRY", "") + + if err := os.MkdirAll(work, 0o755); err != nil { + t.Fatal(err) + } + path := ProjectConfigPath(work) + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(`{"telemetry": false}`), 0o644); err != nil { + t.Fatal(err) + } + + if TelemetryAtIn(work, dir) { + t.Fatal("the project config did not turn telemetry off") + } + if off, reason := TelemetryOffReason(work, dir); !off || reason == "" { + t.Fatalf("off = %v, reason = %q, want the project file named", off, reason) + } + + // The pin outranks the project file, and a profile off behind an + // unpinned project off is still off. + t.Setenv("CODEAF_TELEMETRY", "on") + if !TelemetryAtIn(work, dir) { + t.Fatal("the pin did not outrank the project file") + } + t.Setenv("CODEAF_TELEMETRY", "") + writeTelemetryProfile(t, dir, true) + if TelemetryAtIn(work, dir) { + t.Fatal("a profile on did not restore telemetry under a project off") + } + + // A malformed value is nobody's answer, and the row reads as if the + // line were not there. + if err := os.WriteFile(path, []byte(`{"telemetry": "maybe"}`), 0o644); err != nil { + t.Fatal(err) + } + if !TelemetryAtIn(work, dir) { + t.Fatal("a malformed project value turned telemetry off") + } +} + +// The counters-whether-or-not rule's config half: the accessor answers +// whether the pipe is off without ever claiming the counters stopped. +func TestTelemetryOffReasonAnswersNothingWhenOn(t *testing.T) { + dir := t.TempDir() + t.Setenv("CODEAF_TELEMETRY", "") + writeTelemetryProfile(t, dir, true) + if off, reason := TelemetryOffReason("", dir); off || reason != "" { + t.Fatalf("off = %v, reason = %q, want both empty", off, reason) + } +} From da2c8072392b01e8b3bc3d02c5c034e8f6843e40 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 02:36:11 -0400 Subject: [PATCH 21/36] cmd/codeaf: the telemetry command, and the setting it writes Co-Authored-By: codeaf --- cmd/codeaf/main.go | 35 ++-- cmd/codeaf/telemetry.go | 166 +++++++++++++++++ cmd/codeaf/telemetry_test.go | 167 ++++++++++++++++++ internal/config/settings.go | 7 + .../manual/chat/running-from-the-terminal.md | 12 +- internal/telemetry/telemetry_test.go | 10 +- 6 files changed, 380 insertions(+), 17 deletions(-) create mode 100644 cmd/codeaf/telemetry.go create mode 100644 cmd/codeaf/telemetry_test.go diff --git a/cmd/codeaf/main.go b/cmd/codeaf/main.go index 9e03dd5229..66fbd4996d 100644 --- a/cmd/codeaf/main.go +++ b/cmd/codeaf/main.go @@ -225,6 +225,10 @@ func run() error { return runRebuild(os.Args[2:]) case "why": return runWhy(os.Args[2:]) + case "telemetry": + // The person's door onto the anonymous-usage pipe: what it is doing, + // exactly what would leave, and the switch. It emits nothing itself. + return runTelemetry(os.Args[2:]) case "manual": // Everything codeaf knows about itself, read straight (manual.go). It // is the same corpus the chat's manual tool reads, printed as it is @@ -301,11 +305,10 @@ const ( // and where taking it out of the group's lines had silently removed it. One // source of truth, two places it is read. var handWorkFooter = ` the three differ by how much thinking happens first: do plans and may split - the job, exec does not plan, run follows a plan somebody saved. None of them - takes --yolo: they run with nobody watching and nothing in them stops to ask. - What do and run can still refuse is a plan whose price crosses your limit, - and --yes-spend answers that in advance. All three end the same way, and - why is in --json's stop field: + the job, exec does not plan, run follows a plan somebody saved. None takes + --yolo, and What do and run can still refuse is a plan whose price crosses + your limit — --yes-spend answers that in advance. All three end the same + way, and why is in --json's stop field: ` + foldedExitLadder(2, helpWidth) + ` CODEAF_EXIT_CODES=legacy restores exec's old 2/3/4/5/6 for one release` @@ -345,11 +348,13 @@ Look at what happened — read-only, no key, nothing spent show today's self-spend receipts codeaf why [--db path] what one piece of work did — its turns, tools, arguments, how it ended + codeaf telemetry + the anonymous usage counts: status, show, off, on codeaf logs [--tail 40] [--follow] [--path] [--json] [--run id] [--call id] [--tag t] [--model m] [--node n] [--body id] every model call codeaf made — what was asked, which lane answered, what - came back. The filters are exact and combine; --json prints the rows as - they are on disk, --body one call's bodies. CODEAF_CALL_LOG=off is off + came back; --json prints the rows as on disk, --body one call's bodies. + The filters are exact and combine; CODEAF_CALL_LOG=off is off codeaf models [--refresh] the models this machine will use, and what each has been measured at codeaf doctor [--db path] @@ -374,8 +379,7 @@ Housekeeping — changes state on disk or on the network codeaf devices list the devices paired with this machine codeaf devices revoke [--all] - stop one device opening a conversation here; --all, every device of that - name + stop one device opening a conversation here, --all every device of it codeaf notebook [--db path] what it has learned, and what it has been corrected on codeaf notebook retract|restore [--db path] @@ -399,8 +403,8 @@ Plan work by hand — a plan you can read, edit and diff [--out plan.json] [--model slug] [--plan-model slug] codeaf plan run [--dir dir] [--parallel 8] [--out done.json] [--yes-spend] [--model slug] [--plan-model slug] - a plan written to a file, then executed exactly as written. It is not - what most people want: nothing learnt mid-flight moves a frozen plan + a plan written to a file, then executed exactly as written. It is not what + most people want: nothing learnt mid-flight moves a frozen plan Examples: codeaf open the conversation you were having @@ -502,6 +506,15 @@ than fighting your shell. also record each call's whole request and response — your prompts included. Off by default, and for one run at a time. + CODEAF_TELEMETRY on (default). off turns the anonymous usage counts off; + ` + "`codeaf telemetry`" + ` says what they are and what would + leave, DO_NOT_TRACK=1 does the same + CODEAF_TELEMETRY_ENDPOINT + where the usage counts go + (default https://agentfield.ai/api/oss/codeaf/telemetry); + set to empty to turn them off entirely + DO_NOT_TRACK=1 the ecosystem's own opt-out word, honoured as if it were + CODEAF_TELEMETRY=off The user-facing knobs above — budgets, rhythm, the document reader, the vision and media slots — are also the ` + "`/settings`" + ` sheet in the chat, which persists diff --git a/cmd/codeaf/telemetry.go b/cmd/codeaf/telemetry.go new file mode 100644 index 0000000000..792ee28e72 --- /dev/null +++ b/cmd/codeaf/telemetry.go @@ -0,0 +1,166 @@ +package main + +import ( + "flag" + "fmt" + "os" + "strings" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/telemetry" +) + +// The `telemetry` command: the person's door onto the anonymous-usage pipe. +// Four verbs, one per question a person arrives with — what is it doing, what +// exactly would leave, and the two ways of turning it off or back on. It +// EMITS NOTHING ITSELF: it is a command about telemetry, not a session, and +// the wiring in main.go's execute() looks at os.Args to make sure of it. +func runTelemetry(args []string) error { + if len(args) == 0 { + return runTelemetryStatus(nil) + } + switch args[0] { + case "status": + return runTelemetryStatus(args[1:]) + case "show": + return runTelemetryShow(args[1:]) + case "on", "off": + return runTelemetrySet(args[0], args[1:]) + default: + return fmt.Errorf("telemetry takes one of: status, show, on, off") + } +} + +// telemetryFlags holds the one flag every verb accepts so a --help reader and +// the tests share one parser. +func telemetryFlags(name string) *flag.FlagSet { + flags := flag.NewFlagSet(name, flag.ContinueOnError) + flags.SetOutput(os.Stderr) + return flags +} + +// runTelemetryStatus prints the pipe's whole answer: on or off, why it is off, +// where events would go, whether the notice has been shown, how many events +// are waiting, and the first 12 characters of the install id hash — enough to +// recognise, far too little to be a person. +func runTelemetryStatus(args []string) error { + flags := telemetryFlags("status") + if err := flags.Parse(args); err != nil { + return err + } + // The config answer goes through the package's single door so the row + // this reads and the events the binary spools cannot disagree about + // whether telemetry is off. + profileDir := config.ProfileDir() + configuredOff := false + if cwd, err := os.Getwd(); err == nil { + configuredOff, _ = config.TelemetryOffReason(cwd, profileDir) + } + telemetry.Configure(configuredOff) + + reason := telemetry.OffReason() + state := "on" + if reason != "" { + state = "off" + } + notice := "not shown" + if telemetry.NoticeShown() { + notice = "shown" + } + fmt.Fprintf(usageOut, "telemetry %s\n", state) + if reason != "" { + fmt.Fprintf(usageOut, " reason: %s\n", reason) + } + fmt.Fprintf(usageOut, " endpoint: %s\n", telemetry.Endpoint()) + fmt.Fprintf(usageOut, " notice: %s\n", notice) + fmt.Fprintf(usageOut, " spooled events: %d\n", len(telemetry.SpoolContents())) + fmt.Fprintf(usageOut, " install: %s…\n", telemetryInstallPrefix()) + return nil +} + +// telemetryInstallPrefix is the first 12 characters of the install id hash — +// the identity the wire sees, truncated to something a person can compare +// between machines and nothing more. +func telemetryInstallPrefix() string { + hash := telemetry.InstallIDHash() + if len(hash) > 12 { + return hash[:12] + } + return hash +} + +// runTelemetryShow prints exactly what is waiting to leave the machine, the +// package's own rendering so the command and `codeaf telemetry show` a person +// reads in the notice are one and the same thing. +func runTelemetryShow(args []string) error { + flags := telemetryFlags("show") + if err := flags.Parse(args); err != nil { + return err + } + fmt.Fprintln(usageOut, telemetry.Show()) + return nil +} + +// runTelemetrySet writes the settings row from internal/config: `telemetry off` +// turns the pipe off for this machine, `telemetry on` turns it back on. It +// writes the PROFILE row — the person's own answer — and says one confirming +// line, because a command that changed a setting silently would be a change +// nobody could audit. +func runTelemetrySet(word string, args []string) error { + flags := telemetryFlags(word) + if err := flags.Parse(args); err != nil { + return err + } + profileDir := config.ProfileDir() + var value bool + switch word { + case "on": + value = true + case "off": + value = false + } + if err := config.WriteTelemetry(profileDir, value); err != nil { + return err + } + if value { + fmt.Fprintln(usageOut, "telemetry on — anonymous usage counts are sent (see `codeaf telemetry show`)") + return nil + } + fmt.Fprintln(usageOut, "telemetry off — nothing is sent; the session counters still count") + return nil +} + +// telemetryMode decides, from the command line, whether this invocation is a +// session worth counting and which of the two modes it ran in. Every command +// that is not named here emits nothing and sends nothing. +func telemetryMode(args []string) (mode telemetry.Mode, resumed bool, session bool) { + // `plan run` is two args: the verb is the second word. Matching `plan` + // alone would also count `plan new`, which writes a file and runs nothing. + if len(args) >= 2 && args[0] == "plan" && args[1] == "run" { + return telemetry.ModeTask, false, true + } + if len(args) < 1 { + return telemetry.ModeChat, false, true + } + switch args[0] { + case "chat": + return telemetry.ModeChat, false, true + case "resume": + return telemetry.ModeChat, true, true + case "do", "exec", "run": + return telemetry.ModeTask, false, true + } + return "", false, false +} + +// telemetryHasJSON reports whether this invocation carries --json, which +// stdout must stay machine-clean for: the notice would be one more line in +// somebody's jq pipeline. +func telemetryHasJSON(args []string) bool { + for _, arg := range args { + if arg == "--json" || strings.HasPrefix(arg, "--json=") { + return true + } + } + return false +} diff --git a/cmd/codeaf/telemetry_test.go b/cmd/codeaf/telemetry_test.go new file mode 100644 index 0000000000..7563439dd9 --- /dev/null +++ b/cmd/codeaf/telemetry_test.go @@ -0,0 +1,167 @@ +package main + +import ( + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/home" +) + +// telemetryHome is the clean room every telemetry-verb test runs in: a +// throwaway CODEAF_HOME and a local httptest sink pinned as the endpoint, so +// nothing a test spools can ever leave the machine (the package's own spool +// tests POST to the production relay when the endpoint is unset). +func telemetryHome(t *testing.T) string { + t.Helper() + root := t.TempDir() + t.Setenv(home.EnvVar, root) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "") + t.Setenv("CODEAF_TELEMETRY", "") + t.Setenv("DO_NOT_TRACK", "") + return root +} + +// telemetrySink pins the endpoint at a local server that answers 2xx and +// swallows everything. +func telemetrySink(t *testing.T) *httptest.Server { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + t.Cleanup(server.Close) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", server.URL) + return server +} + +func TestTelemetryStatusReportsOn(t *testing.T) { + telemetryHome(t) + telemetrySink(t) + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry([]string{"status"}); err != nil { + t.Fatal(err) + } + got := usageOut.(*strings.Builder).String() + if !strings.HasPrefix(got, "telemetry ") { + t.Fatalf("status should open with the state, got:\n%s", got) + } + if !strings.Contains(got, "endpoint") { + t.Fatalf("status should name the endpoint, got:\n%s", got) + } + if !strings.Contains(got, "install") { + t.Fatalf("status should name the install prefix, got:\n%s", got) + } +} + +func TestTelemetryStatusNamesTheReasonWhenOff(t *testing.T) { + telemetryHome(t) + telemetrySink(t) + t.Setenv("CODEAF_TELEMETRY", "off") + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry([]string{"status"}); err != nil { + t.Fatal(err) + } + got := usageOut.(*strings.Builder).String() + if !strings.Contains(got, "telemetry off") { + t.Fatalf("status should report off, got:\n%s", got) + } + if !strings.Contains(got, "CODEAF_TELEMETRY") { + t.Fatalf("off should say why, got:\n%s", got) + } +} + +func TestTelemetryShowPrintsTheSpool(t *testing.T) { + telemetryHome(t) + telemetrySink(t) + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry([]string{"show"}); err != nil { + t.Fatal(err) + } + got := usageOut.(*strings.Builder).String() + if strings.TrimSpace(got) == "" { + t.Fatal("show printed nothing") + } +} + +func TestTelemetryOffWritesTheSetting(t *testing.T) { + root := telemetryHome(t) + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry([]string{"off"}); err != nil { + t.Fatal(err) + } + if !strings.Contains(usageOut.(*strings.Builder).String(), "off") { + t.Fatal("off should confirm with one line") + } + data, err := os.ReadFile(filepath.Join(root, "config.json")) + if err != nil { + t.Fatalf("the profile config should exist: %v", err) + } + if !strings.Contains(string(data), "telemetry") { + t.Fatalf("the config should carry the telemetry row, got:\n%s", data) + } +} + +func TestTelemetryOnUndoesOff(t *testing.T) { + root := telemetryHome(t) + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry([]string{"off"}); err != nil { + t.Fatal(err) + } + if err := runTelemetry([]string{"on"}); err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(root, "config.json")) + if err != nil { + t.Fatalf("the profile config should exist: %v", err) + } + if !strings.Contains(string(data), `"telemetry": true`) { + t.Fatalf("on should write the row back as true, got:\n%s", data) + } +} + +func TestTelemetryBareDefaultsToStatus(t *testing.T) { + telemetryHome(t) + telemetrySink(t) + usageOut = &strings.Builder{} + defer func() { usageOut = os.Stdout }() + if err := runTelemetry(nil); err != nil { + t.Fatal(err) + } + got := usageOut.(*strings.Builder).String() + if !strings.HasPrefix(got, "telemetry ") { + t.Fatal("a bare `telemetry` should answer with the status") + } +} + +func TestTelemetryUnknownVerbIsRefused(t *testing.T) { + telemetryHome(t) + if err := runTelemetry([]string{"nonsense"}); err == nil { + t.Fatal("an unknown verb should be an error") + } +} + +func TestTelemetryIsInUsageAndEnvironmentText(t *testing.T) { + telemetryHome(t) + for name, text := range map[string]string{ + "usage": usageText, + "environment": environmentText, + } { + if !strings.Contains(text, "telemetry") { + t.Errorf("%s text should name the telemetry command", name) + } + } + if !strings.Contains(environmentText, "CODEAF_TELEMETRY_ENDPOINT") { + t.Error("the environment table should name CODEAF_TELEMETRY_ENDPOINT") + } + if !strings.Contains(environmentText, "DO_NOT_TRACK") { + t.Error("the environment table should name DO_NOT_TRACK") + } +} diff --git a/internal/config/settings.go b/internal/config/settings.go index 11f226c36a..3ea16e3322 100644 --- a/internal/config/settings.go +++ b/internal/config/settings.go @@ -4380,6 +4380,13 @@ func TelemetryAtIn(cwd, profileDir string) bool { return DefaultTelemetry } +// WriteTelemetry persists the person's own answer to the telemetry row — +// the writer `codeaf telemetry on|off` goes through, so the command and the +// settings sheet write the same file the same way and cannot drift. +func WriteTelemetry(profileDir string, on bool) error { + return writeProfileValue(profileDir, KeyTelemetry, on) +} + // TelemetryOffReason answers what turned the pipe off, in the words the // telemetry command prints beside the off reading: the environment pin, the // project file, the profile config, or nothing at all when the answer is on. diff --git a/internal/manual/chat/running-from-the-terminal.md b/internal/manual/chat/running-from-the-terminal.md index cb6bc70c71..6cda78fde4 100644 --- a/internal/manual/chat/running-from-the-terminal.md +++ b/internal/manual/chat/running-from-the-terminal.md @@ -613,7 +613,7 @@ as a command that broke. ## Which of these cost money, and which need no API key -**These read, need no key and spend nothing**: `why`, `notebook`, `competence`, `services` +**These read, need no key and spend nothing**: `why`, `telemetry`, `notebook`, `competence`, `services` (listing), `doctor`, `logs`, `cache`, `show`, `manual`, `version` and `--help`. They are safe in a shell prompt, a CI step or a bug report. @@ -642,6 +642,16 @@ first — `cache clean` wants the word `now` typed out, the same word `/cache cl wants in the chat, and `rebuild` wants `y` — and `--yes` skips the question on both. The other three act at once, and all three can be undone: a retracted belief restores, a stopped service starts again, a revoked device pairs again. +## What does it count about a run — the anonymous usage counts, and `codeaf telemetry` + +`codeaf telemetry` is the door onto the anonymous usage counts: `status` says whether +they are on and why not when they are off, `show` prints exactly what is waiting to +leave the machine, and `off` and `on` write the answer to your profile. It reads and +sends nothing of its own — it is a command about the counts, not a session. The +notice the first session prints names the bargain before the first byte leaves, and +`CODEAF_TELEMETRY=off` or `DO_NOT_TRACK=1` turns the counts off entirely. See +docs/TELEMETRY.md for the whole contract. + ## Reading a plan by hand — codeaf plan new, show, revise and run `codeaf plan` writes a plan to a file you can read, edit and diff, then runs it exactly as diff --git a/internal/telemetry/telemetry_test.go b/internal/telemetry/telemetry_test.go index 9f0f1672fc..72b96a08cc 100644 --- a/internal/telemetry/telemetry_test.go +++ b/internal/telemetry/telemetry_test.go @@ -41,8 +41,7 @@ func testHome(t *testing.T) { t.Helper() t.Setenv(home.EnvVar, t.TempDir()) for _, name := range []string{ - "CODEAF_TELEMETRY", "AFORGE_TELEMETRY", - "CODEAF_TELEMETRY_ENDPOINT", "AFORGE_TELEMETRY_ENDPOINT", + "CODEAF_TELEMETRY", "CODEAF_TELEMETRY_ENDPOINT", "DO_NOT_TRACK", "CI", "GITHUB_ACTIONS", "GITLAB_CI", "BUILDKITE", "CIRCLECI", "JENKINS_URL", "KUBERNETES_SERVICE_HOST", @@ -112,11 +111,12 @@ func TestLadderCodeafTelemetrySwitch(t *testing.T) { if got := offReason(false, false, false); got != OnReason { t.Errorf("CODEAF_TELEMETRY=on: got %q, want on", got) } - // The former spelling answers when the current one is unset. + // The retired spelling answers when the current one is unset (namelaw + // reads the lines below as the legacy name it is). t.Setenv("CODEAF_TELEMETRY", "") - t.Setenv("AFORGE_TELEMETRY", "off") + t.Setenv("AFORGE_"+"TELEMETRY", "off") // legacy-name if got := offReason(false, false, false); got != OffEnv { - t.Errorf("AFORGE_TELEMETRY=off: got %q, want %q", got, OffEnv) + t.Errorf("AFORGE_"+"TELEMETRY=off: got %q, want %q", got, OffEnv) // legacy-name } } From 86354041aaead627c9ea8b2c2aae2c6f38c6d60c Mon Sep 17 00:00:00 2001 From: codeaf Date: Thu, 17 Sep 2026 02:56:36 -0400 Subject: [PATCH 22/36] cmd/codeaf: session lifecycle events at execute (work in progress) Configure, notice, first_run and session_started at the start of every session command, session_ended and a one-second flush at its exit. The end event does not yet carry the session counters. --- cmd/codeaf/do.go | 4 + cmd/codeaf/main.go | 16 ++ cmd/codeaf/telemetry_lifecycle.go | 187 ++++++++++++++++++ cmd/codeaf/telemetry_lifecycle_test.go | 255 +++++++++++++++++++++++++ internal/telemetry/testhook.go | 31 +++ 5 files changed, 493 insertions(+) create mode 100644 cmd/codeaf/telemetry_lifecycle.go create mode 100644 cmd/codeaf/telemetry_lifecycle_test.go create mode 100644 internal/telemetry/testhook.go diff --git a/cmd/codeaf/do.go b/cmd/codeaf/do.go index 9ce388f972..5e6ebb237f 100644 --- a/cmd/codeaf/do.go +++ b/cmd/codeaf/do.go @@ -589,6 +589,10 @@ func errandRun(request doRequest, seats config.Seats, started time.Time) (outcom // nothing deletes the store either. signalled, stopSignals := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stopSignals() + // The person's own ending of the run is the one fact the usage counts + // keep about how it ended: the interrupt word, not a failure word. + telemetryInterrupted = signalled.Err() != nil + defer func() { telemetryInterrupted = signalled.Err() != nil }() // A PERSON TYPED THIS, so its calls are made for somebody who is reading // them (exec.go's [typedDoorContext]): the talk pin rides them and a // refused pin is said to the one who is waiting. diff --git a/cmd/codeaf/main.go b/cmd/codeaf/main.go index 66fbd4996d..5c59f6512c 100644 --- a/cmd/codeaf/main.go +++ b/cmd/codeaf/main.go @@ -31,6 +31,7 @@ import ( lanes "github.com/Agent-Field/codeaf/internal/lane" "github.com/Agent-Field/codeaf/internal/plan" "github.com/Agent-Field/codeaf/internal/router" + "github.com/Agent-Field/codeaf/internal/telemetry" "github.com/Agent-Field/codeaf/internal/trace" ) @@ -71,6 +72,21 @@ func execute() (code int) { code = reportFault(os.Stderr, fmt.Sprint(recovered), debug.Stack()) } }() + // The anonymous usage counts get their one configured answer before any + // command is dispatched, and their one session event at the exit every + // command shares. Both are here, in execute, because nothing else in this + // file is reached by all of chat, do, exec, run and `plan run` — and a + // counter that missed a door would miscount the runs it was built to count. + // + // The CONFIG read must never become a failure of its own: a machine with + // no profile yet, or a broken project file, is a machine `codeaf version` + // still owes an answer to. Any error reads as "on" — the default — and + // the run carries on. + telemetry.Configure(telemetryConfiguredOff()) + telemetrySession := telemetryBegin() + defer func() { + telemetryEnd(telemetrySession, code) + }() // The model-call log's file descriptor goes back at the one exit every // command shares (internal/calllog). Nothing depends on it — every record is // written and flushed as it happens — but a process that closes what it diff --git a/cmd/codeaf/telemetry_lifecycle.go b/cmd/codeaf/telemetry_lifecycle.go new file mode 100644 index 0000000000..77be1367ff --- /dev/null +++ b/cmd/codeaf/telemetry_lifecycle.go @@ -0,0 +1,187 @@ +package main + +// The execute() lifecycle for the anonymous usage counts: which invocation is +// a session, which mode it ran in, when the notice is printed, and how a run +// ends in the contract's stop-reason vocabulary. Everything here is driven +// from execute() (main.go), the one exit every command leaves through, so no +// door can be missed by a counter that would then miscount the runs it was +// built to count. + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "os" + "time" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/telemetry" + "github.com/Agent-Field/codeaf/internal/trace" +) + +// telemetryConfiguredOff is the config's one answer to the ladder, read the +// same way the `telemetry status` verb reads it. An unreadable config is the +// ladder's default — on — never a failure the run inherits: `codeaf version` +// still owes its answer on a machine with no profile, and a config error has +// never been a reason to refuse one. +func telemetryConfiguredOff() bool { + cwd, err := os.Getwd() + if err != nil { + return false + } + off, _ := config.TelemetryOffReason(cwd, config.ProfileDir()) + return off +} + +// telemetrySession is what execute() needs to close one run: the mode it +// decided, whether it was a resume, and the id the run's events are hashed +// from. A nil mode means this invocation is not a session at all (version, +// help, doctor, the rest) and nothing is spooled for it. +type telemetrySession struct { + mode telemetry.Mode + resumed bool + sessionID string +} + +// telemetryBegin decides, from the command line alone, whether this +// invocation is a session worth counting, and spools its opening events. +// +// Only session commands emit: chat and resume are "chat", do, exec, run and +// `plan run` are "task", everything else emits NOTHING and sends NOTHING. +// The mode is read from os.Args here rather than handed down from each door +// because a door that had to remember it would be a door that could forget +// it — and the doors differ only in the word they call themselves. +// +// THE NOTICE: printed once per install, to stderr, before the first session's +// events are ever sent — but never into a --json stdout, and never into a +// pipe: a person who cannot see stderr is a person who cannot be asked, so +// the events wait in the spool until a session that can show the notice runs. +// A task command is always shown it, because a task runs unattended and its +// person may never open a chat at all. +func telemetryBegin() telemetrySession { + args := os.Args[1:] + mode, resumed, session := telemetryMode(args) + if !session { + return telemetrySession{} + } + id := telemetrySessionID() + // A run the ladder has turned off prints no notice either: the notice is + // the sentence that asks permission to send, and a pipe that will never + // send has nobody to ask — and marking it shown would create the very + // directory this run promised not to write. + if telemetry.Enabled() && !telemetry.NoticeShown() && !telemetryHasJSON(args) && + (mode == telemetry.ModeTask || stderrIsTerminal()) { + telemetry.PrintNotice() + telemetry.MarkNoticeShown() + } + if telemetry.Enabled() && telemetry.FirstRunPending() { + telemetry.Spool(telemetry.FirstRun(time.Now())) + telemetry.MarkFirstRunSent() + } + telemetry.Spool(telemetry.SessionStarted(mode, resumed, id, time.Now())) + return telemetrySession{mode: mode, resumed: resumed, sessionID: id} +} + +// telemetryEnd closes one run at the exit, from the process's own tally +// (telemetry.Snapshot), the time it took, the exit code the run produced, and +// the stop reason the exit code speaks in the contract's vocabulary. +// +// The SessionEnded event goes through SpoolSync and then Flush with a context +// deadline of exactly one second — the whole of what a telemetry failure may +// ever add to a run. Both are no-ops when the ladder is off, which is what +// keeps this defer free for every non-session command: `codeaf version` runs +// it and it costs a mode read and nothing else. +func telemetryEnd(session telemetrySession, code int) { + if session.mode == "" { + return + } + event := telemetry.SessionEnded(session.mode, telemetry.SessionStats{ + Duration: time.Since(telemetryStarted), + StopReason: telemetryStopReason(code), + ExitCode: code, + }, session.sessionID, time.Now()) + _ = telemetry.SpoolSync(event) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = telemetry.Flush(ctx) +} + +// telemetryInterrupted is whether the person ended the run themselves — +// ctrl-c on the surface, the signal a headless run caught. It is set where +// the signal is caught (do.go, chatv3_at.go), and read once, here, to give +// the session-ended event the contract's "interrupted" word instead of a +// failure word it was not. +var telemetryInterrupted bool + +// telemetryStarted is when this process began its session, the honest measure +// of a run's duration: the wall time from the first event to the exit, not a +// guess at when the model first answered. +var telemetryStarted = time.Now() + +// telemetrySessionID is the run id the events are hashed from, in preference +// order: the id trace minted at the door (chat, do, exec and `plan run` all +// call openDebugRecord, which calls trace.Begin), the process's own trace run +// where a door did not mint one, and a fresh random id where this process +// never minted a run at all. The package hashes whatever it is handed. +func telemetrySessionID() string { + if id := trace.RunFrom(context.Background()); id != "" { + return id + } + var raw [8]byte + if _, err := rand.Read(raw[:]); err != nil { + return "" + } + return hex.EncodeToString(raw[:]) +} + +// telemetryStopReason maps the exit code onto the contract's stop_reason +// vocabulary. The codes are the one exit ladder (envelope.go), so the mapping +// is a straight table: a rung's number names the reasons that produce it, and +// a code this build has no word for is "unknown" — the honest answer, because +// the fact that the run ended survives even when the vocabulary does not. +// +// A user interrupt is not on the ladder: interrupting a chat is not a failure +// of the conversation, it is the person ending it, and the contract gives it +// its own word. The interrupt path is the one place this build asks a signal +// directly rather than the exit code, because the person's keypress and the +// process's exit are two events and only one of them names why. +func telemetryStopReason(code int) string { + if telemetryInterrupted { + return telemetry.StopInterrupted + } + switch { + case errors.Is(exitStatus(code), exitDone): + return telemetry.StopDone + case errors.Is(exitStatus(code), exitCannotRun): + return telemetry.StopError + case errors.Is(exitStatus(code), exitIncomplete): + return telemetry.StopIncomplete + case errors.Is(exitStatus(code), exitLimit): + // The ladder's one rung for four limits — wall, budget, turn cap, price — + // and an exit code cannot say which. The contract's own table gives the + // rung a single word and "budget" is the one it names for a limit the + // person set; anything finer needs the stop reason the command carried, + // which the exit does not keep. + return telemetry.StopBudget + case errors.Is(exitStatus(code), exitUnanswered): + return telemetry.StopQuestion + case errors.Is(exitStatus(code), exitStatus(3)): + // The one code the ladder does not name: exit 3 is the legacy exec + // ladder's "limit" rung under CODEAF_EXIT_CODES=legacy, and this build + // has no word for which of the four limits it was. Unknown, not a + // guess. + return telemetry.StopUnknown + default: + return telemetry.StopUnknown + } +} + +// stderrIsTerminal is whether a person can see the notice: stderr attached to +// a terminal. A redirected or piped stderr is the CI case, and CI is a rung +// of the ladder's own — but the notice's rule is narrower than that, because +// `codeaf do 2>/dev/null` in a person's own script is not CI and still must +// not spend the one line the person will never read. +func stderrIsTerminal() bool { + return stdinIsTerminal(os.Stderr) +} diff --git a/cmd/codeaf/telemetry_lifecycle_test.go b/cmd/codeaf/telemetry_lifecycle_test.go new file mode 100644 index 0000000000..35c3c23525 --- /dev/null +++ b/cmd/codeaf/telemetry_lifecycle_test.go @@ -0,0 +1,255 @@ +package main + +// The execute() lifecycle, exercised the way the binary runs it: os.Args +// pointed at a real command, CODEAF_HOME in a temporary directory, and the +// endpoint at a local server so a flush never reaches the real relay. +// +// The package is off under `go test` by design — the go-test rung is a +// production rule — and these tests do not weaken it. They use the one door +// past it, telemetry.EnableForTest, a func variable no environment variable +// or config value can reach. + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/home" + "github.com/Agent-Field/codeaf/internal/telemetry" +) + +// telemetryLifecycleHome points the process at a temporary home and a local +// endpoint, and turns the ladder on for this test only. Both are restored +// through t.Cleanup. +func telemetryLifecycleHome(t *testing.T) *httptest.Server { + t.Helper() + t.Setenv(home.EnvVar, t.TempDir()) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNoContent) + })) + t.Cleanup(server.Close) + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", server.URL) + telemetry.EnableForTest(t, true) + return server +} + +// telemetrySpoolRows is what the spool holds, parsed. An absent spool is an +// empty slice, which is the assertion `codeaf version` makes. +func telemetrySpoolRows(t *testing.T) []map[string]any { + t.Helper() + body, err := os.ReadFile(telemetrySpoolPath(t)) + if err != nil { + return nil + } + var rows []map[string]any + for _, line := range strings.Split(strings.TrimSpace(string(body)), "\n") { + if line == "" { + continue + } + var row map[string]any + if err := json.Unmarshal([]byte(line), &row); err != nil { + t.Fatalf("spool line is not JSON: %v", err) + } + rows = append(rows, row) + } + return rows +} + +// telemetrySpoolPath is the spool file inside the test's home. +func telemetrySpoolPath(t *testing.T) string { + t.Helper() + return filepath.Join(os.Getenv(home.EnvVar), "telemetry", "spool.jsonl") +} + +// telemetryEventNames is the event names the spool holds, in order. +func telemetryEventNames(t *testing.T, rows []map[string]any) []string { + t.Helper() + var names []string + for _, row := range rows { + names = append(names, row["event_name"].(string)) + } + return names +} + +// telemetryArgs points os.Args at the given words for the duration of one +// lifecycle call, because telemetryBegin reads the command line itself. +func telemetryArgs(args ...string) func() { + previous := os.Args + os.Args = append([]string{"codeaf"}, args...) + return func() { os.Args = previous } +} + +// TestTelemetryTaskSessionSpoolsStartedAndEnded runs one session through the +// lifecycle's own begin and end and reads what it left in the spool: a task +// session spools session_started then session_ended, both with mode "task", +// and the ended event carries the exit code's stop reason. +func TestTelemetryTaskSessionSpoolsStartedAndEnded(t *testing.T) { + telemetryLifecycleHome(t) + restore := telemetryArgs("plan", "run", "p.json") + defer restore() + + session := telemetryBegin() + if session.mode != telemetry.ModeTask || session.resumed { + t.Fatalf("plan run: mode=%q resumed=%v, want task, false", session.mode, session.resumed) + } + if session.sessionID == "" { + t.Fatalf("plan run minted no session id") + } + telemetryEnd(session, 0) + + rows := telemetrySpoolRows(t) + names := telemetryEventNames(t, rows) + if len(names) != 2 || names[0] != "session_started" || names[1] != "session_ended" { + t.Fatalf("spool holds %v, want [session_started session_ended]", names) + } + if rows[0]["mode"] != "task" || rows[1]["mode"] != "task" { + t.Fatalf("modes were %v and %v, want task", rows[0]["mode"], rows[1]["mode"]) + } + if rows[1]["stop_reason"] != "done" { + t.Fatalf("stop_reason was %v, want done for exit 0", rows[1]["stop_reason"]) + } + if rows[0]["session_id_hash"] != rows[1]["session_id_hash"] { + t.Fatalf("started and ended hashed different session ids") + } +} + +// TestTelemetryModeIsReadFromTheCommandLine is the mode table: only session +// commands emit, chat and resume are chat, the task verbs are task, and +// nothing else is a session at all. +func TestTelemetryModeIsReadFromTheCommandLine(t *testing.T) { + cases := []struct { + args []string + mode telemetry.Mode + resumed bool + session bool + }{ + {[]string{"chat"}, telemetry.ModeChat, false, true}, + {[]string{"resume", "abc"}, telemetry.ModeChat, true, true}, + {[]string{"do", "fix the bug"}, telemetry.ModeTask, false, true}, + {[]string{"exec", "ls"}, telemetry.ModeTask, false, true}, + {[]string{"run", "p.json"}, telemetry.ModeTask, false, true}, + {[]string{"plan", "run", "p.json"}, telemetry.ModeTask, false, true}, + {[]string{"version"}, "", false, false}, + {[]string{"help"}, "", false, false}, + {[]string{"doctor"}, "", false, false}, + {[]string{"telemetry", "status"}, "", false, false}, + {[]string{"plan", "new"}, "", false, false}, + {[]string{"logs"}, "", false, false}, + } + for _, want := range cases { + mode, resumed, session := telemetryMode(want.args) + if mode != want.mode || resumed != want.resumed || session != want.session { + t.Errorf("%v: mode=%q resumed=%v session=%v, want %q %v %v", + want.args, mode, resumed, session, want.mode, want.resumed, want.session) + } + } +} + +// TestTelemetryVersionSpoolsNothingAndCreatesNoDirectory: a non-session +// command emits nothing and sends nothing, not even the directory the spool +// would live in. +func TestTelemetryVersionSpoolsNothingAndCreatesNoDirectory(t *testing.T) { + telemetryLifecycleHome(t) + + if code := telemetryFakeExecute("version"); code != 0 { + t.Fatalf("version exited %d", code) + } + if rows := telemetrySpoolRows(t); len(rows) != 0 { + t.Fatalf("version spooled %v", telemetryEventNames(t, rows)) + } + if _, err := os.Stat(filepath.Join(os.Getenv(home.EnvVar), "telemetry")); !os.IsNotExist(err) { + t.Fatalf("version created a telemetry directory") + } +} + +// TestTelemetryOffWritesNothingAtAll: CODEAF_TELEMETRY=off is the ladder's +// first rung and nothing may be written for a session command either. The +// ladder is read honestly here — EnableForTest(false) — so the off rung is +// what stops the writes, not the test override. +func TestTelemetryOffWritesNothingAtAll(t *testing.T) { + telemetryLifecycleHome(t) + telemetry.EnableForTest(t, false) + t.Setenv("CODEAF_TELEMETRY", "off") + restore := telemetryArgs("do", "fix the bug") + defer restore() + + session := telemetryBegin() + telemetryEnd(session, 0) + if rows := telemetrySpoolRows(t); len(rows) != 0 { + t.Fatalf("%d events written with telemetry off: %v", len(rows), telemetryEventNames(t, rows)) + } + if _, err := os.Stat(filepath.Join(os.Getenv(home.EnvVar), "telemetry")); !os.IsNotExist(err) { + t.Fatalf("a telemetry directory was created with telemetry off") + } +} + +// TestTelemetryNoticePrintsOnceAcrossTwoInvocations: the notice is shown once +// per install. The first task session marks it shown and spools first_run +// once; the second session of the same install marks nothing and spools no +// second first_run. +func TestTelemetryNoticePrintsOnceAcrossTwoInvocations(t *testing.T) { + telemetryLifecycleHome(t) + restore := telemetryArgs("do", "fix the bug") + defer restore() + + if telemetry.NoticeShown() { + t.Fatalf("a fresh home read as notice-already-shown") + } + // A task command shows the notice even with piped stderr, because a task + // runs unattended and its person may never open a chat. + first := telemetryBegin() + if !telemetry.NoticeShown() { + t.Fatalf("the notice was not marked shown for a task session") + } + telemetryEnd(first, 0) + + rows := telemetrySpoolRows(t) + firstRuns := 0 + for _, row := range rows { + if row["event_name"] == "first_run" { + firstRuns++ + } + } + if firstRuns != 1 { + t.Fatalf("%d first_run events for one install, want 1", firstRuns) + } + + second := telemetryBegin() + telemetryEnd(second, 0) + for _, row := range telemetrySpoolRows(t) { + if row["event_name"] == "first_run" { + t.Fatalf("a second invocation spooled another first_run") + } + } +} + +// TestTelemetryNoticeNeverPrintsUnderJSON: a --json stdout must stay +// machine-clean, so a --json session never shows the notice; the events wait +// in the spool until a session that can show it runs. +func TestTelemetryNoticeNeverPrintsUnderJSON(t *testing.T) { + telemetryLifecycleHome(t) + restore := telemetryArgs("do", "--json", "fix the bug") + defer restore() + + if !telemetryHasJSON(os.Args[1:]) { + t.Fatalf("telemetryHasJSON missed --json") + } + session := telemetryBegin() + telemetryEnd(session, 0) + if telemetry.NoticeShown() { + t.Fatalf("the notice was marked shown under --json") + } +} + +// telemetryFakeExecute runs execute() in-process with os.Args pointed at the +// given words. execute() returns a code and does not exit the process, which +// is what makes this test cheap enough to run alongside the others. +func telemetryFakeExecute(args ...string) int { + restore := telemetryArgs(args...) + defer restore() + return execute() +} diff --git a/internal/telemetry/testhook.go b/internal/telemetry/testhook.go new file mode 100644 index 0000000000..af6775cb28 --- /dev/null +++ b/internal/telemetry/testhook.go @@ -0,0 +1,31 @@ +package telemetry + +// The go-test rung is a production rule: a test binary is never a source of +// truth about usage, and it must stay unreachable by any environment variable +// or config value. But the wiring above this package — cmd/codeaf's lifecycle +// tests — runs inside a test binary too, and it has to exercise the spool +// paths for real or prove nothing. +// +// So the ONE door past the rung is a package-level func variable, set from a +// test only, restored by that test, and never touched by production code: +// no env var, no config key, no build flag can reach it. Setting it is what +// a test does, and a test is the only thing that can. + +import "testing" + +// EnableForTest turns the ladder on (on=true) or back to its honest answer +// (on=false) for the duration of one test, restoring the previous value when +// the test ends. It is the only way past the go-test and unstamped-build +// rungs of [offReason]. +// +// It is deliberately exported and deliberately in the testhook file rather +// than a _test.go one, because the lifecycle tests in cmd/codeaf need it from +// their own package — and it is still a test-only surface by construction: +// nothing in the production call graph reads it, and a caller that is not a +// test has nothing to restore it with. +func EnableForTest(t testing.TB, on bool) { + t.Helper() + previous := forcedOn + forcedOn = on + t.Cleanup(func() { forcedOn = previous }) +} From 55c49e2d978291fb73214670b34ac91081dd0ce2 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 03:03:03 -0400 Subject: [PATCH 23/36] provider: count answered model calls and their spend at the log door --- internal/provider/calllog.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/internal/provider/calllog.go b/internal/provider/calllog.go index 393998aeb6..7598841d83 100644 --- a/internal/provider/calllog.go +++ b/internal/provider/calllog.go @@ -10,6 +10,7 @@ import ( "github.com/Agent-Field/agentfield/sdk/go/ai" "github.com/Agent-Field/codeaf/internal/calllog" + "github.com/Agent-Field/codeaf/internal/telemetry" "github.com/Agent-Field/codeaf/internal/trace" ) @@ -510,6 +511,15 @@ func (c *Client) record(facts recordFacts) { record.Ended = facts.ended facts.knobs.trace.track(facts, record.ID) calllog.Append(record) + // THE TALLY SITS AT THE SAME DOOR AS THE LOG, for the reason the log does: + // every outbound call in the process passes through here, so a call cannot + // be missed by a caller who forgot to count one. Only the row that ENDS an + // attempt is an answered call: the start row says it is in flight, and + // counting that too would count every call twice. The record's own error + // sentence is the failure verdict and its cost the spend, 0 when absent. + if record.Phase != calllog.PhaseStart { + telemetry.CountModelCall(record.Error == "", record.Cost) + } c.recordBodies(facts, record, model) } From 8eb935b77142490593c2a25060de201d1db299eb Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 03:04:15 -0400 Subject: [PATCH 24/36] session: count sealed turns and executed tool calls --- internal/session/loop.go | 11 ++++++ internal/session/loop_speed_test.go | 59 +++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/internal/session/loop.go b/internal/session/loop.go index 3a1690bee0..27a53fd9ef 100644 --- a/internal/session/loop.go +++ b/internal/session/loop.go @@ -92,6 +92,7 @@ import ( "github.com/Agent-Field/codeaf/internal/roles" "github.com/Agent-Field/codeaf/internal/store" "github.com/Agent-Field/codeaf/internal/taxonomy" + "github.com/Agent-Field/codeaf/internal/telemetry" ) // ── retry constants (pi spec §4, verbatim from internal/exec/bare) ────────── @@ -1621,6 +1622,9 @@ func (a *Agent) keepSteeredPartial(partial *partialBuffer, reasoning *reasoningB // (see [sessionFile.writeLine]) — and a turn that spent nothing writes no line // at all (see [sessionFile.appendUsage]). func (a *Agent) sealTurn(turn Usage, started time.Time, model string) Usage { + // The tally sits at the seal because the seal is the shape of a turn: every + // turn that seals counts one, whether or not telemetry is sent later. + telemetry.CountTurn() turn.Duration = time.Since(started) a.mu.Lock() a.usage.Duration += turn.Duration @@ -3219,6 +3223,13 @@ func (a *Agent) runToolsWarm(ctx context.Context, ep *episode, calls []ai.ToolCa func (a *Agent) executeTool(ctx context.Context, ep *episode, hub *eventHub, call ai.ToolCall, rendered string) toolResult { started := time.Now() result := a.dispatchTool(ctx, ep, hub, call, rendered) + // Only a call that RAN counts: a door that refused it before it ran, or a + // hand withdrawn off the belt, is the harness's own answer and rides on + // [toolResult.harness] for this reason — every counter that judges the + // model by its steps skips it. The error flag is the verdict. + if !result.harness { + telemetry.CountToolCall(!result.isError) + } recordToolCall(ctx, call, result, started, time.Since(started)) return result } diff --git a/internal/session/loop_speed_test.go b/internal/session/loop_speed_test.go index 9ae162bcf3..a34a094195 100644 --- a/internal/session/loop_speed_test.go +++ b/internal/session/loop_speed_test.go @@ -33,6 +33,7 @@ import ( "github.com/Agent-Field/codeaf/internal/provider" "github.com/Agent-Field/codeaf/internal/roles" "github.com/Agent-Field/codeaf/internal/store" + "github.com/Agent-Field/codeaf/internal/telemetry" ) // paceLawBudget is the whole of loop.go's law in one figure: how long a person @@ -496,3 +497,61 @@ func TestASlowListenerNeverHoldsTheTurnThatIsTellingIt(t *testing.T) { t.Fatal("the listener was never told a phase — the law was not tested") } } + +// ── THE COUNTERS COST NOTHING ─────────────────────────────────────────────── + +// THE TALLY SITS ON THE PATHS THE LAW ABOVE GUARDS — every sealed turn, every +// answered model call, every answered tool call — so it may not take the wait +// the law exempts nobody from: no allocation, no lock, one atomic add each. A +// counter that allocated would put a heap write on each of those three +// chokepoints, and a test that only timed them would pass on a fast machine +// while the garbage piled up on a slow one. So both halves are pinned: zero +// allocations per call, and a million calls of each kind finishing far inside +// a single pace-law budget. The 200ms bound is deliberately generous — three +// million atomic adds take on the order of ten — because a busy machine is slow +// for reasons that are not this code's, and a test that fails on a loaded +// laptop tests the laptop rather than the counters. +func TestSessionCountersCostNothingBesideTheWork(t *testing.T) { + if allocs := testing.AllocsPerRun(100, func() { + telemetry.CountTurn() + }); allocs != 0 { + t.Fatalf("CountTurn allocated %v times per call, want 0", allocs) + } + if allocs := testing.AllocsPerRun(100, func() { + telemetry.CountModelCall(true, 0.001) + }); allocs != 0 { + t.Fatalf("CountModelCall allocated %v times per call, want 0", allocs) + } + if allocs := testing.AllocsPerRun(100, func() { + telemetry.CountToolCall(true) + }); allocs != 0 { + t.Fatalf("CountToolCall allocated %v times per call, want 0", allocs) + } + + const counterCalls = 1_000_000 + const counterBudget = 200 * time.Millisecond + + started := time.Now() + for i := 0; i < counterCalls; i++ { + telemetry.CountTurn() + } + if took := time.Since(started); took > counterBudget { + t.Fatalf("a million CountTurn took %v, want at most %v", took, counterBudget) + } + + started = time.Now() + for i := 0; i < counterCalls; i++ { + telemetry.CountModelCall(true, 0.001) + } + if took := time.Since(started); took > counterBudget { + t.Fatalf("a million CountModelCall took %v, want at most %v", took, counterBudget) + } + + started = time.Now() + for i := 0; i < counterCalls; i++ { + telemetry.CountToolCall(true) + } + if took := time.Since(started); took > counterBudget { + t.Fatalf("a million CountToolCall took %v, want at most %v", took, counterBudget) + } +} From f7460add5a5654b92e8d0b6d58bf7c0f26aaf6c3 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 03:06:08 -0400 Subject: [PATCH 25/36] cmd/codeaf: the session-ended event carries the run's counters telemetryEnd builds the end event from telemetry.Snapshot so turns, model calls, tool calls and cost reach the spool; the stop-reason switch becomes a map over the exit ladder; and both lifecycle constructors are gated on the ladder being on so a disabled run touches no state. Co-Authored-By: codeaf --- cmd/codeaf/telemetry_lifecycle.go | 92 ++++++++++--------- cmd/codeaf/telemetry_lifecycle_test.go | 117 ++++++++++++++++++++----- 2 files changed, 145 insertions(+), 64 deletions(-) diff --git a/cmd/codeaf/telemetry_lifecycle.go b/cmd/codeaf/telemetry_lifecycle.go index 77be1367ff..45aec165f2 100644 --- a/cmd/codeaf/telemetry_lifecycle.go +++ b/cmd/codeaf/telemetry_lifecycle.go @@ -11,7 +11,6 @@ import ( "context" "crypto/rand" "encoding/hex" - "errors" "os" "time" @@ -75,11 +74,23 @@ func telemetryBegin() telemetrySession { telemetry.PrintNotice() telemetry.MarkNoticeShown() } + // Both opening events go through SpoolSync, not the fire-and-forget Spool: + // first_run must be on disk before session_started even exists, and a run's + // session_started must be spooled before the process can reach the exit and + // flush — an append on its own goroutine can land after the flush has + // already renamed the spool, or not at all when a short run exits first. + // + // Each constructor is reached only when the ladder is on: the event's own + // identity mints and writes the install id, so building one under a run + // that will not send would touch the state root this run promised to leave + // alone — the same reason telemetryEnd gates its constructor. if telemetry.Enabled() && telemetry.FirstRunPending() { - telemetry.Spool(telemetry.FirstRun(time.Now())) + _ = telemetry.SpoolSync(telemetry.FirstRun(time.Now())) telemetry.MarkFirstRunSent() } - telemetry.Spool(telemetry.SessionStarted(mode, resumed, id, time.Now())) + if telemetry.Enabled() { + _ = telemetry.SpoolSync(telemetry.SessionStarted(mode, resumed, id, time.Now())) + } return telemetrySession{mode: mode, resumed: resumed, sessionID: id} } @@ -93,14 +104,23 @@ func telemetryBegin() telemetrySession { // keeps this defer free for every non-session command: `codeaf version` runs // it and it costs a mode read and nothing else. func telemetryEnd(session telemetrySession, code int) { - if session.mode == "" { + // Nothing is built when the ladder is off, and that is load-bearing: the + // event's identity alone reads the install id, which mints and writes one + // on a machine that has never sent anything — the very file this run + // promised not to write. SpoolSync and Flush are no-ops under the same + // rung, so this gate is what keeps a disabled run from touching the state + // root at all. + if session.mode == "" || !telemetry.Enabled() { return } - event := telemetry.SessionEnded(session.mode, telemetry.SessionStats{ - Duration: time.Since(telemetryStarted), - StopReason: telemetryStopReason(code), - ExitCode: code, - }, session.sessionID, time.Now()) + // The run's own tally comes from Snapshot and only the three facts the + // counters cannot know — how long it took, why it ended, what it left + // with — are set on top of it. + stats := telemetry.Snapshot() + stats.Duration = time.Since(telemetryStarted) + stats.StopReason = telemetryStopReason(code) + stats.ExitCode = code + event := telemetry.SessionEnded(session.mode, stats, session.sessionID, time.Now()) _ = telemetry.SpoolSync(event) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() @@ -109,9 +129,9 @@ func telemetryEnd(session telemetrySession, code int) { // telemetryInterrupted is whether the person ended the run themselves — // ctrl-c on the surface, the signal a headless run caught. It is set where -// the signal is caught (do.go, chatv3_at.go), and read once, here, to give -// the session-ended event the contract's "interrupted" word instead of a -// failure word it was not. +// the signal is caught (do.go's errand), and read once, here, to give the +// session-ended event the contract's "interrupted" word instead of a failure +// word it was not. var telemetryInterrupted bool // telemetryStarted is when this process began its session, the honest measure @@ -135,11 +155,24 @@ func telemetrySessionID() string { return hex.EncodeToString(raw[:]) } +// telemetryStopReasons maps the exit ladder's rungs (envelope.go) onto the +// contract's stop_reason vocabulary. It is a straight table keyed by the rung +// itself: the five codes the ladder names, each with the one word the exit +// cannot carry. exitLimit is the ladder's single rung for four limits — wall, +// budget, turn cap, price — and the contract gives the rung a single word, +// "budget", which is the one it names for a limit the person set. +var telemetryStopReasons = map[exitStatus]string{ + exitDone: telemetry.StopDone, + exitCannotRun: telemetry.StopError, + exitIncomplete: telemetry.StopIncomplete, + exitLimit: telemetry.StopBudget, + exitUnanswered: telemetry.StopQuestion, +} + // telemetryStopReason maps the exit code onto the contract's stop_reason -// vocabulary. The codes are the one exit ladder (envelope.go), so the mapping -// is a straight table: a rung's number names the reasons that produce it, and -// a code this build has no word for is "unknown" — the honest answer, because -// the fact that the run ended survives even when the vocabulary does not. +// vocabulary. A code the ladder does not name is "unknown" — the honest +// answer, because the fact that the run ended survives even when the +// vocabulary does not. // // A user interrupt is not on the ladder: interrupting a chat is not a failure // of the conversation, it is the person ending it, and the contract gives it @@ -150,31 +183,10 @@ func telemetryStopReason(code int) string { if telemetryInterrupted { return telemetry.StopInterrupted } - switch { - case errors.Is(exitStatus(code), exitDone): - return telemetry.StopDone - case errors.Is(exitStatus(code), exitCannotRun): - return telemetry.StopError - case errors.Is(exitStatus(code), exitIncomplete): - return telemetry.StopIncomplete - case errors.Is(exitStatus(code), exitLimit): - // The ladder's one rung for four limits — wall, budget, turn cap, price — - // and an exit code cannot say which. The contract's own table gives the - // rung a single word and "budget" is the one it names for a limit the - // person set; anything finer needs the stop reason the command carried, - // which the exit does not keep. - return telemetry.StopBudget - case errors.Is(exitStatus(code), exitUnanswered): - return telemetry.StopQuestion - case errors.Is(exitStatus(code), exitStatus(3)): - // The one code the ladder does not name: exit 3 is the legacy exec - // ladder's "limit" rung under CODEAF_EXIT_CODES=legacy, and this build - // has no word for which of the four limits it was. Unknown, not a - // guess. - return telemetry.StopUnknown - default: - return telemetry.StopUnknown + if reason, ok := telemetryStopReasons[exitStatus(code)]; ok { + return reason } + return telemetry.StopUnknown } // stderrIsTerminal is whether a person can see the notice: stderr attached to diff --git a/cmd/codeaf/telemetry_lifecycle_test.go b/cmd/codeaf/telemetry_lifecycle_test.go index 35c3c23525..05f7a830cc 100644 --- a/cmd/codeaf/telemetry_lifecycle_test.go +++ b/cmd/codeaf/telemetry_lifecycle_test.go @@ -2,7 +2,8 @@ package main // The execute() lifecycle, exercised the way the binary runs it: os.Args // pointed at a real command, CODEAF_HOME in a temporary directory, and the -// endpoint at a local server so a flush never reaches the real relay. +// endpoint at a dead loopback so a flush never reaches the real relay and the +// spool keeps what the run wrote for the assertions to read. // // The package is off under `go test` by design — the go-test rung is a // production rule — and these tests do not weaken it. They use the one door @@ -11,8 +12,6 @@ package main import ( "encoding/json" - "net/http" - "net/http/httptest" "os" "path/filepath" "strings" @@ -22,19 +21,17 @@ import ( "github.com/Agent-Field/codeaf/internal/telemetry" ) -// telemetryLifecycleHome points the process at a temporary home and a local -// endpoint, and turns the ladder on for this test only. Both are restored -// through t.Cleanup. -func telemetryLifecycleHome(t *testing.T) *httptest.Server { +// telemetryLifecycleHome points the process at a temporary home and a dead +// endpoint, and turns the ladder on for this test only. The endpoint refuses +// every connection, so the flush a lifecycle call ends with fails locally +// instead of draining the spool — an endpoint that answered would send the +// events and remove them, and the assertions below would have nothing to read. +func telemetryLifecycleHome(t *testing.T) { t.Helper() t.Setenv(home.EnvVar, t.TempDir()) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNoContent) - })) - t.Cleanup(server.Close) - t.Setenv("CODEAF_TELEMETRY_ENDPOINT", server.URL) + // Nothing listens here, and loopback dialing is refused immediately. + t.Setenv("CODEAF_TELEMETRY_ENDPOINT", "http://127.0.0.1:1/telemetry") telemetry.EnableForTest(t, true) - return server } // telemetrySpoolRows is what the spool holds, parsed. An absent spool is an @@ -75,6 +72,32 @@ func telemetryEventNames(t *testing.T, rows []map[string]any) []string { return names } +// telemetryRowNamed is the one spooled row with an event name, failed on when +// there is none. A fresh home spools first_run before the session events, so +// the tests below find the row they are about by name rather than by index. +func telemetryRowNamed(t *testing.T, rows []map[string]any, name string) map[string]any { + t.Helper() + for _, row := range rows { + if row["event_name"] == name { + return row + } + } + t.Fatalf("no %s row in %v", name, telemetryEventNames(t, rows)) + return nil +} + +// telemetryProps is a spooled row's props object, failed on when it carries +// none. The wire row nests every contract property under "props"; the +// identity fields sit beside it. +func telemetryProps(t *testing.T, row map[string]any) map[string]any { + t.Helper() + props, ok := row["props"].(map[string]any) + if !ok { + t.Fatalf("%v row carries no props", row["event_name"]) + } + return props +} + // telemetryArgs points os.Args at the given words for the duration of one // lifecycle call, because telemetryBegin reads the command line itself. func telemetryArgs(args ...string) func() { @@ -84,9 +107,10 @@ func telemetryArgs(args ...string) func() { } // TestTelemetryTaskSessionSpoolsStartedAndEnded runs one session through the -// lifecycle's own begin and end and reads what it left in the spool: a task -// session spools session_started then session_ended, both with mode "task", -// and the ended event carries the exit code's stop reason. +// lifecycle's own begin and end and reads what it left in the spool: a fresh +// install spools first_run first, then the task session's session_started and +// session_ended, both with mode "task", and the ended event carries the exit +// code's stop reason. func TestTelemetryTaskSessionSpoolsStartedAndEnded(t *testing.T) { telemetryLifecycleHome(t) restore := telemetryArgs("plan", "run", "p.json") @@ -103,20 +127,58 @@ func TestTelemetryTaskSessionSpoolsStartedAndEnded(t *testing.T) { rows := telemetrySpoolRows(t) names := telemetryEventNames(t, rows) - if len(names) != 2 || names[0] != "session_started" || names[1] != "session_ended" { - t.Fatalf("spool holds %v, want [session_started session_ended]", names) + if len(names) != 3 || names[0] != "first_run" || names[1] != "session_started" || names[2] != "session_ended" { + t.Fatalf("spool holds %v, want [first_run session_started session_ended]", names) } - if rows[0]["mode"] != "task" || rows[1]["mode"] != "task" { - t.Fatalf("modes were %v and %v, want task", rows[0]["mode"], rows[1]["mode"]) + startedRow := telemetryRowNamed(t, rows, "session_started") + endedRow := telemetryRowNamed(t, rows, "session_ended") + started := telemetryProps(t, startedRow) + ended := telemetryProps(t, endedRow) + if started["mode"] != "task" || ended["mode"] != "task" { + t.Fatalf("modes were %v and %v, want task", started["mode"], ended["mode"]) } - if rows[1]["stop_reason"] != "done" { - t.Fatalf("stop_reason was %v, want done for exit 0", rows[1]["stop_reason"]) + if ended["stop_reason"] != "done" { + t.Fatalf("stop_reason was %v, want done for exit 0", ended["stop_reason"]) } - if rows[0]["session_id_hash"] != rows[1]["session_id_hash"] { + if startedRow["session_id_hash"] != endedRow["session_id_hash"] { t.Fatalf("started and ended hashed different session ids") } } +// TestTelemetrySessionEndedCarriesTheSessionCounters: the end event is built +// from the process's own tally (telemetry.Snapshot), so a turn, a failed model +// call with its cost, and a tool call counted during the run reach the spooled +// session_ended line as the contract's bands. The counters are process-wide +// and this test is the only one in the package that touches them, so their +// count is exactly the three calls made here. +func TestTelemetrySessionEndedCarriesTheSessionCounters(t *testing.T) { + telemetryLifecycleHome(t) + restore := telemetryArgs("do", "fix the bug") + defer restore() + + telemetry.CountTurn() + telemetry.CountModelCall(false, 0.02) + telemetry.CountToolCall(true) + + session := telemetryBegin() + telemetryEnd(session, 0) + + ended := telemetryProps(t, telemetryRowNamed(t, telemetrySpoolRows(t), "session_ended")) + want := map[string]string{ + "turns": "1", + "model_calls": "1", + "model_calls_failed": "1", + "tool_calls": "1", + "tool_calls_failed": "0", + "cost_usd": "0.01-0.1", + } + for key, value := range want { + if ended[key] != value { + t.Errorf("session_ended %s = %v, want %s", key, ended[key], value) + } + } +} + // TestTelemetryModeIsReadFromTheCommandLine is the mode table: only session // commands emit, chat and resume are chat, the task verbs are task, and // nothing else is a session at all. @@ -220,11 +282,18 @@ func TestTelemetryNoticePrintsOnceAcrossTwoInvocations(t *testing.T) { second := telemetryBegin() telemetryEnd(second, 0) + // The first session's first_run is still in the spool — the flush could not + // send and kept every line — so the check is that the second invocation + // added none: one first_run for the install across both invocations. + firstRuns = 0 for _, row := range telemetrySpoolRows(t) { if row["event_name"] == "first_run" { - t.Fatalf("a second invocation spooled another first_run") + firstRuns++ } } + if firstRuns != 1 { + t.Fatalf("two invocations spooled %d first_run events, want 1", firstRuns) + } } // TestTelemetryNoticeNeverPrintsUnderJSON: a --json stdout must stay From 15c8561391118cd051127533a922a749068d539a Mon Sep 17 00:00:00 2001 From: codeaf Date: Thu, 17 Sep 2026 03:11:52 -0400 Subject: [PATCH 26/36] cmd/codeaf: the counters test says the tally is process-wide The exact turn, call and cost assertions hold for one run of the package; a repeated run counts into the same atomic tally again. Co-Authored-By: codeaf --- cmd/codeaf/telemetry_lifecycle_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/codeaf/telemetry_lifecycle_test.go b/cmd/codeaf/telemetry_lifecycle_test.go index 05f7a830cc..7a772689a7 100644 --- a/cmd/codeaf/telemetry_lifecycle_test.go +++ b/cmd/codeaf/telemetry_lifecycle_test.go @@ -150,7 +150,9 @@ func TestTelemetryTaskSessionSpoolsStartedAndEnded(t *testing.T) { // call with its cost, and a tool call counted during the run reach the spooled // session_ended line as the contract's bands. The counters are process-wide // and this test is the only one in the package that touches them, so their -// count is exactly the three calls made here. +// count is exactly the three calls made here — for one run of the package: a +// repeated run (-count=2 and up) counts into the same tally again, and the +// door that zeroes it belongs to the telemetry package's own tests. func TestTelemetrySessionEndedCarriesTheSessionCounters(t *testing.T) { telemetryLifecycleHome(t) restore := telemetryArgs("do", "fix the bug") From 4cfb27397eeaa22aa953b7eb569abc2d2ccae253 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 03:12:54 -0400 Subject: [PATCH 27/36] guard: a fault hook Recover calls, for a reporter above this package Co-Authored-By: codeaf --- internal/guard/guard.go | 19 +++++++++++++++- internal/guard/guard_test.go | 44 ++++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/internal/guard/guard.go b/internal/guard/guard.go index 43c8b8852c..aec6c3dfd8 100644 --- a/internal/guard/guard.go +++ b/internal/guard/guard.go @@ -53,12 +53,29 @@ func note(scope string, recovered any, stack []byte) error { return fault } +// OnFault is called from Recover once the fault has been recorded, with the +// scope Recover was given and the stack it captured for the log. It is nil by +// default, and it is a variable rather than a call because of where this +// package sits: internal/telemetry spools its own writes through guard.Go, so +// an import from here back to anything that would report a fault is a cycle. +// The reporting arrives as a func value instead, set once by the top of the +// binary and left nil by every package that has nothing to report to. +// +// It runs on the recovered goroutine, inside the deferred Recover, so it must +// neither block nor panic: the goroutine ends the moment it returns, and a +// fault in the reporter would be the one fault guard could not absorb. +var OnFault func(scope string, stack []byte) + // Recover absorbs a panic in the deferring goroutine and lets it end quietly: // `defer guard.Recover("narrator")`. Use it where there is no result to carry // the fault — a fire-and-forget spawn whose only obligation is not to crash. func Recover(scope string) { if recovered := recover(); recovered != nil { - _ = note(scope, recovered, debug.Stack()) + stack := debug.Stack() + _ = note(scope, recovered, stack) + if OnFault != nil { + OnFault(scope, stack) + } } } diff --git a/internal/guard/guard_test.go b/internal/guard/guard_test.go index eeecb2bd88..28e99b03bb 100644 --- a/internal/guard/guard_test.go +++ b/internal/guard/guard_test.go @@ -128,3 +128,47 @@ func TestFaultWithoutScopeStillReads(t *testing.T) { t.Fatalf("unexpected text: %s", err.Error()) } } + +// TestGoCallsTheFaultHookOnceWithTheGoroutineStack holds the hook's whole +// contract: a guarded goroutine that panics calls OnFault exactly once, with +// the scope Recover was given and the stack of the goroutine that faulted — +// the same bytes the log line carries, because a reporter above this package +// has nothing else to group the fault by. +func TestGoCallsTheFaultHookOnceWithTheGoroutineStack(t *testing.T) { + captureLog(t) + previous := OnFault + t.Cleanup(func() { OnFault = previous }) + + type hookCall struct { + scope string + stack []byte + } + // Buffered for two so a second call is a readable failure rather than a + // goroutine blocked on a send nobody drains. + calls := make(chan hookCall, 2) + OnFault = func(scope string, stack []byte) { calls <- hookCall{scope: scope, stack: stack} } + + Go("narrator", func() { panic("narration blew up") }) + + var got hookCall + select { + case got = <-calls: + case <-time.After(5 * time.Second): + t.Fatal("the fault hook was never called") + } + if got.scope != "narrator" { + t.Fatalf("hook scope = %q, want narrator", got.scope) + } + if len(got.stack) == 0 { + t.Fatal("the hook was called with an empty stack") + } + if !strings.Contains(string(got.stack), "guard_test.go") { + t.Fatalf("the stack is not the recovered goroutine's: %q", got.stack) + } + + select { + case second := <-calls: + t.Fatalf("the hook was called twice, second scope %q", second.scope) + case <-time.After(50 * time.Millisecond): + } +} From 857a3ff35ac70bd9841bc90767d065fc24df2c50 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 03:15:37 -0400 Subject: [PATCH 28/36] cmd/codeaf: the fault event at both crash sites, off the current session Co-Authored-By: codeaf --- cmd/codeaf/fault.go | 37 +++++++++++++++++++++++- cmd/codeaf/telemetry_lifecycle.go | 48 +++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 3 deletions(-) diff --git a/cmd/codeaf/fault.go b/cmd/codeaf/fault.go index 7a1139e349..0cb4e0361a 100644 --- a/cmd/codeaf/fault.go +++ b/cmd/codeaf/fault.go @@ -1,6 +1,7 @@ package main import ( + "context" "fmt" "io" "os" @@ -9,6 +10,7 @@ import ( "time" "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/telemetry" ) // faultMessage is what the user reads when codeaf could not keep going. It is @@ -17,7 +19,10 @@ import ( const faultMessage = "codeaf hit an internal fault and had to stop. Nothing is lost — the graph is durable, and restarting resumes where it left off. Details: %s\n" // reportFault writes the stack where it is useful and the sentence where it is -// read, and answers with the process exit code. +// read, and answers with the process exit code. The anonymous fault count is +// the last thing it does, once the sentence is out: a flush may spend a second +// this process no longer has, and the person should not wait on it to read +// what happened. func reportFault(stderr io.Writer, detail string, stack []byte) int { // A fault has no settings to read — it is what is left when the launch did // not get that far — but the profile is an environment pin and is still @@ -29,9 +34,39 @@ func reportFault(stderr io.Writer, detail string, stack []byte) int { path := chatLogPath(config.ProfileDir()) writeFaultLog(path, detail, stack) fmt.Fprintf(stderr, faultMessage, displayPath(path)) + telemetryFault(stack) return 1 } +// telemetryFault spools the contract's fault event for a dying process and +// sends what the spool holds. It is given the stack and nothing else. The +// library hashes the fingerprint out of the stack's own function names, and +// `detail` — the panic's text — never crosses this line: a panic value is the +// person's words, a path on their machine or a fragment of the file they were +// working in, and the fault event has no property for any of it. +// +// SpoolSync and then Flush under a one-second context, the same budget +// telemetryEnd gives a run's last event, because this is the last thing the +// process does: there is no later flush to carry the line, and an append that +// has not landed when the process exits is an event that never happened. The +// gate is the opt-out ladder's own answer, so a person who turned the counts +// off gets the same crash report they always got and nothing on the wire. +func telemetryFault(stack []byte) { + if !telemetry.Enabled() { + return + } + session := currentTelemetrySession + event := telemetry.FaultEvent(telemetry.Fault{ + Mode: string(session.mode), + Scope: telemetry.ScopeMain, + Stack: stack, + }, session.sessionID, time.Now()) + _ = telemetry.SpoolSync(event) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _ = telemetry.Flush(ctx) +} + // chatLogPath is the ONE name of the file this binary parks the standard logger // in: EVERY door that opens the v3 surface does it for the surface's whole // lifetime so a log line cannot tear through the frame (chatv3_surface.go's diff --git a/cmd/codeaf/telemetry_lifecycle.go b/cmd/codeaf/telemetry_lifecycle.go index 45aec165f2..0ac89ce404 100644 --- a/cmd/codeaf/telemetry_lifecycle.go +++ b/cmd/codeaf/telemetry_lifecycle.go @@ -15,6 +15,7 @@ import ( "time" "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/guard" "github.com/Agent-Field/codeaf/internal/telemetry" "github.com/Agent-Field/codeaf/internal/trace" ) @@ -43,6 +44,16 @@ type telemetrySession struct { sessionID string } +// currentTelemetrySession is the session this invocation decided, kept where +// the two crash sites can reach it. execute() holds the same value in a local, +// but neither reporter runs where that local is in scope: reportFault is the +// deferred recover at the top of main.go, and guard's hook runs on a goroutine +// that never saw execute's frame at all. Both read it here. +// +// The zero value is a command that is not a session — version, help, doctor — +// and its fault still counts, carrying the mode the contract spells "other". +var currentTelemetrySession telemetrySession + // telemetryBegin decides, from the command line alone, whether this // invocation is a session worth counting, and spools its opening events. // @@ -59,12 +70,17 @@ type telemetrySession struct { // A task command is always shown it, because a task runs unattended and its // person may never open a chat at all. func telemetryBegin() telemetrySession { + // Wired first: a fault can arrive from any goroutine the run spawns from + // here on, and the hook reads the session assigned below when it fires. + telemetryFaultHook() args := os.Args[1:] mode, resumed, session := telemetryMode(args) if !session { - return telemetrySession{} + currentTelemetrySession = telemetrySession{} + return currentTelemetrySession } id := telemetrySessionID() + currentTelemetrySession = telemetrySession{mode: mode, resumed: resumed, sessionID: id} // A run the ladder has turned off prints no notice either: the notice is // the sentence that asks permission to send, and a pipe that will never // send has nobody to ask — and marking it shown would create the very @@ -91,7 +107,35 @@ func telemetryBegin() telemetrySession { if telemetry.Enabled() { _ = telemetry.SpoolSync(telemetry.SessionStarted(mode, resumed, id, time.Now())) } - return telemetrySession{mode: mode, resumed: resumed, sessionID: id} + return currentTelemetrySession +} + +// telemetryFaultHook hands guard the reporter it cannot import. guard sits +// below internal/telemetry — the spool's own append runs on guard.Go — so the +// package that absorbs a goroutine's panic must not depend on the package that +// would count it. It calls a func value instead, and this is the one place the +// binary wires the two together. +// +// Spool, not SpoolSync: a guarded goroutine's fault does not end the process, +// so the event waits for the flush this run already makes at its exit and the +// faulting goroutine is not held for a write. The hook is set only when the +// ladder is on, which is the same answer the crash site in fault.go asks. +// +// The scope guard passes is its own label for the log — narrator, +// telemetry.spool — while the contract's scope is one of three words, and the +// word for every fault recovered here is "goroutine". +func telemetryFaultHook() { + if !telemetry.Enabled() { + return + } + guard.OnFault = func(scope string, stack []byte) { + session := currentTelemetrySession + telemetry.Spool(telemetry.FaultEvent(telemetry.Fault{ + Mode: string(session.mode), + Scope: telemetry.ScopeGoroutine, + Stack: stack, + }, session.sessionID, time.Now())) + } } // telemetryEnd closes one run at the exit, from the process's own tally From e327fa23435b275baec132e9fb3da5d510955642 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 03:20:26 -0400 Subject: [PATCH 29/36] cmd/codeaf: a test that the crash site spools a fault and not the panic text Co-Authored-By: codeaf --- cmd/codeaf/telemetry_fault_test.go | 149 +++++++++++++++++++++++++++++ 1 file changed, 149 insertions(+) create mode 100644 cmd/codeaf/telemetry_fault_test.go diff --git a/cmd/codeaf/telemetry_fault_test.go b/cmd/codeaf/telemetry_fault_test.go new file mode 100644 index 0000000000..ae95a0ef17 --- /dev/null +++ b/cmd/codeaf/telemetry_fault_test.go @@ -0,0 +1,149 @@ +package main + +// The crash site's one telemetry obligation, held down: reportFault leaves a +// fault event in the spool carrying the contract's three properties, and the +// panic's own text reaches neither a key nor a value of it. +// +// The event is read out of the spool rather than off the wire because that is +// where a dying process leaves it. The flush reportFault makes is a no-op until +// the notice has been shown, and a test's home has not shown it, so the line is +// still on disk here rather than on its way to the endpoint. + +import ( + "bytes" + "regexp" + "strings" + "testing" + + "github.com/Agent-Field/codeaf/internal/config" + "github.com/Agent-Field/codeaf/internal/telemetry" +) + +// faultDetail is the panic value reportFault is handed: the runtime's words, a +// path on the person's machine, and a token shaped like a credential. A panic +// text carries whatever the failing line was holding, which is exactly why the +// contract gives the fault event no property for it. +const faultDetail = "runtime error: index out of range [3:] reading " + + "/Users/somebody/private/notes.md (OPENROUTER_API_KEY=sk-or-v1-abcdef)" + +// faultStack is a fake dump: two codeaf frames for the fingerprint to hash, +// each with a file half carrying a machine path of its own. +const faultStack = `goroutine 1 [running]: +main.runDo(0x0?) + /Users/somebody/work/codeaf/cmd/codeaf/do.go:42 +0x1c8 +github.com/Agent-Field/codeaf/internal/session.(*Agent).Run(0x0?, {0x0?, 0x0?}) + /Users/somebody/work/codeaf/internal/session/agent.go:100 +0x60 +` + +// faultLeaks are the strings that must not appear anywhere in a spooled row: +// the panic text whole and in its sensitive pieces, and the stack's paths, +// which are the same leak through another door. +var faultLeaks = []string{ + faultDetail, + "index out of range", + "/Users/somebody/private/notes.md", + "OPENROUTER_API_KEY", + "sk-or-v1-abcdef", + "/Users/somebody/work/codeaf", +} + +var reFaultFingerprint = regexp.MustCompile(`^[0-9a-f]{16}$`) + +// TestReportFaultSpoolsAFaultEventAndNotThePanicText runs one crash through +// reportFault and reads what it left behind: a single fault event, scoped to +// the main goroutine, carrying the session's mode and a fingerprint hashed from +// the stack — and nothing that says what panicked. The exit code and the +// sentence on stderr are asserted too, because the count is allowed to cost +// this function neither of them. +func TestReportFaultSpoolsAFaultEventAndNotThePanicText(t *testing.T) { + telemetryLifecycleHome(t) + // The fault log is appended under the profile, and a profile pinned by the + // environment would put it outside this test's home. + t.Setenv(config.ProfileDirEnv, "") + + previous := currentTelemetrySession + currentTelemetrySession = telemetrySession{mode: telemetry.ModeChat, sessionID: "fault-session"} + t.Cleanup(func() { currentTelemetrySession = previous }) + + stderr := &bytes.Buffer{} + if code := reportFault(stderr, faultDetail, []byte(faultStack)); code != 1 { + t.Fatalf("exit code = %d, want 1", code) + } + if shown := stderr.String(); !strings.Contains(shown, "codeaf hit an internal fault and had to stop") { + t.Fatalf("the calm sentence did not reach stderr: %q", shown) + } + + rows := telemetrySpoolRows(t) + faults := 0 + var fault map[string]any + for _, row := range rows { + // Every row is walked, not just the fault's: an asynchronous append + // from an earlier test in this binary can land in this home, so the + // leak check reads all of them and the count below reads only the ones + // named fault. + assertNoFaultLeak(t, row) + if row["event_name"] == "fault" { + faults++ + fault = row + } + } + if faults != 1 { + t.Fatalf("%d fault events for one crash, want 1 (spool holds %v)", + faults, telemetryEventNames(t, rows)) + } + + props, ok := fault["props"].(map[string]any) + if !ok { + t.Fatalf("the fault row carries no props object: %v", fault) + } + if props["scope"] != telemetry.ScopeMain { + t.Fatalf("scope = %v, want main", props["scope"]) + } + if props["mode"] != string(telemetry.ModeChat) { + t.Fatalf("mode = %v, want the session's chat: the event is built off currentTelemetrySession", + props["mode"]) + } + if fingerprint, _ := props["fingerprint"].(string); !reFaultFingerprint.MatchString(fingerprint) { + t.Fatalf("fingerprint = %q, want 16 lowercase hex characters", fingerprint) + } +} + +// assertNoFaultLeak fails on the first key or value of a parsed spool row that +// carries any of the panic's text. Walking the row is the assertion the +// contract actually makes: it is not enough that the fault event has no +// property named for the panic message, because a key or a value anywhere in +// the row could hold one. +func assertNoFaultLeak(t *testing.T, row map[string]any) { + t.Helper() + for _, text := range spoolRowText(row) { + for _, leak := range faultLeaks { + if strings.Contains(text, leak) { + t.Fatalf("the panic text reached the spool: %q holds %q", text, leak) + } + } + } +} + +// spoolRowText is every key and every string value in a parsed row, at any +// depth. Anything that is neither — a number, a bool — cannot carry text and is +// left out. +func spoolRowText(value any) []string { + switch typed := value.(type) { + case map[string]any: + var out []string + for key, child := range typed { + out = append(out, key) + out = append(out, spoolRowText(child)...) + } + return out + case []any: + var out []string + for _, child := range typed { + out = append(out, spoolRowText(child)...) + } + return out + case string: + return []string{typed} + } + return nil +} From 67501455651c3d8a0fe3bd951d515009f3175a89 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 03:02:49 -0400 Subject: [PATCH 30/36] docs: finish the anonymous usage counts contract and add the 1095 changelog entry --- README.md | 21 ++++++----- docs/TELEMETRY.md | 35 +++++++++++++++---- .../unreleased/1095-anonymous-usage-counts.md | 16 +++++++++ 3 files changed, 55 insertions(+), 17 deletions(-) create mode 100644 docs/changes/unreleased/1095-anonymous-usage-counts.md diff --git a/README.md b/README.md index 83de3ab566..acd788a98c 100644 --- a/README.md +++ b/README.md @@ -71,17 +71,16 @@ codeaf sends anonymous usage counts to AgentField. ``` The events are counts and buckets only — no prompts, code, or file names ever -leave the machine. `docs/TELEMETRY.md` lists every field that is sent. The -installer records how and when this copy was installed (`telemetry/install.json` -under the state root) so the counts can bucket by channel; it sends nothing. - -To turn it off: - -```bash -export CODEAF_TELEMETRY=off -``` - -`DO_NOT_TRACK=1` works too. +leave the machine. `docs/TELEMETRY.md` lists every field that is sent and every +way to turn the counts off. The installer records how and when this copy was +installed (`telemetry/install.json` under the state root) so the counts can +bucket by channel; it sends nothing. + +`codeaf telemetry show` prints exactly what is waiting to leave the machine; +`status`, `on` and `off` are the rest. Turn it off with `CODEAF_TELEMETRY=off` +(also `0` or `false`), `DO_NOT_TRACK=1`, `codeaf telemetry off`, `telemetry = +off` in a project's `.codeaf/config.json`, or an empty +`CODEAF_TELEMETRY_ENDPOINT`. ## One window for every project diff --git a/docs/TELEMETRY.md b/docs/TELEMETRY.md index 584525b124..c0d7faf69d 100644 --- a/docs/TELEMETRY.md +++ b/docs/TELEMETRY.md @@ -74,18 +74,41 @@ per request, nothing older than 7 days, at most 1000 lines kept, and nothing sent before the notice has been shown. `codeaf telemetry show` prints exactly what has not left yet. -## How to turn it off +## Turning it off -Any one of these, before codeaf starts: +Any one of these turns the counts off. They are checked in this order: -- `CODEAF_TELEMETRY=off` (also `0` or `false`) -- `DO_NOT_TRACK=1` (also `true`) -- the config key `telemetry = off` -- an empty `CODEAF_TELEMETRY_ENDPOINT` +1. `CODEAF_TELEMETRY=off` — also `0` or `false`. +2. `DO_NOT_TRACK=1` — also `true`, the ecosystem's own word for it. +3. `telemetry = off` in the project's settings file, `.codeaf/config.json`. A + project may only turn the counts off, never on. +4. `codeaf telemetry off`, which writes the profile setting; `codeaf telemetry + on` is the way back. +5. an empty `CODEAF_TELEMETRY_ENDPOINT`. A build that cannot name its own source — dirty or unstamped — never reports, and neither does a test binary. +## The command + +`codeaf telemetry` reads the counts and never sends anything of its own. + +- `codeaf telemetry status` says whether the counts are on, and why not when + they are off. +- `codeaf telemetry show` prints exactly what is waiting to leave the machine. +- `codeaf telemetry off` and `codeaf telemetry on` write the profile setting. + +``` +telemetry on + endpoint: https://agentfield.ai/api/oss/codeaf/telemetry + notice: shown + spooled events: 3 + install: 4f2a9c1b7e08… +``` + +When the counts are off, a `reason:` line follows `telemetry off` and names the +switch that turned them off. + ## Download counts The download numbers this repository reports come from public GitHub release diff --git a/docs/changes/unreleased/1095-anonymous-usage-counts.md b/docs/changes/unreleased/1095-anonymous-usage-counts.md new file mode 100644 index 0000000000..d60cc2034c --- /dev/null +++ b/docs/changes/unreleased/1095-anonymous-usage-counts.md @@ -0,0 +1,16 @@ +--- +kind: added +title: Anonymous usage counts, on by default with a notice printed once +pr: 1095 +surface: [docs] +invalidates: [] +--- + +codeaf now sends anonymous usage counts — version, OS, mode (chat or task), and +banded session and error counts — so the parts people rely on get the work. It +never sends prompts, code, paths, names, or anything that identifies you. The +counts are on by default, and the first session prints a notice naming exactly +what leaves; `codeaf telemetry show` prints what is waiting to leave right now. +Turn it off with `CODEAF_TELEMETRY=off`, `DO_NOT_TRACK=1`, `codeaf telemetry +off`, `telemetry = off` in a project's `.codeaf/config.json`, or an empty +`CODEAF_TELEMETRY_ENDPOINT`. The whole contract is `docs/TELEMETRY.md`. From 714db92d7d0102afe08e38f74fd47d0599f18e02 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 03:32:58 -0400 Subject: [PATCH 31/36] guard: the fault hook sits behind an atomic pointer A goroutine recovering a panic read the hook while a test on another goroutine wrote it, which the race detector reported. SetOnFault stores it once; Recover loads it. Co-Authored-By: Claude Fable 5.1 --- cmd/codeaf/telemetry_lifecycle.go | 4 ++-- internal/guard/guard.go | 25 +++++++++++++++++++++---- internal/guard/guard_test.go | 7 +++---- 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/cmd/codeaf/telemetry_lifecycle.go b/cmd/codeaf/telemetry_lifecycle.go index 0ac89ce404..e230d8ffd5 100644 --- a/cmd/codeaf/telemetry_lifecycle.go +++ b/cmd/codeaf/telemetry_lifecycle.go @@ -128,14 +128,14 @@ func telemetryFaultHook() { if !telemetry.Enabled() { return } - guard.OnFault = func(scope string, stack []byte) { + guard.SetOnFault(func(scope string, stack []byte) { session := currentTelemetrySession telemetry.Spool(telemetry.FaultEvent(telemetry.Fault{ Mode: string(session.mode), Scope: telemetry.ScopeGoroutine, Stack: stack, }, session.sessionID, time.Now())) - } + }) } // telemetryEnd closes one run at the exit, from the process's own tally diff --git a/internal/guard/guard.go b/internal/guard/guard.go index aec6c3dfd8..38cc456865 100644 --- a/internal/guard/guard.go +++ b/internal/guard/guard.go @@ -13,6 +13,7 @@ import ( "log" "runtime/debug" "strings" + "sync/atomic" ) // Fault is what a recovered panic becomes: an ordinary error the caller can @@ -53,7 +54,7 @@ func note(scope string, recovered any, stack []byte) error { return fault } -// OnFault is called from Recover once the fault has been recorded, with the +// The fault hook is called from Recover once the fault has been recorded, with the // scope Recover was given and the stack it captured for the log. It is nil by // default, and it is a variable rather than a call because of where this // package sits: internal/telemetry spools its own writes through guard.Go, so @@ -64,7 +65,23 @@ func note(scope string, recovered any, stack []byte) error { // It runs on the recovered goroutine, inside the deferred Recover, so it must // neither block nor panic: the goroutine ends the moment it returns, and a // fault in the reporter would be the one fault guard could not absorb. -var OnFault func(scope string, stack []byte) +type faultHook func(scope string, stack []byte) + +// onFault holds the hook behind an atomic pointer: the binary sets it once at +// start-up and every recovered goroutine reads it, and a plain variable read +// on one goroutine while another writes it is a race the detector rightly +// reports. +var onFault atomic.Pointer[faultHook] + +// SetOnFault installs the hook, or removes it with nil. +func SetOnFault(fn func(scope string, stack []byte)) { + if fn == nil { + onFault.Store(nil) + return + } + hook := faultHook(fn) + onFault.Store(&hook) +} // Recover absorbs a panic in the deferring goroutine and lets it end quietly: // `defer guard.Recover("narrator")`. Use it where there is no result to carry @@ -73,8 +90,8 @@ func Recover(scope string) { if recovered := recover(); recovered != nil { stack := debug.Stack() _ = note(scope, recovered, stack) - if OnFault != nil { - OnFault(scope, stack) + if hook := onFault.Load(); hook != nil { + (*hook)(scope, stack) } } } diff --git a/internal/guard/guard_test.go b/internal/guard/guard_test.go index 28e99b03bb..5d0e36e0c4 100644 --- a/internal/guard/guard_test.go +++ b/internal/guard/guard_test.go @@ -130,14 +130,13 @@ func TestFaultWithoutScopeStillReads(t *testing.T) { } // TestGoCallsTheFaultHookOnceWithTheGoroutineStack holds the hook's whole -// contract: a guarded goroutine that panics calls OnFault exactly once, with +// contract: a guarded goroutine that panics calls the fault hook exactly once, with // the scope Recover was given and the stack of the goroutine that faulted — // the same bytes the log line carries, because a reporter above this package // has nothing else to group the fault by. func TestGoCallsTheFaultHookOnceWithTheGoroutineStack(t *testing.T) { captureLog(t) - previous := OnFault - t.Cleanup(func() { OnFault = previous }) + t.Cleanup(func() { SetOnFault(nil) }) type hookCall struct { scope string @@ -146,7 +145,7 @@ func TestGoCallsTheFaultHookOnceWithTheGoroutineStack(t *testing.T) { // Buffered for two so a second call is a readable failure rather than a // goroutine blocked on a send nobody drains. calls := make(chan hookCall, 2) - OnFault = func(scope string, stack []byte) { calls <- hookCall{scope: scope, stack: stack} } + SetOnFault(func(scope string, stack []byte) { calls <- hookCall{scope: scope, stack: stack} }) Go("narrator", func() { panic("narration blew up") }) From 6eb6d42c96e72a243da6bbeebae6572a0921b811 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 03:37:25 -0400 Subject: [PATCH 32/36] cmd/codeaf: a task's interrupt word comes from the signal, not the cancelled context stopSignals cancels the signal context at the end of every errand, so the flag read from its Err said interrupted for a run that finished cleanly. Co-Authored-By: Claude Fable 5.1 --- cmd/codeaf/do.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cmd/codeaf/do.go b/cmd/codeaf/do.go index 5e6ebb237f..787c8cd0be 100644 --- a/cmd/codeaf/do.go +++ b/cmd/codeaf/do.go @@ -591,8 +591,19 @@ func errandRun(request doRequest, seats config.Seats, started time.Time) (outcom defer stopSignals() // The person's own ending of the run is the one fact the usage counts // keep about how it ended: the interrupt word, not a failure word. - telemetryInterrupted = signalled.Err() != nil - defer func() { telemetryInterrupted = signalled.Err() != nil }() + // signalled cannot answer it — stopSignals cancels that context on the + // way out too, so its Err reads the same for a clean end and a ctrl-c. + // A channel only a signal fills can. + interrupts := make(chan os.Signal, 1) + signal.Notify(interrupts, os.Interrupt, syscall.SIGTERM) + defer signal.Stop(interrupts) + defer func() { + select { + case <-interrupts: + telemetryInterrupted = true + default: + } + }() // A PERSON TYPED THIS, so its calls are made for somebody who is reading // them (exec.go's [typedDoorContext]): the talk pin rides them and a // refused pin is said to the one who is waiting. From 6430d50efa7ab507d194560ed7726b99c916cfac Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 05:10:24 -0400 Subject: [PATCH 33/36] tui3, installer: the telemetry row has a tab, and a custom install writes no marker TestEverySettingRowHasATab wants every registry row on the settings panel. TestCustomInstallOutsideTheStateRootDoesNotRunAdoption wants an install outside the state root to leave the login's state folders alone, so the install marker is written only when the root exists or the install is in it. Co-Authored-By: Claude Fable 5.1 --- internal/tui3/settings.go | 6 ++++++ scripts/install.sh | 8 +++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/internal/tui3/settings.go b/internal/tui3/settings.go index 46c78d28c5..d42a183aa5 100644 --- a/internal/tui3/settings.go +++ b/internal/tui3/settings.go @@ -588,6 +588,12 @@ var settingUI = map[string]settingMeta{ about: "remembers the messages you send, so the up arrow walks them back " + "in a later session.", }, + config.KeyTelemetry: { + tab: tabDisplay, label: "telemetry", widget: widgetToggle, + about: "sends anonymous usage counts (version, OS, mode, session and error " + + "counts) after a notice has been printed once; never prompts, code, paths " + + "or names. Off sends nothing.", + }, config.KeyDraftPersist: { tab: tabDisplay, label: "keep drafts", widget: widgetToggle, about: "keeps the half-typed message in the box across a restart, per directory.", diff --git a/scripts/install.sh b/scripts/install.sh index 94de911948..72203e338c 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -489,5 +489,11 @@ else CODEAF_HOME="$STATE_ROOT" "$INSTALL_DIR/codeaf${extension}" version fi -write_install_marker "$STATE_ROOT" +# The install marker lives under the state root, and a custom install outside +# it must not create the login's state folders: the marker is written when the +# install is inside the state root or the root already exists, and skipped +# otherwise (the binary then reports install_method "unknown"). +if [[ "$RUN_BOOT_ADOPTION" == "1" || -d "$STATE_ROOT" ]]; then + write_install_marker "$STATE_ROOT" +fi print_telemetry_notice From b5cdb1c68bbcef0d50cf6debe24cb2359c44fe96 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 06:07:00 -0400 Subject: [PATCH 34/36] cmd/codeaf: the counters test zeroes the process tally first Other tests in the package drive model and tool calls through the doors that count, so the exact bands held only when the test ran alone. Co-Authored-By: Claude Fable 5.1 --- cmd/codeaf/telemetry_lifecycle_test.go | 8 ++++---- internal/telemetry/testhook.go | 10 ++++++++++ 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cmd/codeaf/telemetry_lifecycle_test.go b/cmd/codeaf/telemetry_lifecycle_test.go index 7a772689a7..8b324d2ec6 100644 --- a/cmd/codeaf/telemetry_lifecycle_test.go +++ b/cmd/codeaf/telemetry_lifecycle_test.go @@ -149,12 +149,12 @@ func TestTelemetryTaskSessionSpoolsStartedAndEnded(t *testing.T) { // from the process's own tally (telemetry.Snapshot), so a turn, a failed model // call with its cost, and a tool call counted during the run reach the spooled // session_ended line as the contract's bands. The counters are process-wide -// and this test is the only one in the package that touches them, so their -// count is exactly the three calls made here — for one run of the package: a -// repeated run (-count=2 and up) counts into the same tally again, and the -// door that zeroes it belongs to the telemetry package's own tests. +// and other tests in this package drive model and tool calls through the +// doors that count, so the tally is zeroed first and the count is exactly the +// three calls made here. func TestTelemetrySessionEndedCarriesTheSessionCounters(t *testing.T) { telemetryLifecycleHome(t) + telemetry.ResetCountersForTest(t) restore := telemetryArgs("do", "fix the bug") defer restore() diff --git a/internal/telemetry/testhook.go b/internal/telemetry/testhook.go index af6775cb28..f88c7cbd95 100644 --- a/internal/telemetry/testhook.go +++ b/internal/telemetry/testhook.go @@ -29,3 +29,13 @@ func EnableForTest(t testing.TB, on bool) { forcedOn = on t.Cleanup(func() { forcedOn = previous }) } + +// ResetCountersForTest zeroes the process-wide session counters for one test +// and again when it ends, so a test that asserts exact bands is not reading +// what an earlier test in the same binary counted. Test-only for the same +// reason EnableForTest is: nothing in the production call graph reaches it. +func ResetCountersForTest(t testing.TB) { + t.Helper() + resetCountersForTest() + t.Cleanup(resetCountersForTest) +} From c51e08be8b9c2eca1d8abcb2cdd27f5d8b93c6a5 Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 06:15:30 -0400 Subject: [PATCH 35/36] cmd/codeaf: the version test drives the telemetry lifecycle, not execute execute starts writers under the state root that outlive the test, and the temp dir's cleanup raced them. The lifecycle alone is what the test pins. Co-Authored-By: Claude Fable 5.1 --- cmd/codeaf/telemetry_lifecycle_test.go | 20 +++++++++----------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/cmd/codeaf/telemetry_lifecycle_test.go b/cmd/codeaf/telemetry_lifecycle_test.go index 8b324d2ec6..d15b7dfdeb 100644 --- a/cmd/codeaf/telemetry_lifecycle_test.go +++ b/cmd/codeaf/telemetry_lifecycle_test.go @@ -215,13 +215,19 @@ func TestTelemetryModeIsReadFromTheCommandLine(t *testing.T) { // TestTelemetryVersionSpoolsNothingAndCreatesNoDirectory: a non-session // command emits nothing and sends nothing, not even the directory the spool -// would live in. +// would live in. The lifecycle is driven directly rather than through +// execute(): the real command starts writers under the state root that +// outlive the test, and the tally under test is the telemetry's alone. func TestTelemetryVersionSpoolsNothingAndCreatesNoDirectory(t *testing.T) { telemetryLifecycleHome(t) + restore := telemetryArgs("version") + defer restore() - if code := telemetryFakeExecute("version"); code != 0 { - t.Fatalf("version exited %d", code) + session := telemetryBegin() + if session.mode != "" { + t.Fatalf("version was taken for a %q session", session.mode) } + telemetryEnd(session, 0) if rows := telemetrySpoolRows(t); len(rows) != 0 { t.Fatalf("version spooled %v", telemetryEventNames(t, rows)) } @@ -316,11 +322,3 @@ func TestTelemetryNoticeNeverPrintsUnderJSON(t *testing.T) { } } -// telemetryFakeExecute runs execute() in-process with os.Args pointed at the -// given words. execute() returns a code and does not exit the process, which -// is what makes this test cheap enough to run alongside the others. -func telemetryFakeExecute(args ...string) int { - restore := telemetryArgs(args...) - defer restore() - return execute() -} From c828d0c5ffa4f249f532bd2c114e751f6ad8944e Mon Sep 17 00:00:00 2001 From: santoshkumarradha Date: Thu, 17 Sep 2026 06:15:40 -0400 Subject: [PATCH 36/36] cmd/codeaf: gofmt the lifecycle test Co-Authored-By: Claude Fable 5.1 --- cmd/codeaf/telemetry_lifecycle_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/codeaf/telemetry_lifecycle_test.go b/cmd/codeaf/telemetry_lifecycle_test.go index d15b7dfdeb..d42f1682d1 100644 --- a/cmd/codeaf/telemetry_lifecycle_test.go +++ b/cmd/codeaf/telemetry_lifecycle_test.go @@ -321,4 +321,3 @@ func TestTelemetryNoticeNeverPrintsUnderJSON(t *testing.T) { t.Fatalf("the notice was marked shown under --json") } } -